From 0ae840934a8175a49088b49a6d8cc4d160039566 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Wed, 9 Dec 2020 13:31:10 +0000 Subject: [PATCH 001/204] [wip] add p2p proof-of-representation --- .../p2p_libp2p/dht/dhtnode/message.pb.go | 337 ++++++++++++++++++ .../p2p_libp2p/dht/dhtnode/message.proto | 25 ++ .../p2p_libp2p/dht/dhtpeer/dhtpeer.go | 72 +++- .../p2p_libp2p/dht/dhtpeer/dhtpeer_test.go | 44 ++- .../fetchai/connections/p2p_libp2p/go.mod | 2 + .../fetchai/connections/p2p_libp2p/go.sum | 1 + .../connections/p2p_libp2p/utils/utils.go | 172 ++++++++- 7 files changed, 638 insertions(+), 15 deletions(-) create mode 100644 packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go create mode 100644 packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go new file mode 100644 index 0000000000..21aa86fb48 --- /dev/null +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go @@ -0,0 +1,337 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc (unknown) +// source: dht/dhtnode/message.proto + +package dhtnode + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type RegisterResponse_Status int32 + +const ( + RegisterResponse_SUCCESS RegisterResponse_Status = 0 + RegisterResponse_ERROR_UNSUPPORTED_VERSION RegisterResponse_Status = 1 + RegisterResponse_ERROR_INVALID_AGENT_ADDRESS RegisterResponse_Status = 2 + RegisterResponse_ERROR_INVALID_PUBLIC_KEY RegisterResponse_Status = 3 + RegisterResponse_ERROR_INVALID_PROOF RegisterResponse_Status = 4 + RegisterResponse_ERROR_GENERIC RegisterResponse_Status = 5 +) + +// Enum value maps for RegisterResponse_Status. +var ( + RegisterResponse_Status_name = map[int32]string{ + 0: "SUCCESS", + 1: "ERROR_UNSUPPORTED_VERSION", + 2: "ERROR_INVALID_AGENT_ADDRESS", + 3: "ERROR_INVALID_PUBLIC_KEY", + 4: "ERROR_INVALID_PROOF", + 5: "ERROR_GENERIC", + } + RegisterResponse_Status_value = map[string]int32{ + "SUCCESS": 0, + "ERROR_UNSUPPORTED_VERSION": 1, + "ERROR_INVALID_AGENT_ADDRESS": 2, + "ERROR_INVALID_PUBLIC_KEY": 3, + "ERROR_INVALID_PROOF": 4, + "ERROR_GENERIC": 5, + } +) + +func (x RegisterResponse_Status) Enum() *RegisterResponse_Status { + p := new(RegisterResponse_Status) + *p = x + return p +} + +func (x RegisterResponse_Status) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (RegisterResponse_Status) Descriptor() protoreflect.EnumDescriptor { + return file_dht_dhtnode_message_proto_enumTypes[0].Descriptor() +} + +func (RegisterResponse_Status) Type() protoreflect.EnumType { + return &file_dht_dhtnode_message_proto_enumTypes[0] +} + +func (x RegisterResponse_Status) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use RegisterResponse_Status.Descriptor instead. +func (RegisterResponse_Status) EnumDescriptor() ([]byte, []int) { + return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{1, 0} +} + +type Register struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ProtocolVersion string `protobuf:"bytes,1,opt,name=protocol_version,json=protocolVersion,proto3" json:"protocol_version,omitempty"` + AgentAddress string `protobuf:"bytes,2,opt,name=agent_address,json=agentAddress,proto3" json:"agent_address,omitempty"` + AgentPublicKey string `protobuf:"bytes,3,opt,name=agent_public_key,json=agentPublicKey,proto3" json:"agent_public_key,omitempty"` + ConnPublicKey string `protobuf:"bytes,4,opt,name=conn_public_key,json=connPublicKey,proto3" json:"conn_public_key,omitempty"` + ProofOfRepresentation string `protobuf:"bytes,5,opt,name=proof_of_representation,json=proofOfRepresentation,proto3" json:"proof_of_representation,omitempty"` +} + +func (x *Register) Reset() { + *x = Register{} + if protoimpl.UnsafeEnabled { + mi := &file_dht_dhtnode_message_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Register) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Register) ProtoMessage() {} + +func (x *Register) ProtoReflect() protoreflect.Message { + mi := &file_dht_dhtnode_message_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Register.ProtoReflect.Descriptor instead. +func (*Register) Descriptor() ([]byte, []int) { + return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{0} +} + +func (x *Register) GetProtocolVersion() string { + if x != nil { + return x.ProtocolVersion + } + return "" +} + +func (x *Register) GetAgentAddress() string { + if x != nil { + return x.AgentAddress + } + return "" +} + +func (x *Register) GetAgentPublicKey() string { + if x != nil { + return x.AgentPublicKey + } + return "" +} + +func (x *Register) GetConnPublicKey() string { + if x != nil { + return x.ConnPublicKey + } + return "" +} + +func (x *Register) GetProofOfRepresentation() string { + if x != nil { + return x.ProofOfRepresentation + } + return "" +} + +type RegisterResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Status RegisterResponse_Status `protobuf:"varint,1,opt,name=status,proto3,enum=dhtnode.RegisterResponse_Status" json:"status,omitempty"` + Msgs []string `protobuf:"bytes,2,rep,name=msgs,proto3" json:"msgs,omitempty"` +} + +func (x *RegisterResponse) Reset() { + *x = RegisterResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_dht_dhtnode_message_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RegisterResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RegisterResponse) ProtoMessage() {} + +func (x *RegisterResponse) ProtoReflect() protoreflect.Message { + mi := &file_dht_dhtnode_message_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RegisterResponse.ProtoReflect.Descriptor instead. +func (*RegisterResponse) Descriptor() ([]byte, []int) { + return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{1} +} + +func (x *RegisterResponse) GetStatus() RegisterResponse_Status { + if x != nil { + return x.Status + } + return RegisterResponse_SUCCESS +} + +func (x *RegisterResponse) GetMsgs() []string { + if x != nil { + return x.Msgs + } + return nil +} + +var File_dht_dhtnode_message_proto protoreflect.FileDescriptor + +var file_dht_dhtnode_message_proto_rawDesc = []byte{ + 0x0a, 0x19, 0x64, 0x68, 0x74, 0x2f, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x07, 0x64, 0x68, 0x74, + 0x6e, 0x6f, 0x64, 0x65, 0x22, 0xe4, 0x01, 0x0a, 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, + 0x72, 0x12, 0x29, 0x0a, 0x10, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x5f, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x0a, 0x0d, + 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, + 0x73, 0x12, 0x28, 0x0a, 0x10, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x70, 0x75, 0x62, 0x6c, 0x69, + 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x61, 0x67, 0x65, + 0x6e, 0x74, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x0f, 0x63, + 0x6f, 0x6e, 0x6e, 0x5f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x6e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, + 0x4b, 0x65, 0x79, 0x12, 0x36, 0x0a, 0x17, 0x70, 0x72, 0x6f, 0x6f, 0x66, 0x5f, 0x6f, 0x66, 0x5f, + 0x72, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x70, 0x72, 0x6f, 0x6f, 0x66, 0x4f, 0x66, 0x52, 0x65, 0x70, + 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x82, 0x02, 0x0a, 0x10, + 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x38, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x20, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, + 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x73, + 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x22, 0x9f, + 0x01, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, + 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, + 0x55, 0x4e, 0x53, 0x55, 0x50, 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x56, 0x45, 0x52, 0x53, + 0x49, 0x4f, 0x4e, 0x10, 0x01, 0x12, 0x1f, 0x0a, 0x1b, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, + 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, 0x44, + 0x52, 0x45, 0x53, 0x53, 0x10, 0x02, 0x12, 0x1c, 0x0a, 0x18, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, + 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x4b, + 0x45, 0x59, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, + 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x4f, 0x46, 0x10, 0x04, 0x12, 0x11, 0x0a, + 0x0d, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x10, 0x05, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_dht_dhtnode_message_proto_rawDescOnce sync.Once + file_dht_dhtnode_message_proto_rawDescData = file_dht_dhtnode_message_proto_rawDesc +) + +func file_dht_dhtnode_message_proto_rawDescGZIP() []byte { + file_dht_dhtnode_message_proto_rawDescOnce.Do(func() { + file_dht_dhtnode_message_proto_rawDescData = protoimpl.X.CompressGZIP(file_dht_dhtnode_message_proto_rawDescData) + }) + return file_dht_dhtnode_message_proto_rawDescData +} + +var file_dht_dhtnode_message_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_dht_dhtnode_message_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_dht_dhtnode_message_proto_goTypes = []interface{}{ + (RegisterResponse_Status)(0), // 0: dhtnode.RegisterResponse.Status + (*Register)(nil), // 1: dhtnode.Register + (*RegisterResponse)(nil), // 2: dhtnode.RegisterResponse +} +var file_dht_dhtnode_message_proto_depIdxs = []int32{ + 0, // 0: dhtnode.RegisterResponse.status:type_name -> dhtnode.RegisterResponse.Status + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_dht_dhtnode_message_proto_init() } +func file_dht_dhtnode_message_proto_init() { + if File_dht_dhtnode_message_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_dht_dhtnode_message_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Register); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_dht_dhtnode_message_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RegisterResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_dht_dhtnode_message_proto_rawDesc, + NumEnums: 1, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_dht_dhtnode_message_proto_goTypes, + DependencyIndexes: file_dht_dhtnode_message_proto_depIdxs, + EnumInfos: file_dht_dhtnode_message_proto_enumTypes, + MessageInfos: file_dht_dhtnode_message_proto_msgTypes, + }.Build() + File_dht_dhtnode_message_proto = out.File + file_dht_dhtnode_message_proto_rawDesc = nil + file_dht_dhtnode_message_proto_goTypes = nil + file_dht_dhtnode_message_proto_depIdxs = nil +} diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto new file mode 100644 index 0000000000..40eb43efd5 --- /dev/null +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package dhtnode; + +message Register { + string protocol_version = 1; + string agent_address = 2; + string agent_public_key = 3; + string conn_public_key = 4; + string proof_of_representation = 5; +} + +message RegisterResponse { + enum Status { + SUCCESS = 0; + ERROR_UNSUPPORTED_VERSION = 1; + ERROR_INVALID_AGENT_ADDRESS = 2; + ERROR_INVALID_PUBLIC_KEY = 3; + ERROR_INVALID_PROOF = 4; + ERROR_GENERIC = 5; + } + + Status status = 1; + repeated string msgs = 2; +} \ No newline at end of file diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go index 75f6bcb5dd..ce4a2a6f02 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go @@ -35,6 +35,7 @@ import ( "sync" "time" + "github.com/golang/protobuf/proto" "github.com/rs/zerolog" "github.com/libp2p/go-libp2p" @@ -505,10 +506,75 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { return } - addr := string(buf) + registration := &dhtnode.Register{} + err = proto.Unmarshal(buf, registration) + if err != nil { + lerror(err).Msg("couldn't deserialize registration message") + response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_ERROR_GENERIC} + buf, err = proto.Marshal(response) + err = utils.WriteBytesConn(conn, buf) + ignore(err) + + nbrConns.Dec() + return + } + + linfo().Msgf("Received registration request %s", registration) + + addr := registration.AgentAddress linfo().Msgf("connection from %s established for Address %s", conn.RemoteAddr().String(), addr) + // check that agent address and public key match + addrFromPubKey, err := utils.FetchAIAddressFromPublicKey(registration.AgentPublicKey) + if err != nil || addrFromPubKey != registration.AgentAddress { + if err == nil { + err = errors.New("Agent address and public key don't match") + } + lerror(err).Msg("Couldn't verify agent address and public key association") + response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_ERROR_INVALID_AGENT_ADDRESS} + buf, err = proto.Marshal(response) + err = utils.WriteBytesConn(conn, buf) + ignore(err) + + nbrConns.Dec() + return + } + + // check that connection public key match + connPubKey, err := utils.PubKeyFromFetchAIPublicKey(registration.ConnPublicKey) + if err != nil || !connPubKey.Equals(dhtPeer.publicKey) { + if err == nil { + err = errors.New("Registration connection public key doesn't match peer key") + } + lerror(err).Msg("Couldn't verify agent address and public key association") + response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_ERROR_INVALID_PUBLIC_KEY} + buf, err = proto.Marshal(response) + err = utils.WriteBytesConn(conn, buf) + ignore(err) + + nbrConns.Dec() + return + + } + + // check that signature is valid + ok, err := utils.VerifyFetchAISignatureBTC([]byte(registration.ConnPublicKey), registration.ProofOfRepresentation, registration.AgentPublicKey) + if !ok || err != nil { + if err == nil { + err = errors.New("Proof of representation of peer public key is not valid") + } + lerror(err).Msg("Couldn't verify agent address and public key association") + response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_ERROR_INVALID_PROOF} + buf, err = proto.Marshal(response) + err = utils.WriteBytesConn(conn, buf) + ignore(err) + + nbrConns.Dec() + return + + } + // Add connection to map dhtPeer.tcpAddressesLock.Lock() dhtPeer.tcpAddresses[addr] = conn @@ -522,7 +588,9 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { } } - err = utils.WriteBytesConn(conn, []byte("DONE")) + response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_SUCCESS} + buf, err = proto.Marshal(response) + err = utils.WriteBytesConn(conn, buf) ignore(err) duration := timer.GetTimer(start) diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go index 11ecb814d2..2fb4996aed 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go @@ -29,7 +29,10 @@ import ( "libp2p_node/aea" "libp2p_node/dht/dhtclient" + "libp2p_node/dht/dhtnode" "libp2p_node/utils" + + "github.com/golang/protobuf/proto" ) /* @@ -73,12 +76,37 @@ var ( "roiuioMXPhu1PRFSYqpnMgvUrDCmRY3canmBQu16CTZozyQAc", "2LcDvsoiTmUPkFFdMTAGEUdZY7Y2xyYCQxEXvLD8MoMhTe4Ldi", } + + AgentsTestKeys = []string{} ) /* DHT Network: DHTPeer-to-DHTPeer */ +func TestFetchAICrypto(t *testing.T) { + publicKey := "02358e3e42a6ba15cf6b2ba6eb05f02b8893acf82b316d7dd9cda702b0892b8c71" + address := "fetch19dq2mkcpp6x0aypxt9c9gz6n4fqvax0x9a7t5r" + connPublicKey := "027af21aff853b9d9589867ea142b0a60a9611fc8e1fae04c2f7144113fa4e938e" + signatureConnPublicKey_str_canonize := "N/GOa7/m3HU8/gpLJ88VCQ6vXsdrfiiYcqnNtF+c2N9VG9ZIiycykN4hdbpbOCGrChMYZQA3G1GpozsShrUBgg==" + + addressFromPublicKey, _ := utils.FetchAIAddressFromPublicKey(publicKey) + if address != addressFromPublicKey { + t.Error("[ERR] Addresses don't match") + } else { + t.Log("[OK] Agent address matches its public key") + } + + valid, err := utils.VerifyFetchAISignatureBTC([]byte(connPublicKey), signatureConnPublicKey_str_canonize, publicKey) + if !valid { + t.Errorf("Signature using BTC don't match %s", err.Error()) + } + valid, err = utils.VerifyFetchAISignatureLibp2p([]byte(connPublicKey), signatureConnPublicKey_str_canonize, publicKey) + if !valid { + t.Errorf("Signature using LPP don't match %s", err.Error()) + } +} + // TestRoutingDHTPeerToSelf dht peer with agent attached func TestRoutingDHTPeerToSelf(t *testing.T) { opts := []Option{ @@ -1392,12 +1420,24 @@ func SetupDelegateClient(address string, host string, port uint16) (*DelegateCli return nil, nil, err } - err = utils.WriteBytesConn(client.Conn, []byte(address)) + registration := &dhtnode.Register{ProtocolVersion: "0.1.0"} + registration.AgentAddress = address + registration.AgentPublicKey = "AgentPublicKey" + registration.ConnPublicKey = "ConnPublicKey" + registration.ProofOfRepresentation = "signature" + data, err := proto.Marshal(registration) + + err = utils.WriteBytesConn(client.Conn, data) ignore(err) - _, err = utils.ReadBytesConn(client.Conn) + data, err = utils.ReadBytesConn(client.Conn) if err != nil { return nil, nil, err } + response := &dhtnode.RegisterResponse{} + err = proto.Unmarshal(data, response) + if response.Status != dhtnode.RegisterResponse_SUCCESS { + println("Registration error") + } go func() { for { diff --git a/packages/fetchai/connections/p2p_libp2p/go.mod b/packages/fetchai/connections/p2p_libp2p/go.mod index 32dcfe257a..8afca70fef 100644 --- a/packages/fetchai/connections/p2p_libp2p/go.mod +++ b/packages/fetchai/connections/p2p_libp2p/go.mod @@ -4,6 +4,7 @@ go 1.13 require ( github.com/btcsuite/btcd v0.20.1-beta + github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d github.com/golang/protobuf v1.4.2 github.com/ipfs/go-cid v0.0.5 github.com/joho/godotenv v1.3.0 @@ -15,5 +16,6 @@ require ( github.com/multiformats/go-multihash v0.0.13 github.com/prometheus/client_golang v1.7.1 github.com/rs/zerolog v1.19.0 + golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d google.golang.org/protobuf v1.25.0 ) diff --git a/packages/fetchai/connections/p2p_libp2p/go.sum b/packages/fetchai/connections/p2p_libp2p/go.sum index dbcdd0cd1c..e0ef651797 100644 --- a/packages/fetchai/connections/p2p_libp2p/go.sum +++ b/packages/fetchai/connections/p2p_libp2p/go.sum @@ -22,6 +22,7 @@ github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQj github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= diff --git a/packages/fetchai/connections/p2p_libp2p/utils/utils.go b/packages/fetchai/connections/p2p_libp2p/utils/utils.go index 99acea128c..ad56a1e15c 100644 --- a/packages/fetchai/connections/p2p_libp2p/utils/utils.go +++ b/packages/fetchai/connections/p2p_libp2p/utils/utils.go @@ -23,6 +23,8 @@ package utils import ( "bufio" "context" + "crypto/sha256" + "encoding/base64" "encoding/binary" "encoding/hex" "errors" @@ -40,11 +42,13 @@ import ( "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multihash" "github.com/rs/zerolog" + "golang.org/x/crypto/ripemd160" host "github.com/libp2p/go-libp2p-core/host" peerstore "github.com/libp2p/go-libp2p-core/peerstore" btcec "github.com/btcsuite/btcd/btcec" + "github.com/btcsuite/btcutil/bech32" proto "google.golang.org/protobuf/proto" "libp2p_node/aea" @@ -184,6 +188,117 @@ func ComputeCID(addr string) (cid.Cid, error) { return c, nil } +// GetPeersAddrInfo Parse multiaddresses and convert them to peer.AddrInfo +func GetPeersAddrInfo(peers []string) ([]peer.AddrInfo, error) { + pinfos := make([]peer.AddrInfo, len(peers)) + for i, addr := range peers { + maddr := multiaddr.StringCast(addr) + p, err := peer.AddrInfoFromP2pAddr(maddr) + if err != nil { + return pinfos, err + } + pinfos[i] = *p + } + return pinfos, nil +} + +/* + FetchAI Crypto Helpers +*/ + +// PubKeyFromFetchAIPublicKey create libp2p public key from fetchai hex encoded secp256k1 key +func PubKeyFromFetchAIPublicKey(publicKey string) (crypto.PubKey, error) { + hexBytes, _ := hex.DecodeString(publicKey) + return crypto.UnmarshalSecp256k1PublicKey(hexBytes) +} + +// BTCPubKeyFromFetchAIPublicKey +func BTCPubKeyFromFetchAIPublicKey(publicKey string) (*btcec.PublicKey, error) { + pbkBytes, err := hex.DecodeString(publicKey) + if err != nil { + return nil, err + } + pbk, err := btcec.ParsePubKey(pbkBytes, btcec.S256()) + return pbk, err +} + +// ConvertStrEncodedSignatureToDER +// References: +// - https://github.com/fetchai/agents-aea/blob/master/aea/crypto/cosmos.py#L258 +// - https://github.com/btcsuite/btcd/blob/master/btcec/signature.go#L47 +func ConvertStrEncodedSignatureToDER(signature []byte) []byte { + rb := signature[:len(signature)/2] + sb := signature[len(signature)/2:] + length := 6 + len(rb) + len(sb) + sigDER := make([]byte, length) + sigDER[0] = 0x30 + sigDER[1] = byte(length - 2) + sigDER[2] = 0x02 + sigDER[3] = byte(len(rb)) + offset := copy(sigDER[4:], rb) + 4 + sigDER[offset] = 0x02 + sigDER[offset+1] = byte(len(sb)) + copy(sigDER[offset+2:], sb) + return sigDER +} + +// ParseFetchAISignature create btcec Signature from base64 formated, string (not DER) encoded RFC6979 signature +func ParseFetchAISignature(signature string) (*btcec.Signature, error) { + // First convert the signature into a DER one + sigBytes, err := base64.StdEncoding.DecodeString(signature) + if err != nil { + return nil, err + } + sigDER := ConvertStrEncodedSignatureToDER(sigBytes) + + // Parse + sigBTC, err := btcec.ParseSignature(sigDER, btcec.S256()) + return sigBTC, err +} + +// VerifyFetchAISignatureBTC verify the RFC6967 string-encoded signature of message using FetchAI public key +func VerifyFetchAISignatureBTC(message []byte, signature string, pubkey string) (bool, error) { + // construct verifying key + verifyKey, err := BTCPubKeyFromFetchAIPublicKey(pubkey) + if err != nil { + return false, err + } + + // construct signature + signatureBTC, err := ParseFetchAISignature(signature) + if err != nil { + return false, err + } + + // verify signature + messageHash := sha256.New() + _, err = messageHash.Write([]byte(message)) + if err != nil { + return false, err + } + + return signatureBTC.Verify(messageHash.Sum(nil), verifyKey), nil +} + +// VerifyFetchAISignatureLibp2p verify RFC6967 string-encoded signature of message using FetchAI public key +func VerifyFetchAISignatureLibp2p(message []byte, signature string, pubkey string) (bool, error) { + // construct verifying key + verifyKey, err := PubKeyFromFetchAIPublicKey(pubkey) + if err != nil { + return false, err + } + + // Convert signature into DER encoding + sigBytes, err := base64.StdEncoding.DecodeString(signature) + if err != nil { + return false, err + } + sigDER := ConvertStrEncodedSignatureToDER(sigBytes) + + // verify signature + return verifyKey.Verify(message, sigDER) +} + // KeyPairFromFetchAIKey key pair from hex encoded secp256k1 private key func KeyPairFromFetchAIKey(key string) (crypto.PrivKey, crypto.PubKey, error) { pk_bytes, err := hex.DecodeString(key) @@ -200,18 +315,33 @@ func KeyPairFromFetchAIKey(key string) (crypto.PrivKey, crypto.PubKey, error) { return prvKey, pubKey, nil } -// GetPeersAddrInfo Parse multiaddresses and convert them to peer.AddrInfo -func GetPeersAddrInfo(peers []string) ([]peer.AddrInfo, error) { - pinfos := make([]peer.AddrInfo, len(peers)) - for i, addr := range peers { - maddr := multiaddr.StringCast(addr) - p, err := peer.AddrInfoFromP2pAddr(maddr) - if err != nil { - return pinfos, err - } - pinfos[i] = *p +// FetchAIAddressFromPublicKey get wallet address from hex encoded secp256k1 public key +// format from: https://github.com/fetchai/agents-aea/blob/master/aea/crypto/cosmos.py#L120 +func FetchAIAddressFromPublicKey(publicKey string) (string, error) { + var addr string + var err error + hexBytes, err := hex.DecodeString(publicKey) + if err != nil { + return addr, err } - return pinfos, nil + hash := sha256.New() + _, err = hash.Write(hexBytes) + if err != nil { + return addr, err + } + sha256Hash := hash.Sum(nil) + hash = ripemd160.New() + _, err = hash.Write(sha256Hash) + if err != nil { + return addr, err + } + ripemd160Hash := hash.Sum(nil) + fiveBitsChar, err := bech32.ConvertBits(ripemd160Hash, 8, 5, true) + if err != nil { + return addr, err + } + addr, err = bech32.Encode("fetch", fiveBitsChar) + return addr, err } // IDFromFetchAIPublicKey Get PeeID (multihash) from fetchai public key @@ -221,6 +351,26 @@ func IDFromFetchAIPublicKey(public_key string) (peer.ID, error) { return "", err } + pub_key, err := btcec.ParsePubKey(b, btcec.S256()) + if err != nil { + return "", err + } + + multihash, err := peer.IDFromPublicKey((*crypto.Secp256k1PublicKey)(pub_key)) + if err != nil { + return "", err + } + + return multihash, nil +} + +// IDFromFetchAIPublicKeyUncompressed Get PeeID (multihash) from fetchai public key +func IDFromFetchAIPublicKeyUncompressed(public_key string) (peer.ID, error) { + b, err := hex.DecodeString(public_key) + if err != nil { + return "", err + } + pub_bytes := make([]byte, 0, btcec.PubKeyBytesLenUncompressed) pub_bytes = append(pub_bytes, 0x4) // btcec.pubkeyUncompressed pub_bytes = append(pub_bytes, b...) From bc1e419d7e6046e11c0eb62a4d09866aa3b67167 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Fri, 11 Dec 2020 20:58:00 +0000 Subject: [PATCH 002/204] Add registation with proof-of-representation for Delegate service --- .../p2p_libp2p/dht/dhtpeer/dhtpeer.go | 71 ++++++----- .../p2p_libp2p/dht/dhtpeer/dhtpeer_test.go | 118 +++++++++++++++--- .../fetchai/connections/p2p_libp2p/go.mod | 1 + .../connections/p2p_libp2p/utils/utils.go | 26 ++++ 4 files changed, 166 insertions(+), 50 deletions(-) diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go index ce4a2a6f02..f54833296a 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go @@ -481,6 +481,35 @@ func (dhtPeer *DHTPeer) handleDelegateService(ready *sync.WaitGroup) { } } +func isValidProofOfRepresentation(registration *dhtnode.Register) (*dhtnode.RegisterResponse, error) { + + // check that agent address and public key match + addrFromPubKey, err := utils.FetchAIAddressFromPublicKey(registration.AgentPublicKey) + if err != nil || addrFromPubKey != registration.AgentAddress { + if err == nil { + err = errors.New("Agent address and public key don't match") + } + response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_ERROR_INVALID_AGENT_ADDRESS} + return response, err + } + + // check that signature is valid + ok, err := utils.VerifyFetchAISignatureBTC([]byte(registration.ConnPublicKey), registration.ProofOfRepresentation, registration.AgentPublicKey) + if !ok || err != nil { + if err == nil { + err = errors.New("Signature is not valid") + } + response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_ERROR_INVALID_PROOF} + return response, err + + } + + // PoR is valid + response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_SUCCESS} + return response, nil + +} + func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { defer dhtPeer.goroutines.Done() defer conn.Close() @@ -498,7 +527,7 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { //linfo().Msgf("received a new connection from %s", conn.RemoteAddr().String()) - // read agent address + // read agent registration message buf, err := utils.ReadBytesConn(conn) if err != nil { lerror(err).Msg("while receiving agent's Address") @@ -525,22 +554,6 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { linfo().Msgf("connection from %s established for Address %s", conn.RemoteAddr().String(), addr) - // check that agent address and public key match - addrFromPubKey, err := utils.FetchAIAddressFromPublicKey(registration.AgentPublicKey) - if err != nil || addrFromPubKey != registration.AgentAddress { - if err == nil { - err = errors.New("Agent address and public key don't match") - } - lerror(err).Msg("Couldn't verify agent address and public key association") - response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_ERROR_INVALID_AGENT_ADDRESS} - buf, err = proto.Marshal(response) - err = utils.WriteBytesConn(conn, buf) - ignore(err) - - nbrConns.Dec() - return - } - // check that connection public key match connPubKey, err := utils.PubKeyFromFetchAIPublicKey(registration.ConnPublicKey) if err != nil || !connPubKey.Equals(dhtPeer.publicKey) { @@ -555,24 +568,25 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { nbrConns.Dec() return - } - // check that signature is valid - ok, err := utils.VerifyFetchAISignatureBTC([]byte(registration.ConnPublicKey), registration.ProofOfRepresentation, registration.AgentPublicKey) - if !ok || err != nil { - if err == nil { - err = errors.New("Proof of representation of peer public key is not valid") - } - lerror(err).Msg("Couldn't verify agent address and public key association") - response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_ERROR_INVALID_PROOF} + // check if the PoR is valid + response, err := isValidProofOfRepresentation(registration) + if err != nil { + lerror(err).Msg("PoR is not valid") buf, err = proto.Marshal(response) err = utils.WriteBytesConn(conn, buf) ignore(err) nbrConns.Dec() return + } + buf, err = proto.Marshal(response) + err = utils.WriteBytesConn(conn, buf) + if err != nil { + nbrConns.Dec() + return } // Add connection to map @@ -588,11 +602,6 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { } } - response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_SUCCESS} - buf, err = proto.Marshal(response) - err = utils.WriteBytesConn(conn, buf) - ignore(err) - duration := timer.GetTimer(start) opLatencyRegister.Observe(float64(duration.Microseconds())) diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go index 2fb4996aed..45773d1975 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go @@ -22,6 +22,7 @@ package dhtpeer import ( "context" + "encoding/base64" "net" "strconv" "testing" @@ -33,6 +34,7 @@ import ( "libp2p_node/utils" "github.com/golang/protobuf/proto" + "github.com/pkg/errors" ) /* @@ -64,20 +66,31 @@ var ( "9427c1472b66f6abd94a6c246eee495e3709ec45882ae0badcbc71ad2cd8f8b2", } + AgentsTestKeys = []string{ + "730c22474709a6d17cf11599a80413a84ddb691a3c7b11a6d8d47a2c024b7b56", + "a085c5eeb39636a21c85a9bc667bae18bf3e327a220ecb3998e317b62ab20ec6", + "0b7af750e7e96ceb9fe5582bdf9bdafae726427d34447f7245a084b6cf0aa5e5", + "dffaa5a9779931a2c1194794e6e9a89787557d6cd708d84c74de20ec5e03a7bf", + "509c4019dd96a337a36149031869e6de5db014ab9ae5d8097ac997ca8f10422a", + "a385fa48b4f40a2f4ea66de88c0021532299865fe6137d765788f9f856e79453", + "ff212371e454f8292fd3b13020a3910fc91002a7ab5eb3f297b71df6b7ff9bc1", + "04289e97041fc025c103141909d2cce649944153822f032b646214a850363618", + "116294510fba759d19af7a65b915467384258d997695ed7018d8c19d38c29412", + "dc2f0238e65c0291bedae58cb1c013bd03e0f41f78e1779744ac401952ec2b51", + } + AgentsTestAddresses = []string{ - "PMRHuYJRhrbRHDagMMtkwfdFwJi7cbG9oxWkf9Au5zTi4kqng", - "19SkNL4ozZbnL3xenQiCq8267KDmRpy1BTFtQoYRbVruXDamH", - "2U1id59VqSx4cD6pxRDGnDQJA8UQj1r8X4iyti7k4F6u3Aayfb", - "sgaaoJ3rW3g9vkvUdUMTqMW6ZTD3bdnr6Drg8Ro9FcenNo6RM", - "2Rn9GTp5NHt8B8k4w5Ct44RrDKErRYsu5sgBrHAqBTkfCCKqLP", - "2sTsbPFCxbfVUENtLt62bNjTYFPffdASbZAUGast4ZZUVdkN4r", - "2EBBRDJWJ3NoRUJK1sjNh6gi3iRpMcUHqGU9JHiuuvVyuZyA4n", - "fTFcTd8wJ4PmiffhTwFhP2J45A6V6XuMDWrA59hheHaWgdrPv", - "roiuioMXPhu1PRFSYqpnMgvUrDCmRY3canmBQu16CTZozyQAc", - "2LcDvsoiTmUPkFFdMTAGEUdZY7Y2xyYCQxEXvLD8MoMhTe4Ldi", - } - - AgentsTestKeys = []string{} + "fetch1y39e4tec9fll66x2k7wed5qn7zhaneayjm55kk", + "fetch1ufjmhth6dnhrckxrvk05lmt8s2vture23xvwjl", + "fetch1dja5uazc9n7jpjm94rhmkkmcyv5nj3kt8aexgf", + "fetch18v5lz9psp53akm26ztk3exytqfdvpnfdsyx232", + "fetch10u6ra4qmukhf57xadv64jt9jhr9gdrg707x6l9", + "fetch1hys3k2anw5mxe0y2vksccpe58jyk5gksrsjd60", + "fetch1t07jnjjtlqa07mstg4gw9twjs2ddtqs3sgtx7c", + "fetch18sxxgat6uaxqxvd7mgt99y7avyy3c24av36u2l", + "fetch1sx2rmtndc5t97pn00x76sksrzgc9s2watpgw64", + "fetch1mwd8n27t68svv4w5urztgw7e3kjh7nqkqz0j94", + } ) /* @@ -627,8 +640,25 @@ func TestRoutingDHTClientToDHTClientIndirect(t *testing.T) { DHT network: DelegateClient */ +func signFetchAI(message []byte, privKey string) (string, error) { + signingKey, _, err := utils.KeyPairFromFetchAIKey(privKey) + if err != nil { + return "", err + } + signature, err := signingKey.Sign(message) + if err != nil { + return "", err + } + strSignature, err := utils.ConvertDEREncodedSignatureToStr(signature) + if err != nil { + return "", err + } + encodedSignature := base64.StdEncoding.EncodeToString(strSignature) + return encodedSignature, nil +} + // TestRoutingDelegateClientToDHTPeer -func TestRoutingDelegateClientToDHTPeer(t *testing.T) { +func TestRoutingDelegateClientToDHTPeerX(t *testing.T) { peer, peerCleanup, err := SetupLocalDHTPeer( FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, []string{}, @@ -638,7 +668,21 @@ func TestRoutingDelegateClientToDHTPeer(t *testing.T) { } defer peerCleanup() - client, clientCleanup, err := SetupDelegateClient(AgentsTestAddresses[1], DefaultLocalHost, DefaultDelegatePort) + peerPubKey, err := utils.FetchAIPublicKeyFromFetchAIPrivateKey(FetchAITestKeys[0]) + if err != nil { + t.Fatal("Failed to get public key from DHTPeer:", err) + } + + agentPublicKey, err := utils.FetchAIPublicKeyFromFetchAIPrivateKey(AgentsTestKeys[1]) + if err != nil { + t.Fatal("Failed to get public key from Agent key:", err) + } + + signature, err := signFetchAI([]byte(peerPubKey), AgentsTestKeys[1]) + if err != nil { + t.Fatal("Failed to sign peer public key:", err) + } + client, clientCleanup, err := SetupDelegateClientWithPoR(AgentsTestAddresses[1], DefaultLocalHost, DefaultDelegatePort, agentPublicKey, peerPubKey, signature) if err != nil { t.Fatal("Failed to initialize DelegateClient:", err) } @@ -1420,11 +1464,46 @@ func SetupDelegateClient(address string, host string, port uint16) (*DelegateCli return nil, nil, err } + err = utils.WriteBytesConn(client.Conn, []byte(address)) + ignore(err) + _, err = utils.ReadBytesConn(client.Conn) + if err != nil { + return nil, nil, err + } + + go func() { + for { + envel, err := utils.ReadEnvelopeConn(client.Conn) + if err != nil { + break + } + if client.processEnvelope != nil { + err = client.processEnvelope(envel) + ignore(err) + } else { + client.Rx <- envel + } + } + }() + + return client, func() { client.Close() }, nil +} + +func SetupDelegateClientWithPoR(address string, host string, port uint16, pubkey string, connKey string, sig string) (*DelegateClient, func(), error) { + var err error + client := &DelegateClient{} + client.AgentAddress = address + client.Rx = make(chan *aea.Envelope) + client.Conn, err = net.Dial("tcp", host+":"+strconv.FormatInt(int64(port), 10)) + if err != nil { + return nil, nil, err + } + registration := &dhtnode.Register{ProtocolVersion: "0.1.0"} registration.AgentAddress = address - registration.AgentPublicKey = "AgentPublicKey" - registration.ConnPublicKey = "ConnPublicKey" - registration.ProofOfRepresentation = "signature" + registration.AgentPublicKey = pubkey + registration.ConnPublicKey = connKey + registration.ProofOfRepresentation = sig data, err := proto.Marshal(registration) err = utils.WriteBytesConn(client.Conn, data) @@ -1436,7 +1515,8 @@ func SetupDelegateClient(address string, host string, port uint16) (*DelegateCli response := &dhtnode.RegisterResponse{} err = proto.Unmarshal(data, response) if response.Status != dhtnode.RegisterResponse_SUCCESS { - println("Registration error") + println("Registration error:", response.Status.String()) + return nil, nil, errors.New(response.Status.String()) } go func() { diff --git a/packages/fetchai/connections/p2p_libp2p/go.mod b/packages/fetchai/connections/p2p_libp2p/go.mod index 8afca70fef..1ae69cece4 100644 --- a/packages/fetchai/connections/p2p_libp2p/go.mod +++ b/packages/fetchai/connections/p2p_libp2p/go.mod @@ -14,6 +14,7 @@ require ( github.com/libp2p/go-libp2p-kad-dht v0.7.11 github.com/multiformats/go-multiaddr v0.2.1 github.com/multiformats/go-multihash v0.0.13 + github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.7.1 github.com/rs/zerolog v1.19.0 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d diff --git a/packages/fetchai/connections/p2p_libp2p/utils/utils.go b/packages/fetchai/connections/p2p_libp2p/utils/utils.go index ad56a1e15c..e60eecf6bc 100644 --- a/packages/fetchai/connections/p2p_libp2p/utils/utils.go +++ b/packages/fetchai/connections/p2p_libp2p/utils/utils.go @@ -242,6 +242,22 @@ func ConvertStrEncodedSignatureToDER(signature []byte) []byte { return sigDER } +// ConvertDEREncodedSignatureToStr +// References: +// - https://github.com/fetchai/agents-aea/blob/master/aea/crypto/cosmos.py#L258 +// - https://github.com/btcsuite/btcd/blob/master/btcec/signature.go#L47 +func ConvertDEREncodedSignatureToStr(signature []byte) ([]byte, error) { + length := len(signature) - 6 + if length%2 != 0 { + return []byte{}, errors.New("DER signature length must be even") + } + sigStr := make([]byte, length) + offset := 4 + length/2 + copy(sigStr, signature[4:offset]) + copy(sigStr[length/2:], signature[offset+2:]) + return sigStr, nil +} + // ParseFetchAISignature create btcec Signature from base64 formated, string (not DER) encoded RFC6979 signature func ParseFetchAISignature(signature string) (*btcec.Signature, error) { // First convert the signature into a DER one @@ -388,6 +404,16 @@ func IDFromFetchAIPublicKeyUncompressed(public_key string) (peer.ID, error) { return multihash, nil } +func FetchAIPublicKeyFromFetchAIPrivateKey(privateKey string) (string, error) { + pkBytes, err := hex.DecodeString(privateKey) + if err != nil { + return "", err + } + _, btcPublicKey := btcec.PrivKeyFromBytes(btcec.S256(), pkBytes) + + return hex.EncodeToString(btcPublicKey.SerializeCompressed()), nil +} + /* Utils */ From 60af99b1928f60b797e0608aedd2c3b437b41923 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Mon, 14 Dec 2020 09:34:18 +0000 Subject: [PATCH 003/204] Refactor Acn proto messages for PoR and registration --- .../p2p_libp2p/dht/dhtnode/message.pb.go | 405 +++++++++++++----- .../p2p_libp2p/dht/dhtnode/message.proto | 40 +- .../p2p_libp2p/dht/dhtpeer/dhtpeer.go | 52 ++- .../p2p_libp2p/dht/dhtpeer/dhtpeer_test.go | 89 ++-- 4 files changed, 420 insertions(+), 166 deletions(-) diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go index 21aa86fb48..5b6e5f4ada 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go @@ -25,78 +25,83 @@ const ( // of the legacy proto package is being used. const _ = proto.ProtoPackageIsVersion4 -type RegisterResponse_Status int32 +type Status_ErrCode int32 const ( - RegisterResponse_SUCCESS RegisterResponse_Status = 0 - RegisterResponse_ERROR_UNSUPPORTED_VERSION RegisterResponse_Status = 1 - RegisterResponse_ERROR_INVALID_AGENT_ADDRESS RegisterResponse_Status = 2 - RegisterResponse_ERROR_INVALID_PUBLIC_KEY RegisterResponse_Status = 3 - RegisterResponse_ERROR_INVALID_PROOF RegisterResponse_Status = 4 - RegisterResponse_ERROR_GENERIC RegisterResponse_Status = 5 + // common (0x) + Status_SUCCESS Status_ErrCode = 0 + Status_ERROR_UNSUPPORTED_VERSION Status_ErrCode = 1 + Status_ERROR_UNEXPECTED_PAYLOAD Status_ErrCode = 2 + Status_ERROR_GENERIC Status_ErrCode = 3 + // register (1x) + Status_ERROR_INVALID_AGENT_ADDRESS Status_ErrCode = 10 + Status_ERROR_INVALID_PUBLIC_KEY Status_ErrCode = 11 + Status_ERROR_INVALID_PROOF Status_ErrCode = 12 // ) -// Enum value maps for RegisterResponse_Status. +// Enum value maps for Status_ErrCode. var ( - RegisterResponse_Status_name = map[int32]string{ - 0: "SUCCESS", - 1: "ERROR_UNSUPPORTED_VERSION", - 2: "ERROR_INVALID_AGENT_ADDRESS", - 3: "ERROR_INVALID_PUBLIC_KEY", - 4: "ERROR_INVALID_PROOF", - 5: "ERROR_GENERIC", + Status_ErrCode_name = map[int32]string{ + 0: "SUCCESS", + 1: "ERROR_UNSUPPORTED_VERSION", + 2: "ERROR_UNEXPECTED_PAYLOAD", + 3: "ERROR_GENERIC", + 10: "ERROR_INVALID_AGENT_ADDRESS", + 11: "ERROR_INVALID_PUBLIC_KEY", + 12: "ERROR_INVALID_PROOF", } - RegisterResponse_Status_value = map[string]int32{ + Status_ErrCode_value = map[string]int32{ "SUCCESS": 0, "ERROR_UNSUPPORTED_VERSION": 1, - "ERROR_INVALID_AGENT_ADDRESS": 2, - "ERROR_INVALID_PUBLIC_KEY": 3, - "ERROR_INVALID_PROOF": 4, - "ERROR_GENERIC": 5, + "ERROR_UNEXPECTED_PAYLOAD": 2, + "ERROR_GENERIC": 3, + "ERROR_INVALID_AGENT_ADDRESS": 10, + "ERROR_INVALID_PUBLIC_KEY": 11, + "ERROR_INVALID_PROOF": 12, } ) -func (x RegisterResponse_Status) Enum() *RegisterResponse_Status { - p := new(RegisterResponse_Status) +func (x Status_ErrCode) Enum() *Status_ErrCode { + p := new(Status_ErrCode) *p = x return p } -func (x RegisterResponse_Status) String() string { +func (x Status_ErrCode) String() string { return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } -func (RegisterResponse_Status) Descriptor() protoreflect.EnumDescriptor { +func (Status_ErrCode) Descriptor() protoreflect.EnumDescriptor { return file_dht_dhtnode_message_proto_enumTypes[0].Descriptor() } -func (RegisterResponse_Status) Type() protoreflect.EnumType { +func (Status_ErrCode) Type() protoreflect.EnumType { return &file_dht_dhtnode_message_proto_enumTypes[0] } -func (x RegisterResponse_Status) Number() protoreflect.EnumNumber { +func (x Status_ErrCode) Number() protoreflect.EnumNumber { return protoreflect.EnumNumber(x) } -// Deprecated: Use RegisterResponse_Status.Descriptor instead. -func (RegisterResponse_Status) EnumDescriptor() ([]byte, []int) { - return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{1, 0} +// Deprecated: Use Status_ErrCode.Descriptor instead. +func (Status_ErrCode) EnumDescriptor() ([]byte, []int) { + return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{2, 0} } -type Register struct { +type AgentRecord struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ProtocolVersion string `protobuf:"bytes,1,opt,name=protocol_version,json=protocolVersion,proto3" json:"protocol_version,omitempty"` - AgentAddress string `protobuf:"bytes,2,opt,name=agent_address,json=agentAddress,proto3" json:"agent_address,omitempty"` - AgentPublicKey string `protobuf:"bytes,3,opt,name=agent_public_key,json=agentPublicKey,proto3" json:"agent_public_key,omitempty"` - ConnPublicKey string `protobuf:"bytes,4,opt,name=conn_public_key,json=connPublicKey,proto3" json:"conn_public_key,omitempty"` - ProofOfRepresentation string `protobuf:"bytes,5,opt,name=proof_of_representation,json=proofOfRepresentation,proto3" json:"proof_of_representation,omitempty"` + ServiceId string `protobuf:"bytes,1,opt,name=service_id,json=serviceId,proto3" json:"service_id,omitempty"` + Address string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"` + PublicKey string `protobuf:"bytes,3,opt,name=public_key,json=publicKey,proto3" json:"public_key,omitempty"` + PeerPublicKey string `protobuf:"bytes,4,opt,name=peer_public_key,json=peerPublicKey,proto3" json:"peer_public_key,omitempty"` + Signature string `protobuf:"bytes,5,opt,name=signature,proto3" json:"signature,omitempty"` } -func (x *Register) Reset() { - *x = Register{} +func (x *AgentRecord) Reset() { + *x = AgentRecord{} if protoimpl.UnsafeEnabled { mi := &file_dht_dhtnode_message_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -104,13 +109,13 @@ func (x *Register) Reset() { } } -func (x *Register) String() string { +func (x *AgentRecord) String() string { return protoimpl.X.MessageStringOf(x) } -func (*Register) ProtoMessage() {} +func (*AgentRecord) ProtoMessage() {} -func (x *Register) ProtoReflect() protoreflect.Message { +func (x *AgentRecord) ProtoReflect() protoreflect.Message { mi := &file_dht_dhtnode_message_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -122,57 +127,56 @@ func (x *Register) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use Register.ProtoReflect.Descriptor instead. -func (*Register) Descriptor() ([]byte, []int) { +// Deprecated: Use AgentRecord.ProtoReflect.Descriptor instead. +func (*AgentRecord) Descriptor() ([]byte, []int) { return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{0} } -func (x *Register) GetProtocolVersion() string { +func (x *AgentRecord) GetServiceId() string { if x != nil { - return x.ProtocolVersion + return x.ServiceId } return "" } -func (x *Register) GetAgentAddress() string { +func (x *AgentRecord) GetAddress() string { if x != nil { - return x.AgentAddress + return x.Address } return "" } -func (x *Register) GetAgentPublicKey() string { +func (x *AgentRecord) GetPublicKey() string { if x != nil { - return x.AgentPublicKey + return x.PublicKey } return "" } -func (x *Register) GetConnPublicKey() string { +func (x *AgentRecord) GetPeerPublicKey() string { if x != nil { - return x.ConnPublicKey + return x.PeerPublicKey } return "" } -func (x *Register) GetProofOfRepresentation() string { +func (x *AgentRecord) GetSignature() string { if x != nil { - return x.ProofOfRepresentation + return x.Signature } return "" } -type RegisterResponse struct { +type Register struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Status RegisterResponse_Status `protobuf:"varint,1,opt,name=status,proto3,enum=dhtnode.RegisterResponse_Status" json:"status,omitempty"` - Msgs []string `protobuf:"bytes,2,rep,name=msgs,proto3" json:"msgs,omitempty"` + Record *AgentRecord `protobuf:"bytes,1,opt,name=record,proto3" json:"record,omitempty"` } -func (x *RegisterResponse) Reset() { - *x = RegisterResponse{} +func (x *Register) Reset() { + *x = Register{} if protoimpl.UnsafeEnabled { mi := &file_dht_dhtnode_message_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -180,13 +184,13 @@ func (x *RegisterResponse) Reset() { } } -func (x *RegisterResponse) String() string { +func (x *Register) String() string { return protoimpl.X.MessageStringOf(x) } -func (*RegisterResponse) ProtoMessage() {} +func (*Register) ProtoMessage() {} -func (x *RegisterResponse) ProtoReflect() protoreflect.Message { +func (x *Register) ProtoReflect() protoreflect.Message { mi := &file_dht_dhtnode_message_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -198,62 +202,208 @@ func (x *RegisterResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use RegisterResponse.ProtoReflect.Descriptor instead. -func (*RegisterResponse) Descriptor() ([]byte, []int) { +// Deprecated: Use Register.ProtoReflect.Descriptor instead. +func (*Register) Descriptor() ([]byte, []int) { return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{1} } -func (x *RegisterResponse) GetStatus() RegisterResponse_Status { +func (x *Register) GetRecord() *AgentRecord { if x != nil { - return x.Status + return x.Record + } + return nil +} + +type Status struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Code Status_ErrCode `protobuf:"varint,1,opt,name=code,proto3,enum=dhtnode.Status_ErrCode" json:"code,omitempty"` + Msgs []string `protobuf:"bytes,2,rep,name=msgs,proto3" json:"msgs,omitempty"` +} + +func (x *Status) Reset() { + *x = Status{} + if protoimpl.UnsafeEnabled { + mi := &file_dht_dhtnode_message_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Status) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Status) ProtoMessage() {} + +func (x *Status) ProtoReflect() protoreflect.Message { + mi := &file_dht_dhtnode_message_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Status.ProtoReflect.Descriptor instead. +func (*Status) Descriptor() ([]byte, []int) { + return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{2} +} + +func (x *Status) GetCode() Status_ErrCode { + if x != nil { + return x.Code } - return RegisterResponse_SUCCESS + return Status_SUCCESS } -func (x *RegisterResponse) GetMsgs() []string { +func (x *Status) GetMsgs() []string { if x != nil { return x.Msgs } return nil } +type AcnMessage struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + // Types that are assignable to Payload: + // *AcnMessage_Status + // *AcnMessage_Register + Payload isAcnMessage_Payload `protobuf_oneof:"payload"` +} + +func (x *AcnMessage) Reset() { + *x = AcnMessage{} + if protoimpl.UnsafeEnabled { + mi := &file_dht_dhtnode_message_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AcnMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AcnMessage) ProtoMessage() {} + +func (x *AcnMessage) ProtoReflect() protoreflect.Message { + mi := &file_dht_dhtnode_message_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AcnMessage.ProtoReflect.Descriptor instead. +func (*AcnMessage) Descriptor() ([]byte, []int) { + return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{3} +} + +func (x *AcnMessage) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (m *AcnMessage) GetPayload() isAcnMessage_Payload { + if m != nil { + return m.Payload + } + return nil +} + +func (x *AcnMessage) GetStatus() *Status { + if x, ok := x.GetPayload().(*AcnMessage_Status); ok { + return x.Status + } + return nil +} + +func (x *AcnMessage) GetRegister() *Register { + if x, ok := x.GetPayload().(*AcnMessage_Register); ok { + return x.Register + } + return nil +} + +type isAcnMessage_Payload interface { + isAcnMessage_Payload() +} + +type AcnMessage_Status struct { + Status *Status `protobuf:"bytes,2,opt,name=status,proto3,oneof"` +} + +type AcnMessage_Register struct { + Register *Register `protobuf:"bytes,3,opt,name=register,proto3,oneof"` +} + +func (*AcnMessage_Status) isAcnMessage_Payload() {} + +func (*AcnMessage_Register) isAcnMessage_Payload() {} + var File_dht_dhtnode_message_proto protoreflect.FileDescriptor var file_dht_dhtnode_message_proto_rawDesc = []byte{ 0x0a, 0x19, 0x64, 0x68, 0x74, 0x2f, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x07, 0x64, 0x68, 0x74, - 0x6e, 0x6f, 0x64, 0x65, 0x22, 0xe4, 0x01, 0x0a, 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, - 0x72, 0x12, 0x29, 0x0a, 0x10, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x5f, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x0a, 0x0d, - 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x12, 0x28, 0x0a, 0x10, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x70, 0x75, 0x62, 0x6c, 0x69, - 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x61, 0x67, 0x65, - 0x6e, 0x74, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x0f, 0x63, - 0x6f, 0x6e, 0x6e, 0x5f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x6e, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, - 0x4b, 0x65, 0x79, 0x12, 0x36, 0x0a, 0x17, 0x70, 0x72, 0x6f, 0x6f, 0x66, 0x5f, 0x6f, 0x66, 0x5f, - 0x72, 0x65, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x70, 0x72, 0x6f, 0x6f, 0x66, 0x4f, 0x66, 0x52, 0x65, 0x70, - 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x82, 0x02, 0x0a, 0x10, - 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x38, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x20, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, - 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x73, - 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x22, 0x9f, - 0x01, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, - 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, - 0x55, 0x4e, 0x53, 0x55, 0x50, 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x56, 0x45, 0x52, 0x53, - 0x49, 0x4f, 0x4e, 0x10, 0x01, 0x12, 0x1f, 0x0a, 0x1b, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, - 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, 0x44, - 0x52, 0x45, 0x53, 0x53, 0x10, 0x02, 0x12, 0x1c, 0x0a, 0x18, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, - 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x4b, - 0x45, 0x59, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, - 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x4f, 0x46, 0x10, 0x04, 0x12, 0x11, 0x0a, - 0x0d, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x10, 0x05, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x6f, 0x64, 0x65, 0x22, 0xab, 0x01, 0x0a, 0x0b, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x63, 0x6f, 0x72, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1d, 0x0a, + 0x0a, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x26, 0x0a, 0x0f, + 0x70, 0x65, 0x65, 0x72, 0x5f, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x70, 0x65, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6c, 0x69, + 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x22, 0x38, 0x0a, 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x2c, + 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, + 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0x8a, 0x02, 0x0a, + 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x45, 0x72, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, + 0x63, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x07, 0x45, 0x72, 0x72, + 0x43, 0x6f, 0x64, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, + 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x53, 0x55, 0x50, + 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x10, 0x01, + 0x12, 0x1c, 0x0a, 0x18, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x45, 0x58, 0x50, 0x45, + 0x43, 0x54, 0x45, 0x44, 0x5f, 0x50, 0x41, 0x59, 0x4c, 0x4f, 0x41, 0x44, 0x10, 0x02, 0x12, 0x11, + 0x0a, 0x0d, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x10, + 0x03, 0x12, 0x1f, 0x0a, 0x1b, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, + 0x49, 0x44, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, + 0x10, 0x0a, 0x12, 0x1c, 0x0a, 0x18, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, + 0x4c, 0x49, 0x44, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x4b, 0x45, 0x59, 0x10, 0x0b, + 0x12, 0x17, 0x0a, 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, + 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x4f, 0x46, 0x10, 0x0c, 0x22, 0x8d, 0x01, 0x0a, 0x0a, 0x41, 0x63, + 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x48, 0x00, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a, + 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x11, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, + 0x65, 0x72, 0x48, 0x00, 0x52, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x42, 0x09, + 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -269,19 +419,24 @@ func file_dht_dhtnode_message_proto_rawDescGZIP() []byte { } var file_dht_dhtnode_message_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_dht_dhtnode_message_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_dht_dhtnode_message_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_dht_dhtnode_message_proto_goTypes = []interface{}{ - (RegisterResponse_Status)(0), // 0: dhtnode.RegisterResponse.Status - (*Register)(nil), // 1: dhtnode.Register - (*RegisterResponse)(nil), // 2: dhtnode.RegisterResponse + (Status_ErrCode)(0), // 0: dhtnode.Status.ErrCode + (*AgentRecord)(nil), // 1: dhtnode.AgentRecord + (*Register)(nil), // 2: dhtnode.Register + (*Status)(nil), // 3: dhtnode.Status + (*AcnMessage)(nil), // 4: dhtnode.AcnMessage } var file_dht_dhtnode_message_proto_depIdxs = []int32{ - 0, // 0: dhtnode.RegisterResponse.status:type_name -> dhtnode.RegisterResponse.Status - 1, // [1:1] is the sub-list for method output_type - 1, // [1:1] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 1, // 0: dhtnode.Register.record:type_name -> dhtnode.AgentRecord + 0, // 1: dhtnode.Status.code:type_name -> dhtnode.Status.ErrCode + 3, // 2: dhtnode.AcnMessage.status:type_name -> dhtnode.Status + 2, // 3: dhtnode.AcnMessage.register:type_name -> dhtnode.Register + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_dht_dhtnode_message_proto_init() } @@ -291,7 +446,7 @@ func file_dht_dhtnode_message_proto_init() { } if !protoimpl.UnsafeEnabled { file_dht_dhtnode_message_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Register); i { + switch v := v.(*AgentRecord); i { case 0: return &v.state case 1: @@ -303,7 +458,7 @@ func file_dht_dhtnode_message_proto_init() { } } file_dht_dhtnode_message_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RegisterResponse); i { + switch v := v.(*Register); i { case 0: return &v.state case 1: @@ -314,6 +469,34 @@ func file_dht_dhtnode_message_proto_init() { return nil } } + file_dht_dhtnode_message_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Status); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_dht_dhtnode_message_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AcnMessage); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_dht_dhtnode_message_proto_msgTypes[3].OneofWrappers = []interface{}{ + (*AcnMessage_Status)(nil), + (*AcnMessage_Register)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -321,7 +504,7 @@ func file_dht_dhtnode_message_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_dht_dhtnode_message_proto_rawDesc, NumEnums: 1, - NumMessages: 2, + NumMessages: 4, NumExtensions: 0, NumServices: 0, }, diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto index 40eb43efd5..ff1c7aad1d 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto @@ -2,24 +2,40 @@ syntax = "proto3"; package dhtnode; +message AgentRecord { + string service_id = 1; + string address = 2; + string public_key = 3; + string peer_public_key = 4; + string signature = 5; +} + message Register { - string protocol_version = 1; - string agent_address = 2; - string agent_public_key = 3; - string conn_public_key = 4; - string proof_of_representation = 5; + AgentRecord record = 1; } -message RegisterResponse { - enum Status { +message Status { + enum ErrCode { + // common (0x) SUCCESS = 0; ERROR_UNSUPPORTED_VERSION = 1; - ERROR_INVALID_AGENT_ADDRESS = 2; - ERROR_INVALID_PUBLIC_KEY = 3; - ERROR_INVALID_PROOF = 4; - ERROR_GENERIC = 5; + ERROR_UNEXPECTED_PAYLOAD = 2; + ERROR_GENERIC = 3; + // register (1x) + ERROR_INVALID_AGENT_ADDRESS = 10; + ERROR_INVALID_PUBLIC_KEY = 11; + ERROR_INVALID_PROOF = 12; + // } - Status status = 1; + ErrCode code = 1; repeated string msgs = 2; +} + +message AcnMessage { + string version = 1; + oneof payload { + Status status = 2; + Register register = 3; + } } \ No newline at end of file diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go index f54833296a..0213f2cfcb 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go @@ -481,31 +481,31 @@ func (dhtPeer *DHTPeer) handleDelegateService(ready *sync.WaitGroup) { } } -func isValidProofOfRepresentation(registration *dhtnode.Register) (*dhtnode.RegisterResponse, error) { +func isValidProofOfRepresentation(record *dhtnode.AgentRecord) (*dhtnode.Status, error) { // check that agent address and public key match - addrFromPubKey, err := utils.FetchAIAddressFromPublicKey(registration.AgentPublicKey) - if err != nil || addrFromPubKey != registration.AgentAddress { + addrFromPubKey, err := utils.FetchAIAddressFromPublicKey(record.PublicKey) + if err != nil || addrFromPubKey != record.Address { if err == nil { err = errors.New("Agent address and public key don't match") } - response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_ERROR_INVALID_AGENT_ADDRESS} + response := &dhtnode.Status{Code: dhtnode.Status_ERROR_INVALID_AGENT_ADDRESS} return response, err } // check that signature is valid - ok, err := utils.VerifyFetchAISignatureBTC([]byte(registration.ConnPublicKey), registration.ProofOfRepresentation, registration.AgentPublicKey) + ok, err := utils.VerifyFetchAISignatureBTC([]byte(record.PeerPublicKey), record.Signature, record.PublicKey) if !ok || err != nil { if err == nil { err = errors.New("Signature is not valid") } - response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_ERROR_INVALID_PROOF} + response := &dhtnode.Status{Code: dhtnode.Status_ERROR_INVALID_PROOF} return response, err } // PoR is valid - response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_SUCCESS} + response := &dhtnode.Status{Code: dhtnode.Status_SUCCESS} return response, nil } @@ -535,11 +535,11 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { return } - registration := &dhtnode.Register{} - err = proto.Unmarshal(buf, registration) + msg := &dhtnode.AcnMessage{} + err = proto.Unmarshal(buf, msg) if err != nil { - lerror(err).Msg("couldn't deserialize registration message") - response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_ERROR_GENERIC} + lerror(err).Msg("couldn't deserialize acn registration message") + response := &dhtnode.Status{Code: dhtnode.Status_ERROR_GENERIC, Msgs: []string{err.Error()}} buf, err = proto.Marshal(response) err = utils.WriteBytesConn(conn, buf) ignore(err) @@ -548,20 +548,37 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { return } - linfo().Msgf("Received registration request %s", registration) + linfo().Msgf("Received registration request %s", msg) + + // Get Register message + var register *dhtnode.Register + switch pl := msg.Payload.(type) { + case *dhtnode.AcnMessage_Register: + register = pl.Register + default: + err = errors.New("Unexpected payload") + response := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNEXPECTED_PAYLOAD} + buf, err = proto.Marshal(response) + err = utils.WriteBytesConn(conn, buf) + ignore(err) + + nbrConns.Dec() + return + } + record := register.Record + addr := record.Address - addr := registration.AgentAddress linfo().Msgf("connection from %s established for Address %s", conn.RemoteAddr().String(), addr) // check that connection public key match - connPubKey, err := utils.PubKeyFromFetchAIPublicKey(registration.ConnPublicKey) + connPubKey, err := utils.PubKeyFromFetchAIPublicKey(record.PeerPublicKey) if err != nil || !connPubKey.Equals(dhtPeer.publicKey) { if err == nil { err = errors.New("Registration connection public key doesn't match peer key") } lerror(err).Msg("Couldn't verify agent address and public key association") - response := &dhtnode.RegisterResponse{Status: dhtnode.RegisterResponse_ERROR_INVALID_PUBLIC_KEY} + response := &dhtnode.Status{Code: dhtnode.Status_ERROR_INVALID_PUBLIC_KEY} buf, err = proto.Marshal(response) err = utils.WriteBytesConn(conn, buf) ignore(err) @@ -571,7 +588,7 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { } // check if the PoR is valid - response, err := isValidProofOfRepresentation(registration) + response, err := isValidProofOfRepresentation(record) if err != nil { lerror(err).Msg("PoR is not valid") buf, err = proto.Marshal(response) @@ -582,7 +599,8 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { return } - buf, err = proto.Marshal(response) + msg = &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{response}} + buf, err = proto.Marshal(msg) err = utils.WriteBytesConn(conn, buf) if err != nil { nbrConns.Dec() diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go index 45773d1975..3c9bac1355 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go @@ -66,6 +66,19 @@ var ( "9427c1472b66f6abd94a6c246eee495e3709ec45882ae0badcbc71ad2cd8f8b2", } + FetchAITestPublicKeys = []string{ + "03b7e977f498dce004e2614764ff576e17cc6691135497e7bcb5d3441e816ba9e1", + "02344c3f0e79f56aef8e167a6fea912745f1f770b66b4c5096040c0e8c9e3c68b3", + "023d6021c001c7b562af8b6e54ace4f98b1b14170d7db4749ecab2b1f0e4252794", + "02a0eb20ae23f2f78650b42dfafa6bf4e4752657905da8598b2c0806478e0bfa0d", + "023db373d1fc21212f2f03fec1ddd49f193f54f71545e72f37c8a70ca20ef1622b", + "03290b4e5dabcca2a994a8d63057f5c83f60d999ede181a8d9b42084e3bee256c2", + "03510651fbb9d2ce5b7ae00968339055fbc552e565c54cce8c69f5a52209d3d6a7", + "02c11df29b5873e0c37d1427c488ba84e5ccc57405d39299757cee06893ab8595d", + "031545edc0fe81a17c77a391a343f95547745b28703bbe664e12c523e3272b637e", + "02dd78522785e4175e7db9794b03adcdcfaf707153f307caa3368da5a30594d369", + } + AgentsTestKeys = []string{ "730c22474709a6d17cf11599a80413a84ddb691a3c7b11a6d8d47a2c024b7b56", "a085c5eeb39636a21c85a9bc667bae18bf3e327a220ecb3998e317b62ab20ec6", @@ -668,21 +681,7 @@ func TestRoutingDelegateClientToDHTPeerX(t *testing.T) { } defer peerCleanup() - peerPubKey, err := utils.FetchAIPublicKeyFromFetchAIPrivateKey(FetchAITestKeys[0]) - if err != nil { - t.Fatal("Failed to get public key from DHTPeer:", err) - } - - agentPublicKey, err := utils.FetchAIPublicKeyFromFetchAIPrivateKey(AgentsTestKeys[1]) - if err != nil { - t.Fatal("Failed to get public key from Agent key:", err) - } - - signature, err := signFetchAI([]byte(peerPubKey), AgentsTestKeys[1]) - if err != nil { - t.Fatal("Failed to sign peer public key:", err) - } - client, clientCleanup, err := SetupDelegateClientWithPoR(AgentsTestAddresses[1], DefaultLocalHost, DefaultDelegatePort, agentPublicKey, peerPubKey, signature) + client, clientCleanup, err := SetupDelegateClientWithPoR(AgentsTestKeys[1], DefaultLocalHost, DefaultDelegatePort, FetchAITestPublicKeys[0]) if err != nil { t.Fatal("Failed to initialize DelegateClient:", err) } @@ -1437,6 +1436,10 @@ func SetupDHTClient(key string, address string, entry []string) (*dhtclient.DHTC type DelegateClient struct { AgentAddress string + AgentKey string + AgentPubKey string + PeerPubKey string + PoR string Rx chan *aea.Envelope Conn net.Conn processEnvelope func(*aea.Envelope) error @@ -1489,22 +1492,43 @@ func SetupDelegateClient(address string, host string, port uint16) (*DelegateCli return client, func() { client.Close() }, nil } -func SetupDelegateClientWithPoR(address string, host string, port uint16, pubkey string, connKey string, sig string) (*DelegateClient, func(), error) { +func SetupDelegateClientWithPoR(key string, host string, port uint16, peerPubKey string) (*DelegateClient, func(), error) { var err error client := &DelegateClient{} + client.AgentKey = key + + pubKey, err := utils.FetchAIPublicKeyFromFetchAIPrivateKey(key) + if err != nil { + return nil, nil, err + } + client.AgentPubKey = pubKey + + address, err := utils.FetchAIAddressFromPublicKey(pubKey) + if err != nil { + return nil, nil, err + } client.AgentAddress = address + + signature, err := signFetchAI([]byte(peerPubKey), key) + if err != nil { + return nil, nil, err + } + client.PoR = signature + client.Rx = make(chan *aea.Envelope) client.Conn, err = net.Dial("tcp", host+":"+strconv.FormatInt(int64(port), 10)) if err != nil { return nil, nil, err } - registration := &dhtnode.Register{ProtocolVersion: "0.1.0"} - registration.AgentAddress = address - registration.AgentPublicKey = pubkey - registration.ConnPublicKey = connKey - registration.ProofOfRepresentation = sig - data, err := proto.Marshal(registration) + record := &dhtnode.AgentRecord{} + record.Address = address + record.PublicKey = pubKey + record.PeerPublicKey = peerPubKey + record.Signature = signature + registration := &dhtnode.Register{Record: record} + msg := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Register{registration}} + data, err := proto.Marshal(msg) err = utils.WriteBytesConn(client.Conn, data) ignore(err) @@ -1512,11 +1536,24 @@ func SetupDelegateClientWithPoR(address string, host string, port uint16, pubkey if err != nil { return nil, nil, err } - response := &dhtnode.RegisterResponse{} + response := &dhtnode.AcnMessage{} err = proto.Unmarshal(data, response) - if response.Status != dhtnode.RegisterResponse_SUCCESS { - println("Registration error:", response.Status.String()) - return nil, nil, errors.New(response.Status.String()) + if err != nil { + return nil, nil, err + } + + // Get Status message + var status *dhtnode.Status + switch pl := response.Payload.(type) { + case *dhtnode.AcnMessage_Status: + status = pl.Status + default: + return nil, nil, err + } + + if status.Code != dhtnode.Status_SUCCESS { + println("Registration error:", status.String()) + return nil, nil, errors.New(status.String()) } go func() { From 618fc8cf02994ab6d179f7a72a4954318be1b767 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Tue, 15 Dec 2020 09:21:22 +0000 Subject: [PATCH 004/204] Add PoR to Relay and Direct connection registration --- .../fetchai/connections/p2p_libp2p/aea/api.go | 25 +++ .../p2p_libp2p/dht/dhtclient/dhtclient.go | 59 ++++++- .../p2p_libp2p/dht/dhtclient/options.go | 22 ++- .../p2p_libp2p/dht/dhtnode/utils.go | 62 +++++++ .../p2p_libp2p/dht/dhtpeer/dhtpeer.go | 164 +++++++++++------- .../p2p_libp2p/dht/dhtpeer/dhtpeer_test.go | 100 ++++++++++- .../p2p_libp2p/dht/dhtpeer/options.go | 18 ++ .../connections/p2p_libp2p/libp2p_node.go | 9 + .../connections/p2p_libp2p/utils/utils.go | 24 ++- 9 files changed, 401 insertions(+), 82 deletions(-) create mode 100644 packages/fetchai/connections/p2p_libp2p/dht/dhtnode/utils.go diff --git a/packages/fetchai/connections/p2p_libp2p/aea/api.go b/packages/fetchai/connections/p2p_libp2p/aea/api.go index e15cc27f71..e900b55465 100644 --- a/packages/fetchai/connections/p2p_libp2p/aea/api.go +++ b/packages/fetchai/connections/p2p_libp2p/aea/api.go @@ -51,6 +51,15 @@ type Pipe interface { Close() error } +// Needed to break import cycle +type AgentRecord struct { + ServiceId string + Address string + PublicKey string + PeerPublicKey string + Signature string +} + /* AeaApi type @@ -61,6 +70,7 @@ type AeaApi struct { msgin_path string msgout_path string agent_addr string + agent_record *AgentRecord id string entry_peers []string host string @@ -111,6 +121,10 @@ func (aea AeaApi) EntryPeers() []string { return aea.entry_peers } +func (aea AeaApi) AgentRecord() *AgentRecord { + return aea.agent_record +} + func (aea AeaApi) RegistrationDelayInSeconds() float64 { return aea.registrationDelay } @@ -176,6 +190,17 @@ func (aea *AeaApi) Init() error { uri_public := os.Getenv("AEA_P2P_URI_PUBLIC") uri_delegate := os.Getenv("AEA_P2P_DELEGATE_URI") uri_monitoring := os.Getenv("AEA_P2P_URI_MONITORING") + + por_address := os.Getenv("AEA_P2P_POR_ADDRESS") + if por_address != "" { + record := &AgentRecord{Address: por_address} + record.PublicKey = os.Getenv("AEA_P2P_POR_PUBKEY") + record.PeerPublicKey = os.Getenv("AEA_P2P_POR_PEER_PUBKEY") + record.Signature = os.Getenv("AEA_P2P_POR_SIGNATURE") + record.ServiceId = os.Getenv("AEA_P2P_POR_SERVICE_ID") + aea.agent_record = record + } + registrationDelay := os.Getenv("AEA_P2P_CFG_REGISTRATION_DELAY") logger.Debug().Msgf("msgin_path: %s", aea.msgin_path) diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go index 580bd13a49..9f1c335667 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go @@ -28,9 +28,11 @@ import ( "errors" "log" "math/rand" + "strings" "time" "github.com/rs/zerolog" + "google.golang.org/protobuf/proto" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p-core/crypto" @@ -73,6 +75,7 @@ type DHTClient struct { routedHost *routedhost.RoutedHost myAgentAddress string + myAgentRecord *dhtnode.AgentRecord myAgentReady func() bool processEnvelope func(*aea.Envelope) error @@ -105,6 +108,26 @@ func New(opts ...Option) (*DHTClient, error) { return nil, errors.New("missing agent address") } + // agent record is mandatory + // FIXME + if dhtClient.myAgentRecord == nil { + // return nil, errors.New("missing agent record") + } + + // check if the PoR is delivered for my public key + myPublicKey, err := utils.FetchAIPublicKeyFromPubKey(dhtClient.publicKey) + status, errPoR := dhtnode.IsValidProofOfRepresentation(dhtClient.myAgentRecord, myPublicKey) + if status.Code != dhtnode.Status_SUCCESS { + msg := "Invalid AgentRecord" + if err != nil { + msg += " - " + err.Error() + } + if errPoR != nil { + msg += " - " + errPoR.Error() + } + return nil, errors.New(msg) + } + // bootsrap peers if len(dhtClient.bootstrapPeers) < 1 { return nil, errors.New("at least one boostrap peer should be provided") @@ -426,7 +449,6 @@ func (dhtClient *DHTClient) RouteEnvelope(envel *aea.Envelope) error { stream.Close() } - println("Envelope sent:::::::::::::::::::::::::::::::::::::::::::::::", envel.String()) return err } @@ -518,21 +540,48 @@ func (dhtClient *DHTClient) registerAgentAddress() error { Str("op", "register"). Str("addr", dhtClient.myAgentAddress). Msgf("registering addr and peerID to relay peer") - err = utils.WriteBytes(stream, []byte(dhtClient.myAgentAddress)) + + registration := &dhtnode.Register{Record: dhtClient.myAgentRecord} + msg := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Register{registration}} + buf, err := proto.Marshal(msg) + + err = utils.WriteBytes(stream, buf) if err != nil { errReset := stream.Reset() ignore(errReset) return err } - _, _ = utils.ReadBytes(stream) - err = utils.WriteBytes(stream, []byte(dhtClient.routedHost.ID().Pretty())) + buf, err = utils.ReadBytes(stream) if err != nil { errReset := stream.Reset() ignore(errReset) return err } + response := &dhtnode.AcnMessage{} + err = proto.Unmarshal(buf, response) + if err != nil { + errReset := stream.Reset() + ignore(errReset) + return err + } + + // Get Status message + var status *dhtnode.Status + switch pl := response.Payload.(type) { + case *dhtnode.AcnMessage_Status: + status = pl.Status + default: + errReset := stream.Reset() + ignore(errReset) + return err + } + + if status.Code != dhtnode.Status_SUCCESS { + errReset := stream.Reset() + ignore(errReset) + return errors.New("Registration failed: " + strings.Join(status.Msgs, ",")) + } - _, _ = utils.ReadBytes(stream) stream.Close() return nil diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/options.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/options.go index d129b1d7f0..1fd402ef65 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/options.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/options.go @@ -20,7 +20,11 @@ package dhtclient -import "libp2p_node/utils" +import ( + "libp2p_node/aea" + "libp2p_node/dht/dhtnode" + "libp2p_node/utils" +) // Option for dhtclient.New type Option func(*DHTClient) error @@ -46,6 +50,22 @@ func RegisterAgentAddress(addr string, isReady func() bool) Option { } } +func RegisterAgentAddressWithPoR(record *aea.AgentRecord, isReady func() bool) Option { + return func(dhtClient *DHTClient) error { + pbRecord := &dhtnode.AgentRecord{} + pbRecord.Address = record.Address + pbRecord.PublicKey = record.PublicKey + pbRecord.PeerPublicKey = record.PeerPublicKey + pbRecord.Signature = record.Signature + pbRecord.ServiceId = record.ServiceId + + dhtClient.myAgentAddress = record.Address + dhtClient.myAgentRecord = pbRecord + dhtClient.myAgentReady = isReady + return nil + } +} + // BootstrapFrom for dhtclient.New func BootstrapFrom(entryPeers []string) Option { return func(dhtClient *DHTClient) error { diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/utils.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/utils.go new file mode 100644 index 0000000000..621992703b --- /dev/null +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/utils.go @@ -0,0 +1,62 @@ +/* -*- coding: utf-8 -*- +* ------------------------------------------------------------------------------ +* +* Copyright 2018-2019 Fetch.AI Limited +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* ------------------------------------------------------------------------------ + */ + +// Package dhtnode (in progress) contains the common interface between dhtpeer and dhtclient +package dhtnode + +import ( + "errors" + utils "libp2p_node/utils" +) + +func IsValidProofOfRepresentation(record *AgentRecord, representativePeerPubKey string) (*Status, error) { + // check public key matches + if record.PeerPublicKey != representativePeerPubKey { + err := errors.New("Wrong peer public key, expected " + representativePeerPubKey) + response := &Status{Code: Status_ERROR_INVALID_PUBLIC_KEY, Msgs: []string{err.Error()}} + return response, err + } + + // check that agent address and public key match + addrFromPubKey, err := utils.FetchAIAddressFromPublicKey(record.PublicKey) + if err != nil || addrFromPubKey != record.Address { + if err == nil { + err = errors.New("Agent address and public key don't match") + } + response := &Status{Code: Status_ERROR_INVALID_AGENT_ADDRESS} + return response, err + } + + // check that signature is valid + ok, err := utils.VerifyFetchAISignatureBTC([]byte(record.PeerPublicKey), record.Signature, record.PublicKey) + if !ok || err != nil { + if err == nil { + err = errors.New("Signature is not valid") + } + response := &Status{Code: Status_ERROR_INVALID_PROOF} + return response, err + + } + + // PoR is valid + response := &Status{Code: Status_SUCCESS} + return response, nil + +} diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go index 0213f2cfcb..f27346aec4 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go @@ -120,12 +120,16 @@ type DHTPeer struct { addressAnnounced bool myAgentAddress string + myAgentRecord *dhtnode.AgentRecord myAgentReady func() bool dhtAddresses map[string]string tcpAddresses map[string]net.Conn + agentRecords map[string]*dhtnode.AgentRecord dhtAddressesLock sync.RWMutex tcpAddressesLock sync.RWMutex - processEnvelope func(*aea.Envelope) error + agentRecordsLock sync.RWMutex + // TOFIX(LR): maps and locks need refactoring for better abstraction + processEnvelope func(*aea.Envelope) error monitor monitoring.MonitoringService closing chan struct{} @@ -140,8 +144,10 @@ func New(opts ...Option) (*DHTPeer, error) { dhtPeer.dhtAddresses = map[string]string{} dhtPeer.tcpAddresses = map[string]net.Conn{} + dhtPeer.agentRecords = map[string]*dhtnode.AgentRecord{} dhtPeer.dhtAddressesLock = sync.RWMutex{} dhtPeer.tcpAddressesLock = sync.RWMutex{} + dhtPeer.agentRecordsLock = sync.RWMutex{} for _, opt := range opts { if err := opt(dhtPeer); err != nil { @@ -168,6 +174,15 @@ func New(opts ...Option) (*DHTPeer, error) { return nil, errors.New("public host and port must be set") } + // check if the PoR is delivered for my public key + if dhtPeer.myAgentRecord != nil { + myPublicKey, err := utils.FetchAIPublicKeyFromPubKey(dhtPeer.publicKey) + status, errPoR := dhtnode.IsValidProofOfRepresentation(dhtPeer.myAgentRecord, myPublicKey) + if status.Code != dhtnode.Status_SUCCESS { + return nil, errors.New("Invalid AgentRecord - " + err.Error() + " - " + errPoR.Error()) + } + } + /* setup libp2p node */ ctx := context.Background() @@ -481,35 +496,6 @@ func (dhtPeer *DHTPeer) handleDelegateService(ready *sync.WaitGroup) { } } -func isValidProofOfRepresentation(record *dhtnode.AgentRecord) (*dhtnode.Status, error) { - - // check that agent address and public key match - addrFromPubKey, err := utils.FetchAIAddressFromPublicKey(record.PublicKey) - if err != nil || addrFromPubKey != record.Address { - if err == nil { - err = errors.New("Agent address and public key don't match") - } - response := &dhtnode.Status{Code: dhtnode.Status_ERROR_INVALID_AGENT_ADDRESS} - return response, err - } - - // check that signature is valid - ok, err := utils.VerifyFetchAISignatureBTC([]byte(record.PeerPublicKey), record.Signature, record.PublicKey) - if !ok || err != nil { - if err == nil { - err = errors.New("Signature is not valid") - } - response := &dhtnode.Status{Code: dhtnode.Status_ERROR_INVALID_PROOF} - return response, err - - } - - // PoR is valid - response := &dhtnode.Status{Code: dhtnode.Status_SUCCESS} - return response, nil - -} - func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { defer dhtPeer.goroutines.Done() defer conn.Close() @@ -530,7 +516,7 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { // read agent registration message buf, err := utils.ReadBytesConn(conn) if err != nil { - lerror(err).Msg("while receiving agent's Address") + lerror(err).Msg("while receiving agent's registration request") nbrConns.Dec() return } @@ -539,7 +525,8 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { err = proto.Unmarshal(buf, msg) if err != nil { lerror(err).Msg("couldn't deserialize acn registration message") - response := &dhtnode.Status{Code: dhtnode.Status_ERROR_GENERIC, Msgs: []string{err.Error()}} + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_GENERIC, Msgs: []string{err.Error()}} + response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} buf, err = proto.Marshal(response) err = utils.WriteBytesConn(conn, buf) ignore(err) @@ -557,7 +544,8 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { register = pl.Register default: err = errors.New("Unexpected payload") - response := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNEXPECTED_PAYLOAD} + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNEXPECTED_PAYLOAD} + response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} buf, err = proto.Marshal(response) err = utils.WriteBytesConn(conn, buf) ignore(err) @@ -571,26 +559,13 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { linfo().Msgf("connection from %s established for Address %s", conn.RemoteAddr().String(), addr) - // check that connection public key match - connPubKey, err := utils.PubKeyFromFetchAIPublicKey(record.PeerPublicKey) - if err != nil || !connPubKey.Equals(dhtPeer.publicKey) { - if err == nil { - err = errors.New("Registration connection public key doesn't match peer key") - } - lerror(err).Msg("Couldn't verify agent address and public key association") - response := &dhtnode.Status{Code: dhtnode.Status_ERROR_INVALID_PUBLIC_KEY} - buf, err = proto.Marshal(response) - err = utils.WriteBytesConn(conn, buf) - ignore(err) - - nbrConns.Dec() - return - } - // check if the PoR is valid - response, err := isValidProofOfRepresentation(record) + myPubKey, err := utils.FetchAIPublicKeyFromPubKey(dhtPeer.publicKey) + ignore(err) + status, err := dhtnode.IsValidProofOfRepresentation(record, myPubKey) if err != nil { lerror(err).Msg("PoR is not valid") + response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} buf, err = proto.Marshal(response) err = utils.WriteBytesConn(conn, buf) ignore(err) @@ -599,7 +574,7 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { return } - msg = &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{response}} + msg = &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} buf, err = proto.Marshal(msg) err = utils.WriteBytesConn(conn, buf) if err != nil { @@ -608,11 +583,15 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { } // Add connection to map + dhtPeer.agentRecordsLock.Lock() + dhtPeer.agentRecords[addr] = record + dhtPeer.agentRecordsLock.Unlock() dhtPeer.tcpAddressesLock.Lock() dhtPeer.tcpAddresses[addr] = conn dhtPeer.tcpAddressesLock.Unlock() if dhtPeer.addressAnnounced { //linfo().Msgf("announcing tcp client address %s...", addr) + // TOFIX(LR) disconnect client? err = dhtPeer.registerAgentAddress(addr) if err != nil { lerror(err).Msgf("while announcing tcp client address %s to the dht", addr) @@ -652,6 +631,7 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { dhtPeer.tcpAddressesLock.Lock() delete(dhtPeer.tcpAddresses, addr) dhtPeer.tcpAddressesLock.Unlock() + // TOFIX(LR) currently I am keeping the agent record } @@ -1058,46 +1038,102 @@ func (dhtPeer *DHTPeer) handleAeaRegisterStream(stream network.Stream) { //linfo().Str("op", "register"). // Msg("Got a new aea register stream") - clientAddr, err := utils.ReadBytes(stream) + buf, err := utils.ReadBytes(stream) if err != nil { lerror(err).Str("op", "register"). - Msg("while reading client Address from stream") + Msg("while reading relay client registration request from stream") err = stream.Reset() ignore(err) return } + /////////// - err = utils.WriteBytes(stream, []byte("doneAddress")) - ignore(err) - - clientPeerID, err := utils.ReadBytes(stream) + msg := &dhtnode.AcnMessage{} + err = proto.Unmarshal(buf, msg) if err != nil { - lerror(err).Str("op", "register"). - Msgf("while reading client peerID from stream") + lerror(err).Msg("couldn't deserialize acn registration message") + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_GENERIC, Msgs: []string{err.Error()}} + response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) err = stream.Reset() ignore(err) return } - err = utils.WriteBytes(stream, []byte("donePeerID")) + linfo().Msgf("Received relay registration request %s", msg) + + // Get Register message + var register *dhtnode.Register + switch pl := msg.Payload.(type) { + case *dhtnode.AcnMessage_Register: + register = pl.Register + default: + err = errors.New("Unexpected payload") + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNEXPECTED_PAYLOAD} + response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Reset() + ignore(err) + return + } + record := register.Record + clientAddr := record.Address + + //linfo().Msgf("connection from %s established for Address %s", + // stream.Conn().RemotePeer().Pretty(), clientAddr) + + // check if the PoR is valid + clientPubKey, err := utils.FetchAIPublicKeyFromPubKey(stream.Conn().RemotePublicKey()) ignore(err) + status, err := dhtnode.IsValidProofOfRepresentation(record, clientPubKey) + if err != nil { + lerror(err).Msg("PoR is not valid") + response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Reset() + ignore(err) + return + } + + msg = &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} + buf, err = proto.Marshal(msg) + err = utils.WriteBytes(stream, buf) + if err != nil { + err = stream.Reset() + ignore(err) + return + } nbrClients.Inc() //linfo().Str("op", "register"). // Str("addr", string(clientAddr)). // Msgf("Received address registration request for peer id %s", string(clientPeerID)) + clientPeerID := stream.Conn().RemotePeer().Pretty() + dhtPeer.agentRecordsLock.Lock() + dhtPeer.agentRecords[clientAddr] = record + dhtPeer.agentRecordsLock.Unlock() dhtPeer.dhtAddressesLock.Lock() - dhtPeer.dhtAddresses[string(clientAddr)] = string(clientPeerID) + dhtPeer.dhtAddresses[clientAddr] = clientPeerID dhtPeer.dhtAddressesLock.Unlock() if dhtPeer.addressAnnounced { linfo().Str("op", "register"). - Str("addr", string(clientAddr)). - Msgf("Announcing client address on behalf of %s...", string(clientPeerID)) + Str("addr", clientAddr). + Msgf("Announcing client address on behalf of %s...", clientPeerID) err = dhtPeer.registerAgentAddress(string(clientAddr)) if err != nil { + //TOFIX(LR) remove agent from map, or don't add it unless announcement done lerror(err).Str("op", "register"). - Str("addr", string(clientAddr)). + Str("addr", clientAddr). Msg("while announcing client address to the dht") err = stream.Reset() ignore(err) diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go index 3c9bac1355..d0edbd0ee8 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go @@ -430,8 +430,8 @@ func TestRoutingDHTClientToDHTPeerX(t *testing.T) { } defer peerCleanup() - client, clientCleanup, err := SetupDHTClient( - FetchAITestKeys[1], AgentsTestAddresses[1], []string{peer.MultiAddr()}, + client, clientCleanup, err := SetupDHTClientWithPoR( + FetchAITestKeys[1], AgentsTestKeys[1], []string{peer.MultiAddr()}, ) if err != nil { t.Fatal("Failed to initialize DHTClient:", err) @@ -1395,7 +1395,6 @@ func SetupLocalDHTPeer(key string, addr string, dhtPort uint16, delegatePort uin IdentityFromFetchAIKey(key), EnableRelayService(), BootstrapFrom(entry), - WithRegistrationDelay(5 * time.Second), } if addr != "" { @@ -1415,6 +1414,58 @@ func SetupLocalDHTPeer(key string, addr string, dhtPort uint16, delegatePort uin } +func SetupLocalDHTPeerWithPoR(key string, agentKey string, dhtPort uint16, delegatePort uint16, entry []string) (*DHTPeer, func(), error) { + opts := []Option{ + LocalURI(DefaultLocalHost, dhtPort), + PublicURI(DefaultLocalHost, dhtPort), + IdentityFromFetchAIKey(key), + EnableRelayService(), + BootstrapFrom(entry), + } + + if agentKey != "" { + agentPubKey, err := utils.FetchAIPublicKeyFromFetchAIPrivateKey(agentKey) + if err != nil { + return nil, nil, err + } + + agentAddress, err := utils.FetchAIAddressFromPublicKey(agentPubKey) + if err != nil { + return nil, nil, err + } + + peerPubKey, err := utils.FetchAIPublicKeyFromFetchAIPrivateKey(key) + if err != nil { + return nil, nil, err + } + + signature, err := signFetchAI([]byte(peerPubKey), agentKey) + if err != nil { + return nil, nil, err + } + + record := &aea.AgentRecord{} + record.Address = agentAddress + record.PublicKey = agentPubKey + record.PeerPublicKey = peerPubKey + record.Signature = signature + + opts = append(opts, RegisterAgentAddressWithPoR(record, func() bool { return true })) + } + + if delegatePort != 0 { + opts = append(opts, EnableDelegateService(delegatePort)) + } + + dhtPeer, err := New(opts...) + if err != nil { + return nil, nil, err + } + + return dhtPeer, func() { dhtPeer.Close() }, nil + +} + // DHTClient func SetupDHTClient(key string, address string, entry []string) (*dhtclient.DHTClient, func(), error) { @@ -1432,6 +1483,49 @@ func SetupDHTClient(key string, address string, entry []string) (*dhtclient.DHTC return dhtClient, func() { dhtClient.Close() }, nil } +func SetupDHTClientWithPoR(key string, agentKey string, entry []string) (*dhtclient.DHTClient, func(), error) { + + agentPubKey, err := utils.FetchAIPublicKeyFromFetchAIPrivateKey(agentKey) + if err != nil { + return nil, nil, err + } + + agentAddress, err := utils.FetchAIAddressFromPublicKey(agentPubKey) + if err != nil { + return nil, nil, err + } + + peerPubKey, err := utils.FetchAIPublicKeyFromFetchAIPrivateKey(key) + if err != nil { + return nil, nil, err + } + + signature, err := signFetchAI([]byte(peerPubKey), agentKey) + if err != nil { + return nil, nil, err + } + + record := &aea.AgentRecord{} + record.Address = agentAddress + record.PublicKey = agentPubKey + record.PeerPublicKey = peerPubKey + record.Signature = signature + + opts := []dhtclient.Option{ + dhtclient.IdentityFromFetchAIKey(key), + dhtclient.RegisterAgentAddressWithPoR(record, func() bool { return true }), + dhtclient.BootstrapFrom(entry), + } + + dhtClient, err := dhtclient.New(opts...) + if err != nil { + println(err) + return nil, nil, err + } + + return dhtClient, func() { dhtClient.Close() }, nil +} + // Delegate tcp client for tests only type DelegateClient struct { diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/options.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/options.go index 94210c2934..2dd4badca2 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/options.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/options.go @@ -27,6 +27,8 @@ import ( "github.com/multiformats/go-multiaddr" "github.com/rs/zerolog" + "libp2p_node/aea" + "libp2p_node/dht/dhtnode" utils "libp2p_node/utils" ) @@ -54,6 +56,22 @@ func RegisterAgentAddress(addr string, isReady func() bool) Option { } } +func RegisterAgentAddressWithPoR(record *aea.AgentRecord, isReady func() bool) Option { + return func(dhtPeer *DHTPeer) error { + pbRecord := &dhtnode.AgentRecord{} + pbRecord.Address = record.Address + pbRecord.PublicKey = record.PublicKey + pbRecord.PeerPublicKey = record.PeerPublicKey + pbRecord.Signature = record.Signature + pbRecord.ServiceId = record.ServiceId + + dhtPeer.myAgentAddress = record.Address + dhtPeer.myAgentRecord = pbRecord + dhtPeer.myAgentReady = isReady + return nil + } +} + // BootstrapFrom for dhtpeer.New func BootstrapFrom(entryPeers []string) Option { return func(dhtPeer *DHTPeer) error { diff --git a/packages/fetchai/connections/p2p_libp2p/libp2p_node.go b/packages/fetchai/connections/p2p_libp2p/libp2p_node.go index 17c53679e1..020bb5a716 100644 --- a/packages/fetchai/connections/p2p_libp2p/libp2p_node.go +++ b/packages/fetchai/connections/p2p_libp2p/libp2p_node.go @@ -90,6 +90,9 @@ func main() { // entry peers entryPeers := agent.EntryPeers() + // agent proof of representation + record := agent.AgentRecord() + // add artificial delay for agent registration registrationDelay := agent.RegistrationDelayInSeconds() @@ -106,6 +109,9 @@ func main() { if aeaAddr != "" { opts = append(opts, dhtclient.RegisterAgentAddress(aeaAddr, agent.Connected)) } + if record != nil { + opts = append(opts, dhtclient.RegisterAgentAddressWithPoR(record, agent.Connected)) + } node, err = dhtclient.New(opts...) } else { opts := []dhtpeer.Option{ @@ -119,6 +125,9 @@ func main() { if aeaAddr != "" { opts = append(opts, dhtpeer.RegisterAgentAddress(aeaAddr, agent.Connected)) } + if record != nil { + opts = append(opts, dhtPeer.RegisterAgentAddressWithPoR(record, agent.Connected)) + } if nodePortMonitoring != 0 { opts = append(opts, dhtpeer.EnablePrometheusMonitoring(nodePortMonitoring)) } diff --git a/packages/fetchai/connections/p2p_libp2p/utils/utils.go b/packages/fetchai/connections/p2p_libp2p/utils/utils.go index e60eecf6bc..b81d1e537c 100644 --- a/packages/fetchai/connections/p2p_libp2p/utils/utils.go +++ b/packages/fetchai/connections/p2p_libp2p/utils/utils.go @@ -212,6 +212,15 @@ func PubKeyFromFetchAIPublicKey(publicKey string) (crypto.PubKey, error) { return crypto.UnmarshalSecp256k1PublicKey(hexBytes) } +// FetchAIPublicKeyFromPubKey return FetchAI's format serialized public key +func FetchAIPublicKeyFromPubKey(publicKey crypto.PubKey) (string, error) { + raw, err := publicKey.Raw() + if err != nil { + return "", err + } + return hex.EncodeToString(raw), nil +} + // BTCPubKeyFromFetchAIPublicKey func BTCPubKeyFromFetchAIPublicKey(publicKey string) (*btcec.PublicKey, error) { pbkBytes, err := hex.DecodeString(publicKey) @@ -239,6 +248,7 @@ func ConvertStrEncodedSignatureToDER(signature []byte) []byte { sigDER[offset] = 0x02 sigDER[offset+1] = byte(len(sb)) copy(sigDER[offset+2:], sb) + println(len(sigDER)) return sigDER } @@ -247,15 +257,11 @@ func ConvertStrEncodedSignatureToDER(signature []byte) []byte { // - https://github.com/fetchai/agents-aea/blob/master/aea/crypto/cosmos.py#L258 // - https://github.com/btcsuite/btcd/blob/master/btcec/signature.go#L47 func ConvertDEREncodedSignatureToStr(signature []byte) ([]byte, error) { - length := len(signature) - 6 - if length%2 != 0 { - return []byte{}, errors.New("DER signature length must be even") - } - sigStr := make([]byte, length) - offset := 4 + length/2 - copy(sigStr, signature[4:offset]) - copy(sigStr[length/2:], signature[offset+2:]) - return sigStr, nil + sig, err := btcec.ParseDERSignature(signature, btcec.S256()) + if err != nil { + return []byte{}, err + } + return append(sig.R.Bytes(), sig.S.Bytes()...), nil } // ParseFetchAISignature create btcec Signature from base64 formated, string (not DER) encoded RFC6979 signature From a60561554b4832509138dc834f605f221050ee4b Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Tue, 15 Dec 2020 21:12:15 +0000 Subject: [PATCH 005/204] Fix golang libp2p tests for PoR --- .../connections/p2p_libp2p/connection.yaml | 29 +- .../dht/dhtclient/dhtclient_test.go | 34 ++- .../p2p_libp2p/dht/dhtclient/options.go | 4 +- .../p2p_libp2p/dht/dhtpeer/benchmarks_test.go | 78 +++-- .../p2p_libp2p/dht/dhtpeer/dhtpeer_test.go | 289 +++++++----------- .../p2p_libp2p/dht/dhtpeer/options.go | 10 +- .../p2p_libp2p/dht/dhttests/dhttests.go | 27 +- .../fetchai/connections/p2p_libp2p/go.mod | 13 +- .../fetchai/connections/p2p_libp2p/go.sum | 82 +++++ .../connections/p2p_libp2p/libp2p_node.go | 10 +- .../connections/p2p_libp2p/utils/utils.go | 17 ++ packages/hashes.csv | 2 +- 12 files changed, 342 insertions(+), 253 deletions(-) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index d0ff7b4cfe..eec20053ac 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -10,30 +10,33 @@ aea_version: '>=0.7.0, <0.8.0' fingerprint: README.md: QmWRXWUuSawx6DoLjYTPHFk7peMKfnY6gKTiGzAtrEXRp9 __init__.py: QmYQuLNyQ8WTjgRYAoKAzoJEb7ocKXvM2hTyK4hsGch5D6 - aea/api.go: QmThjcaiYGdPBniGAnrc66cEVtZaB4K1cgJb5iig6hTCK9 + aea/api.go: QmfXwePWUCCcYpKNYe9Uz7dfnZUVSLN66aaBHXQ9jWZb3J aea/envelope.pb.go: QmRfUNGpCeVJfsW3H1MzCN4pwDWgumfyWufVFp6xvUjjug aea/envelope.proto: QmSC8EGCKiNFR2vf5bSWymSzYDFMipQW9aQVMwPzQoKb4n aea/pipe_unix.go: QmSzbAexrSUSA9ZE6VT6MmcWF5MhEcmceQjAbnfRoK7Ruv aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK check_dependencies.py: QmY8EiWapBspyR3zKGEY3TnQayz7hF7DxZ2d9zozVfFhtA connection.py: QmVf4XX25S8FQHcZca6o1xC77Ez9dUNp66K3dGnVB724GC - dht/dhtclient/dhtclient.go: QmeHnMoBdff59vSqRRsYUGGJPUMi5Euk8pKiKa5jTzHHQD - dht/dhtclient/dhtclient_test.go: QmPfnHSHXtbaW5VYuq1QsKQWey64pUEvLEaKKkT9eAcmws - dht/dhtclient/options.go: QmPorj38wNrxGrzsbFe5wwLmiHzxbTJ2VsgvSd8tLDYS8s + dht/dhtclient/dhtclient.go: QmeKS6wgfeoYDaHRfb9FEDSCT2fDYnRXZxmEjc8yz7Zbec + dht/dhtclient/dhtclient_test.go: QmYVnf5j8Jyahi5jZNswkGR2tdnM4D1r5Fjdrz654EcD1F + dht/dhtclient/options.go: QmWs7fQqHRavjwe1PjihHFi85HeJu9qSEVyMdcRviZzaT2 dht/dhtnode/dhtnode.go: QmbyhgbCSAbQ1QsDw7FM7Nt5sZcvhbupA1jv5faxutbV7N + dht/dhtnode/message.pb.go: QmZDrnCfXTLWRqmenTHVKwiuywypuZz5VMXvGjaWqtR6np + dht/dhtnode/message.proto: QmaMTai7UMwAfoLHYib5FBuqXa8XWNht6uhHGZ1bbLtzTo dht/dhtnode/streams.go: Qmc2JcyiU4wHsgDj6aUunMAp4c5yMzo2ixeqRZHSW5PVwo - dht/dhtpeer/benchmarks_test.go: QmXDDpoThiFPZ3DDb1L9rNaJANSYeZZgfu5EQ4vSARaG5f - dht/dhtpeer/dhtpeer.go: QmNhwnereuoWd2Dw2azYNYTrwydGDqprUZSALV7yCms9wv - dht/dhtpeer/dhtpeer_test.go: QmRdsFpDLNf1KwWyRKGkrX7nD1orFyPKweP9BtZkQx7SqL - dht/dhtpeer/options.go: QmfQMgMJyPeNXroH2faDuLCWhjh48eJnez3pevyatjMxPW - dht/dhttests/dhttests.go: QmZpYRCiVARGL1n4nDwqjhzHA95Y4ACNWoa3HSDnB6PitK + dht/dhtnode/utils.go: QmZi91a8RWrUUM6LztpMWBhLKj47W7UwMoTvooptF6oNHU + dht/dhtpeer/benchmarks_test.go: QmUj4zGAG8QgDVFf9PeP5kKdjUbqBi5nzdYmhrUfX2eZBD + dht/dhtpeer/dhtpeer.go: QmWDqiqMiz7MNZAiJLLZnNWCjgCSwyjwMbPPf6bQgSB26G + dht/dhtpeer/dhtpeer_test.go: QmVn5BRKtyBAqR52RH5frBGiynyPoaWr9737bZZniMGgex + dht/dhtpeer/options.go: Qme5L9bRBQZaR5fciPJ1feFwqaQpkdYfW8TaiY1TDXs91H + dht/dhttests/dhttests.go: QmQJxw2rJCGUFTXKxH3sf3Zaqta7H76EvaB7mWNNfikFkA dht/monitoring/file.go: QmZB8pNf7infMXWaC5A46rr3mghNfkuSd8UKH4Ceg8nDSM dht/monitoring/prometheus.go: QmQk143Qn3WcKnk7Taf4fkvxm2zcCz2VXGi249S1rmgCbv dht/monitoring/service.go: QmT47y2LHZECYcoE2uJ9QCGh3Kq8ePhYedo8dQE7X7v6YV - go.mod: QmQEau8Bi7gBoKQhxsvTkLGWv1C6LzTBdgpzRzvsDrGXvV - go.sum: QmSYfnsHtt2ALbjnSSDbkYioDrEEzkUGeg47j9dF5Zssxg - libp2p_node.go: QmQXu6h9Q3T1Upjd8JdRggEHdXBQHvkfZoJCXG8YQT9Dq2 - utils/utils.go: QmYuokyZ1zMHEhUzv172bKKzUY5G2U5uouBZBy3BhgFckW + go.mod: QmScMYT69B7HbQApxXmZFpSwsKBZVPAJimVFYpTVYTvzbe + go.sum: QmawFS81vDFYVZUZMKxQjNdKkLxDzfRRfqD1Tcpzze66yd + libp2p_node.go: QmSMPBU6GMJ7gAHWMsYGVDSUv5sw48x2rvNKydcxVv7MqL + utils/utils.go: QmcgF9WCrP74kmQSSQWFyhbNBV9QwfXiF6J9BP7Yqnuyva fingerprint_ignore_patterns: [] build_entrypoint: check_dependencies.py connections: [] diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient_test.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient_test.go index 23f6b46d13..ccf4c961c1 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient_test.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient_test.go @@ -26,12 +26,16 @@ import ( "libp2p_node/aea" "libp2p_node/dht/dhttests" + utils "libp2p_node/utils" ) // const ( - DefaultFetchAIKey = "3916b301d1a0ec09de1db4833b0c945531004290caee0b4a5d7b554caa39dbf1" - DefaultAgentAddress = "2TsHmM9JXeFgK928LYc6HV96gi78pBv6sWprJAXaS6ydg9MTC6" + DefaultFetchAIKey = "04ab8ac134ec727917cf4f9e47685e84622151bc8b12838bc54c8ffe5d44d04a" + DefaultFetchAIPublicKey = "03b07ef4513e3f0372245b3d6d474d871ba58eacaf3a2a07c487af6d82006b86b4" + DefaultAgentKey = "f76137a61c1ad3ee8a0a9a185bc8e6fa51be1a2528f86042c11f9cc00880024a" + DefaultAgentPublicKey = "021820ce23b5f3a6ef01988149e724af854f89d37b9cabc3b1702cc5287f617b92" + DefaultAgentAddress = "fetch1ver6u7xdvkjy4dq8xxrkc6ualu98k7ykumv08q" EnvelopeDeliveryTimeout = 10 * time.Second ) @@ -46,9 +50,20 @@ func TestNew(t *testing.T) { } defer cleanup() + signature, err := utils.SignFetchAI([]byte(DefaultFetchAIPublicKey), DefaultAgentKey) + if err != nil { + t.Fatal("Failed to initialize DHTClient:", err) + } + + record := &aea.AgentRecord{} + record.Address = DefaultAgentAddress + record.PublicKey = DefaultAgentPublicKey + record.PeerPublicKey = DefaultFetchAIPublicKey + record.Signature = signature + opts := []Option{ IdentityFromFetchAIKey(DefaultFetchAIKey), - RegisterAgentAddress(DefaultAgentAddress, func() bool { return true }), + RegisterAgentAddress(record, func() bool { return true }), BootstrapFrom([]string{dhtPeer.MultiAddr()}), } @@ -78,9 +93,20 @@ func TestRouteEnvelopeToPeerAgent(t *testing.T) { } defer cleanup() + signature, err := utils.SignFetchAI([]byte(DefaultFetchAIPublicKey), DefaultAgentKey) + if err != nil { + t.Fatal("Failed to initialize DHTClient:", err) + } + + record := &aea.AgentRecord{} + record.Address = DefaultAgentAddress + record.PublicKey = DefaultAgentPublicKey + record.PeerPublicKey = DefaultFetchAIPublicKey + record.Signature = signature + opts := []Option{ IdentityFromFetchAIKey(DefaultFetchAIKey), - RegisterAgentAddress(DefaultAgentAddress, func() bool { return true }), + RegisterAgentAddress(record, func() bool { return true }), BootstrapFrom([]string{dhtPeer.MultiAddr()}), } diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/options.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/options.go index 1fd402ef65..9f247f1253 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/options.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/options.go @@ -42,7 +42,7 @@ func IdentityFromFetchAIKey(key string) Option { } // RegisterAgentAddress for dhtclient.New -func RegisterAgentAddress(addr string, isReady func() bool) Option { +func RegisterAgentAddressOld(addr string, isReady func() bool) Option { return func(dhtClient *DHTClient) error { dhtClient.myAgentAddress = addr dhtClient.myAgentReady = isReady @@ -50,7 +50,7 @@ func RegisterAgentAddress(addr string, isReady func() bool) Option { } } -func RegisterAgentAddressWithPoR(record *aea.AgentRecord, isReady func() bool) Option { +func RegisterAgentAddress(record *aea.AgentRecord, isReady func() bool) Option { return func(dhtClient *DHTClient) error { pbRecord := &dhtnode.AgentRecord{} pbRecord.Address = record.Address diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/benchmarks_test.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/benchmarks_test.go index 21fd1ae913..f63bde1556 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/benchmarks_test.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/benchmarks_test.go @@ -37,17 +37,17 @@ import ( * How to run * *********** - $ go test -p 1 -count 20 libp2p_node/dht/dhtpeer/ -run=XXX -bench . -benchtime=20x -keys-file=/path/to/file/benchmark_keys.txt -addrs-file=/path/to/file/benchmark_addrs.txt + $ go test -p 1 -count 20 libp2p_node/dht/dhtpeer/ -run=XXX -bench . -benchtime=20x -peers-keys-file=/path/to/file/benchmark_peers_keys.txt -agents-keys-file=/path/to/file/benchmark_agents_keys.txt */ -var keysFilePath string -var addrsFilePath string +var peersKeysFilePath string +var agentsKeysFilePath string var tcpUri = "localhost:12345" func init() { - flag.StringVar(&keysFilePath, "keys-file", "", "File with list of EC private keys") - flag.StringVar(&addrsFilePath, "addrs-file", "", "File with list of agents addresses") + flag.StringVar(&peersKeysFilePath, "peers-keys-file", "", "File with list of EC private keys") + flag.StringVar(&agentsKeysFilePath, "agents-keys-file", "", "File with list of agents EC private keys") } /* ********************************** @@ -155,31 +155,31 @@ func BenchmarkBaselineTCPConnectAndEcho(b *testing.B) { // helpers -func getKeysAndAddrs(b *testing.B) (keys []string, addrs []string) { - keysFile, err := os.Open(keysFilePath) +func getKeysAndAddrs(b *testing.B) (peers []string, agents []string) { + peersKeysFile, err := os.Open(peersKeysFilePath) if err != nil { b.Fatal(err) } - defer keysFile.Close() - addrsFile, err := os.Open(addrsFilePath) + defer peersKeysFile.Close() + agentsKeysFile, err := os.Open(agentsKeysFilePath) if err != nil { b.Fatal(err) } - defer addrsFile.Close() + defer agentsKeysFile.Close() - ksc := bufio.NewScanner(keysFile) - asc := bufio.NewScanner(addrsFile) + ksc := bufio.NewScanner(peersKeysFile) + asc := bufio.NewScanner(agentsKeysFile) - keys = []string{} - addrs = []string{} + peers = []string{} + agents = []string{} for ksc.Scan() && asc.Scan() { - keys = append(keys, ksc.Text()) - addrs = append(addrs, asc.Text()) + peers = append(peers, ksc.Text()) + agents = append(agents, asc.Text()) } - return keys, addrs + return peers, agents } -func setupLocalDHTPeerForBench(key string, addr string, dhtPort uint16, delegatePort uint16, entry []string) (*DHTPeer, func(), error) { +func setupLocalDHTPeerForBench(key string, agentKey string, dhtPort uint16, delegatePort uint16, entry []string) (*DHTPeer, func(), error) { /* peer, peerCleanup, err := SetupLocalDHTPeer(key, addr, dhtPort, delegatePort, entry) if err == nil { @@ -197,8 +197,34 @@ func setupLocalDHTPeerForBench(key string, addr string, dhtPort uint16, delegate BootstrapFrom(entry), } - if addr != "" { - opts = append(opts, RegisterAgentAddress(addr, func() bool { return true })) + if agentKey != "" { + agentPubKey, err := utils.FetchAIPublicKeyFromFetchAIPrivateKey(agentKey) + if err != nil { + return nil, nil, err + } + + agentAddress, err := utils.FetchAIAddressFromPublicKey(agentPubKey) + if err != nil { + return nil, nil, err + } + + peerPubKey, err := utils.FetchAIPublicKeyFromFetchAIPrivateKey(key) + if err != nil { + return nil, nil, err + } + + signature, err := utils.SignFetchAI([]byte(peerPubKey), agentKey) + if err != nil { + return nil, nil, err + } + + record := &aea.AgentRecord{} + record.Address = agentAddress + record.PublicKey = agentPubKey + record.PeerPublicKey = peerPubKey + record.Signature = signature + + opts = append(opts, RegisterAgentAddress(record, func() bool { return true })) } if delegatePort != 0 { @@ -216,7 +242,7 @@ func setupLocalDHTPeerForBench(key string, addr string, dhtPort uint16, delegate } func deployPeers(number uint16, b *testing.B) ([]*DHTPeer, []string) { - keys, addrs := getKeysAndAddrs(b) + peerKeys, agentsKeys := getKeysAndAddrs(b) peers := make([]*DHTPeer, 0, number) for i := uint16(0); i < number; i++ { entry := []string{} @@ -224,7 +250,7 @@ func deployPeers(number uint16, b *testing.B) ([]*DHTPeer, []string) { entry = append(entry, peers[i-1].MultiAddr()) } peer, _, err := setupLocalDHTPeerForBench( - keys[i], addrs[i], DefaultLocalPort+i, 0, + peerKeys[i], agentsKeys[i], DefaultLocalPort+i, 0, entry, ) if err != nil { @@ -232,7 +258,7 @@ func deployPeers(number uint16, b *testing.B) ([]*DHTPeer, []string) { } peers = append(peers, peer) } - return peers, addrs + return peers, agentsKeys } func closePeers(peers ...*DHTPeer) { @@ -284,7 +310,7 @@ func benchmarkAgentLookup(npeers uint16, b *testing.B) { defer closePeers(peers...) peer, peerCleanup, err := setupLocalDHTPeerForBench( - FetchAITestKeys[1], AgentsTestAddresses[1], DefaultLocalPort+npeers+1, 0, + FetchAITestKeys[1], AgentsTestKeys[1], DefaultLocalPort+npeers+1, 0, []string{peers[len(peers)-1].MultiAddr()}, ) if err != nil { @@ -310,7 +336,7 @@ func benchmarkPeerJoin(npeers uint16, b *testing.B) { for i := 0; i < b.N; i++ { b.ResetTimer() peer, peerCleanup, err := setupLocalDHTPeerForBench( - FetchAITestKeys[1], AgentsTestAddresses[1], DefaultLocalPort+npeers+1, 0, + FetchAITestKeys[1], AgentsTestKeys[1], DefaultLocalPort+npeers+1, 0, []string{peers[i%len(peers)].MultiAddr()}, ) if err != nil { @@ -330,7 +356,7 @@ func benchmarkPeerEcho(npeers uint16, b *testing.B) { setupEchoServicePeers(peers...) peer, peerCleanup, err := setupLocalDHTPeerForBench( - FetchAITestKeys[1], AgentsTestAddresses[1], DefaultLocalPort+npeers+1, 0, + FetchAITestKeys[1], AgentsTestKeys[1], DefaultLocalPort+npeers+1, 0, []string{peers[len(peers)-1].MultiAddr()}, ) if err != nil { diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go index d0edbd0ee8..cee1ab5e26 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go @@ -22,7 +22,6 @@ package dhtpeer import ( "context" - "encoding/base64" "net" "strconv" "testing" @@ -44,8 +43,6 @@ import ( const ( DefaultLocalHost = "127.0.0.1" DefaultLocalPort = 2000 - DefaultFetchAIKey = "5071fbef50ed1fa1061d84dbf8152c7811f9a3a992ca6c43ae70b80c5ceb56df" - DefaultAgentAddress = "2FRCqDBo7Yw3E2VJc1tAkggppWzLnCCYjPN9zHrQrj8Fupzmkr" DefaultDelegatePort = 3000 EnvelopeDeliveryTimeout = 20 * time.Second @@ -110,40 +107,34 @@ var ( DHT Network: DHTPeer-to-DHTPeer */ -func TestFetchAICrypto(t *testing.T) { - publicKey := "02358e3e42a6ba15cf6b2ba6eb05f02b8893acf82b316d7dd9cda702b0892b8c71" - address := "fetch19dq2mkcpp6x0aypxt9c9gz6n4fqvax0x9a7t5r" - connPublicKey := "027af21aff853b9d9589867ea142b0a60a9611fc8e1fae04c2f7144113fa4e938e" - signatureConnPublicKey_str_canonize := "N/GOa7/m3HU8/gpLJ88VCQ6vXsdrfiiYcqnNtF+c2N9VG9ZIiycykN4hdbpbOCGrChMYZQA3G1GpozsShrUBgg==" - - addressFromPublicKey, _ := utils.FetchAIAddressFromPublicKey(publicKey) - if address != addressFromPublicKey { - t.Error("[ERR] Addresses don't match") - } else { - t.Log("[OK] Agent address matches its public key") - } - - valid, err := utils.VerifyFetchAISignatureBTC([]byte(connPublicKey), signatureConnPublicKey_str_canonize, publicKey) - if !valid { - t.Errorf("Signature using BTC don't match %s", err.Error()) - } - valid, err = utils.VerifyFetchAISignatureLibp2p([]byte(connPublicKey), signatureConnPublicKey_str_canonize, publicKey) - if !valid { - t.Errorf("Signature using LPP don't match %s", err.Error()) - } -} - // TestRoutingDHTPeerToSelf dht peer with agent attached func TestRoutingDHTPeerToSelf(t *testing.T) { opts := []Option{ LocalURI(DefaultLocalHost, DefaultLocalPort), PublicURI(DefaultLocalHost, DefaultLocalPort), - IdentityFromFetchAIKey(DefaultFetchAIKey), - RegisterAgentAddress(DefaultAgentAddress, func() bool { return true }), + IdentityFromFetchAIKey(FetchAITestKeys[0]), EnableRelayService(), EnableDelegateService(DefaultDelegatePort), } + agentPubKey, err := utils.FetchAIPublicKeyFromFetchAIPrivateKey(AgentsTestKeys[0]) + if err != nil { + t.Fatal("Failed at DHTPeer initialization:", err) + } + + signature, err := utils.SignFetchAI([]byte(FetchAITestPublicKeys[0]), AgentsTestKeys[0]) + if err != nil { + t.Fatal("Failed at DHTPeer initialization:", err) + } + + record := &aea.AgentRecord{} + record.Address = AgentsTestAddresses[0] + record.PublicKey = agentPubKey + record.PeerPublicKey = FetchAITestPublicKeys[0] + record.Signature = signature + + opts = append(opts, RegisterAgentAddress(record, func() bool { return true })) + dhtPeer, err := New(opts...) if err != nil { t.Fatal("Failed at DHTPeer initialization:", err) @@ -157,7 +148,7 @@ func TestRoutingDHTPeerToSelf(t *testing.T) { }) err = dhtPeer.RouteEnvelope(&aea.Envelope{ - To: DefaultAgentAddress, + To: AgentsTestAddresses[0], }) if err != nil { t.Error("Failed to Route envelope to local Agent") @@ -172,7 +163,7 @@ func TestRoutingDHTPeerToSelf(t *testing.T) { // TestRoutingDHTPeerToDHTPeerDirect from a dht peer to its bootstrap peer func TestRoutingDHTPeerToDHTPeerDirect(t *testing.T) { dhtPeer1, cleanup1, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -181,7 +172,7 @@ func TestRoutingDHTPeerToDHTPeerDirect(t *testing.T) { defer cleanup1() dhtPeer2, cleanup2, err := SetupLocalDHTPeer( - FetchAITestKeys[1], AgentsTestAddresses[1], DefaultLocalPort+1, DefaultDelegatePort+1, + FetchAITestKeys[1], AgentsTestKeys[1], DefaultLocalPort+1, DefaultDelegatePort+1, []string{dhtPeer1.MultiAddr()}, ) if err != nil { @@ -222,7 +213,7 @@ func TestRoutingDHTPeerToDHTPeerDirect(t *testing.T) { // TestRoutingDHTPeerToDHTPeerIndirect two dht peers connected to the same peer func TestRoutingDHTPeerToDHTPeerIndirect(t *testing.T) { entryPeer, cleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -231,7 +222,7 @@ func TestRoutingDHTPeerToDHTPeerIndirect(t *testing.T) { defer cleanup() dhtPeer1, cleanup1, err := SetupLocalDHTPeer( - FetchAITestKeys[1], AgentsTestAddresses[1], DefaultLocalPort+1, DefaultDelegatePort+1, + FetchAITestKeys[1], AgentsTestKeys[1], DefaultLocalPort+1, DefaultDelegatePort+1, []string{entryPeer.MultiAddr()}, ) if err != nil { @@ -240,7 +231,7 @@ func TestRoutingDHTPeerToDHTPeerIndirect(t *testing.T) { defer cleanup1() dhtPeer2, cleanup2, err := SetupLocalDHTPeer( - FetchAITestKeys[2], AgentsTestAddresses[2], DefaultLocalPort+2, DefaultDelegatePort+2, + FetchAITestKeys[2], AgentsTestKeys[2], DefaultLocalPort+2, DefaultDelegatePort+2, []string{entryPeer.MultiAddr()}, ) if err != nil { @@ -281,7 +272,7 @@ func TestRoutingDHTPeerToDHTPeerIndirect(t *testing.T) { // TestRoutingDHTPeerToDHTPeerIndirectTwoHops two dht peers connected to different peers func TestRoutingDHTPeerToDHTPeerIndirectTwoHops(t *testing.T) { entryPeer1, cleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -290,7 +281,7 @@ func TestRoutingDHTPeerToDHTPeerIndirectTwoHops(t *testing.T) { defer cleanup() entryPeer2, cleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[1], AgentsTestAddresses[1], DefaultLocalPort+1, DefaultDelegatePort+1, + FetchAITestKeys[1], AgentsTestKeys[1], DefaultLocalPort+1, DefaultDelegatePort+1, []string{entryPeer1.MultiAddr()}, ) if err != nil { @@ -300,7 +291,7 @@ func TestRoutingDHTPeerToDHTPeerIndirectTwoHops(t *testing.T) { time.Sleep(1 * time.Second) dhtPeer1, cleanup1, err := SetupLocalDHTPeer( - FetchAITestKeys[2], AgentsTestAddresses[2], DefaultLocalPort+2, DefaultDelegatePort+2, + FetchAITestKeys[2], AgentsTestKeys[2], DefaultLocalPort+2, DefaultDelegatePort+2, []string{entryPeer1.MultiAddr()}, ) if err != nil { @@ -309,7 +300,7 @@ func TestRoutingDHTPeerToDHTPeerIndirectTwoHops(t *testing.T) { defer cleanup1() dhtPeer2, cleanup2, err := SetupLocalDHTPeer( - FetchAITestKeys[3], AgentsTestAddresses[3], DefaultLocalPort+3, DefaultDelegatePort+3, + FetchAITestKeys[3], AgentsTestKeys[3], DefaultLocalPort+3, DefaultDelegatePort+3, []string{entryPeer2.MultiAddr()}, ) if err != nil { @@ -354,7 +345,7 @@ func TestRoutingDHTPeerToDHTPeerFullConnectivity(t *testing.T) { for i := range FetchAITestKeys { peer, cleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[i], AgentsTestAddresses[i], + FetchAITestKeys[i], AgentsTestKeys[i], DefaultLocalPort+uint16(i), DefaultDelegatePort+uint16(i), func() []string { multiaddrs := []string{} @@ -422,7 +413,7 @@ func TestRoutingDHTPeerToDHTPeerFullConnectivity(t *testing.T) { // TestRoutingDHTClientToDHTPeer dht client to its bootstrap peer func TestRoutingDHTClientToDHTPeerX(t *testing.T) { peer, peerCleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -430,7 +421,7 @@ func TestRoutingDHTClientToDHTPeerX(t *testing.T) { } defer peerCleanup() - client, clientCleanup, err := SetupDHTClientWithPoR( + client, clientCleanup, err := SetupDHTClient( FetchAITestKeys[1], AgentsTestKeys[1], []string{peer.MultiAddr()}, ) if err != nil { @@ -470,7 +461,7 @@ func TestRoutingDHTClientToDHTPeerX(t *testing.T) { // TestRoutingDHTClientToDHTPeerIndirect dht client to dht peer different than its bootstrap one func TestRoutingDHTClientToDHTPeerIndirect(t *testing.T) { entryPeer, entryPeerCleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -479,7 +470,7 @@ func TestRoutingDHTClientToDHTPeerIndirect(t *testing.T) { defer entryPeerCleanup() peer, peerCleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[1], AgentsTestAddresses[1], DefaultLocalPort+1, DefaultDelegatePort+1, + FetchAITestKeys[1], AgentsTestKeys[1], DefaultLocalPort+1, DefaultDelegatePort+1, []string{entryPeer.MultiAddr()}, ) if err != nil { @@ -489,7 +480,7 @@ func TestRoutingDHTClientToDHTPeerIndirect(t *testing.T) { time.Sleep(1 * time.Second) client, clientCleanup, err := SetupDHTClient( - FetchAITestKeys[2], AgentsTestAddresses[2], []string{entryPeer.MultiAddr()}, + FetchAITestKeys[2], AgentsTestKeys[2], []string{entryPeer.MultiAddr()}, ) if err != nil { t.Fatal("Failed to initialize DHTClient:", err) @@ -529,7 +520,7 @@ func TestRoutingDHTClientToDHTPeerIndirect(t *testing.T) { // TestRoutingDHTClientToDHTClient dht client to dht client connected to the same peer func TestRoutingDHTClientToDHTClient(t *testing.T) { peer, peerCleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -538,7 +529,7 @@ func TestRoutingDHTClientToDHTClient(t *testing.T) { defer peerCleanup() client1, clientCleanup1, err := SetupDHTClient( - FetchAITestKeys[1], AgentsTestAddresses[1], []string{peer.MultiAddr()}, + FetchAITestKeys[1], AgentsTestKeys[1], []string{peer.MultiAddr()}, ) if err != nil { t.Fatal("Failed to initialize DHTClient:", err) @@ -546,7 +537,7 @@ func TestRoutingDHTClientToDHTClient(t *testing.T) { defer clientCleanup1() client2, clientCleanup2, err := SetupDHTClient( - FetchAITestKeys[2], AgentsTestAddresses[2], []string{peer.MultiAddr()}, + FetchAITestKeys[2], AgentsTestKeys[2], []string{peer.MultiAddr()}, ) if err != nil { t.Fatal("Failed to initialize DHTClient:", err) @@ -585,7 +576,7 @@ func TestRoutingDHTClientToDHTClient(t *testing.T) { // TestRoutingDHTClientToDHTClientIndirect dht client to dht client connected to a different peer func TestRoutingDHTClientToDHTClientIndirect(t *testing.T) { peer1, peerCleanup1, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -594,7 +585,7 @@ func TestRoutingDHTClientToDHTClientIndirect(t *testing.T) { defer peerCleanup1() peer2, peerCleanup2, err := SetupLocalDHTPeer( - FetchAITestKeys[1], AgentsTestAddresses[1], DefaultLocalPort+1, DefaultDelegatePort+1, + FetchAITestKeys[1], AgentsTestKeys[1], DefaultLocalPort+1, DefaultDelegatePort+1, []string{peer1.MultiAddr()}, ) if err != nil { @@ -603,7 +594,7 @@ func TestRoutingDHTClientToDHTClientIndirect(t *testing.T) { defer peerCleanup2() client1, clientCleanup1, err := SetupDHTClient( - FetchAITestKeys[2], AgentsTestAddresses[2], []string{peer1.MultiAddr()}, + FetchAITestKeys[2], AgentsTestKeys[2], []string{peer1.MultiAddr()}, ) if err != nil { t.Fatal("Failed to initialize DHTClient:", err) @@ -611,7 +602,7 @@ func TestRoutingDHTClientToDHTClientIndirect(t *testing.T) { defer clientCleanup1() client2, clientCleanup2, err := SetupDHTClient( - FetchAITestKeys[3], AgentsTestAddresses[3], []string{peer2.MultiAddr()}, + FetchAITestKeys[3], AgentsTestKeys[3], []string{peer2.MultiAddr()}, ) if err != nil { t.Fatal("Failed to initialize DHTClient:", err) @@ -653,27 +644,10 @@ func TestRoutingDHTClientToDHTClientIndirect(t *testing.T) { DHT network: DelegateClient */ -func signFetchAI(message []byte, privKey string) (string, error) { - signingKey, _, err := utils.KeyPairFromFetchAIKey(privKey) - if err != nil { - return "", err - } - signature, err := signingKey.Sign(message) - if err != nil { - return "", err - } - strSignature, err := utils.ConvertDEREncodedSignatureToStr(signature) - if err != nil { - return "", err - } - encodedSignature := base64.StdEncoding.EncodeToString(strSignature) - return encodedSignature, nil -} - // TestRoutingDelegateClientToDHTPeer func TestRoutingDelegateClientToDHTPeerX(t *testing.T) { peer, peerCleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -681,7 +655,7 @@ func TestRoutingDelegateClientToDHTPeerX(t *testing.T) { } defer peerCleanup() - client, clientCleanup, err := SetupDelegateClientWithPoR(AgentsTestKeys[1], DefaultLocalHost, DefaultDelegatePort, FetchAITestPublicKeys[0]) + client, clientCleanup, err := SetupDelegateClient(AgentsTestKeys[1], DefaultLocalHost, DefaultDelegatePort, FetchAITestPublicKeys[0]) if err != nil { t.Fatal("Failed to initialize DelegateClient:", err) } @@ -717,7 +691,7 @@ func TestRoutingDelegateClientToDHTPeerX(t *testing.T) { // TestRoutingDelegateClientToDHTPeerIndirect func TestRoutingDelegateClientToDHTPeerIndirect(t *testing.T) { peer1, peerCleanup1, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -726,7 +700,7 @@ func TestRoutingDelegateClientToDHTPeerIndirect(t *testing.T) { defer peerCleanup1() peer2, peerCleanup2, err := SetupLocalDHTPeer( - FetchAITestKeys[1], AgentsTestAddresses[1], DefaultLocalPort+1, DefaultDelegatePort+1, + FetchAITestKeys[1], AgentsTestKeys[1], DefaultLocalPort+1, DefaultDelegatePort+1, []string{peer1.MultiAddr()}, ) if err != nil { @@ -735,7 +709,7 @@ func TestRoutingDelegateClientToDHTPeerIndirect(t *testing.T) { defer peerCleanup2() time.Sleep(1 * time.Second) - client, clientCleanup, err := SetupDelegateClient(AgentsTestAddresses[2], DefaultLocalHost, DefaultDelegatePort+1) + client, clientCleanup, err := SetupDelegateClient(AgentsTestKeys[2], DefaultLocalHost, DefaultDelegatePort+1, FetchAITestPublicKeys[1]) if err != nil { t.Fatal("Failed to initialize DelegateClient:", err) } @@ -773,7 +747,7 @@ func TestRoutingDelegateClientToDHTPeerIndirect(t *testing.T) { // TestRoutingDelegateClientToDHTPeerIndirectTwoHops func TestRoutingDelegateClientToDHTPeerIndirectTwoHops(t *testing.T) { entryPeer, entryPeerCleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -782,7 +756,7 @@ func TestRoutingDelegateClientToDHTPeerIndirectTwoHops(t *testing.T) { defer entryPeerCleanup() peer1, peerCleanup1, err := SetupLocalDHTPeer( - FetchAITestKeys[1], AgentsTestAddresses[1], DefaultLocalPort+1, DefaultDelegatePort+1, + FetchAITestKeys[1], AgentsTestKeys[1], DefaultLocalPort+1, DefaultDelegatePort+1, []string{entryPeer.MultiAddr()}, ) if err != nil { @@ -791,7 +765,7 @@ func TestRoutingDelegateClientToDHTPeerIndirectTwoHops(t *testing.T) { defer peerCleanup1() peer2, peerCleanup2, err := SetupLocalDHTPeer( - FetchAITestKeys[2], AgentsTestAddresses[2], DefaultLocalPort+2, DefaultDelegatePort+2, + FetchAITestKeys[2], AgentsTestKeys[2], DefaultLocalPort+2, DefaultDelegatePort+2, []string{entryPeer.MultiAddr()}, ) if err != nil { @@ -800,7 +774,7 @@ func TestRoutingDelegateClientToDHTPeerIndirectTwoHops(t *testing.T) { defer peerCleanup2() time.Sleep(1 * time.Second) - client, clientCleanup, err := SetupDelegateClient(AgentsTestAddresses[3], DefaultLocalHost, DefaultDelegatePort+2) + client, clientCleanup, err := SetupDelegateClient(AgentsTestKeys[3], DefaultLocalHost, DefaultDelegatePort+2, FetchAITestPublicKeys[2]) if err != nil { t.Fatal("Failed to initialize DelegateClient:", err) } @@ -838,7 +812,7 @@ func TestRoutingDelegateClientToDHTPeerIndirectTwoHops(t *testing.T) { // TestRoutingDelegateClientToDelegateClient func TestRoutingDelegateClientToDelegateClient(t *testing.T) { _, peerCleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -846,13 +820,13 @@ func TestRoutingDelegateClientToDelegateClient(t *testing.T) { } defer peerCleanup() - client1, clientCleanup1, err := SetupDelegateClient(AgentsTestAddresses[1], DefaultLocalHost, DefaultDelegatePort) + client1, clientCleanup1, err := SetupDelegateClient(AgentsTestKeys[1], DefaultLocalHost, DefaultDelegatePort, FetchAITestPublicKeys[0]) if err != nil { t.Fatal("Failed to initialize DelegateClient:", err) } defer clientCleanup1() - client2, clientCleanup2, err := SetupDelegateClient(AgentsTestAddresses[2], DefaultLocalHost, DefaultDelegatePort) + client2, clientCleanup2, err := SetupDelegateClient(AgentsTestKeys[2], DefaultLocalHost, DefaultDelegatePort, FetchAITestPublicKeys[0]) if err != nil { t.Fatal("Failed to initialize DelegateClient:", err) } @@ -883,7 +857,7 @@ func TestRoutingDelegateClientToDelegateClient(t *testing.T) { // TestRoutingDelegateClientToDelegateClientIndirect func TestRoutingDelegateClientToDelegateClientIndirect(t *testing.T) { peer1, peer1Cleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -892,7 +866,7 @@ func TestRoutingDelegateClientToDelegateClientIndirect(t *testing.T) { defer peer1Cleanup() peer2, peer2Cleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[1], AgentsTestAddresses[1], DefaultLocalPort+1, DefaultDelegatePort+1, + FetchAITestKeys[1], AgentsTestKeys[1], DefaultLocalPort+1, DefaultDelegatePort+1, []string{peer1.MultiAddr()}, ) if err != nil { @@ -900,13 +874,13 @@ func TestRoutingDelegateClientToDelegateClientIndirect(t *testing.T) { } defer peer2Cleanup() - client1, clientCleanup1, err := SetupDelegateClient(AgentsTestAddresses[2], DefaultLocalHost, DefaultDelegatePort) + client1, clientCleanup1, err := SetupDelegateClient(AgentsTestKeys[2], DefaultLocalHost, DefaultDelegatePort, FetchAITestPublicKeys[0]) if err != nil { t.Fatal("Failed to initialize DelegateClient:", err) } defer clientCleanup1() - client2, clientCleanup2, err := SetupDelegateClient(AgentsTestAddresses[3], DefaultLocalHost, DefaultDelegatePort+1) + client2, clientCleanup2, err := SetupDelegateClient(AgentsTestKeys[3], DefaultLocalHost, DefaultDelegatePort+1, FetchAITestPublicKeys[1]) if err != nil { t.Fatal("Failed to initialize DelegateClient:", err) } @@ -938,7 +912,7 @@ func TestRoutingDelegateClientToDelegateClientIndirect(t *testing.T) { // TestRoutingDelegateClientToDHTClientDirect func TestRoutingDelegateClientToDHTClientDirect(t *testing.T) { peer, peerCleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -947,14 +921,14 @@ func TestRoutingDelegateClientToDHTClientDirect(t *testing.T) { defer peerCleanup() dhtClient, dhtClientCleanup, err := SetupDHTClient( - FetchAITestKeys[1], AgentsTestAddresses[1], []string{peer.MultiAddr()}, + FetchAITestKeys[1], AgentsTestKeys[1], []string{peer.MultiAddr()}, ) if err != nil { t.Fatal("Failed to initialize DHTClient:", err) } defer dhtClientCleanup() - delegateClient, delegateClientCleanup, err := SetupDelegateClient(AgentsTestAddresses[2], DefaultLocalHost, DefaultDelegatePort) + delegateClient, delegateClientCleanup, err := SetupDelegateClient(AgentsTestKeys[2], DefaultLocalHost, DefaultDelegatePort, FetchAITestPublicKeys[0]) if err != nil { t.Fatal("Failed to initialize DelegateClient:", err) } @@ -985,7 +959,7 @@ func TestRoutingDelegateClientToDHTClientDirect(t *testing.T) { // TestRoutingDelegateClientToDHTClientIndirect func TestRoutingDelegateClientToDHTClientIndirect(t *testing.T) { peer1, peerCleanup1, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -994,7 +968,7 @@ func TestRoutingDelegateClientToDHTClientIndirect(t *testing.T) { defer peerCleanup1() peer2, peerCleanup2, err := SetupLocalDHTPeer( - FetchAITestKeys[1], AgentsTestAddresses[1], DefaultLocalPort+1, DefaultDelegatePort+1, + FetchAITestKeys[1], AgentsTestKeys[1], DefaultLocalPort+1, DefaultDelegatePort+1, []string{peer1.MultiAddr()}, ) if err != nil { @@ -1003,7 +977,7 @@ func TestRoutingDelegateClientToDHTClientIndirect(t *testing.T) { defer peerCleanup2() dhtClient, dhtClientCleanup, err := SetupDHTClient( - FetchAITestKeys[2], AgentsTestAddresses[2], []string{peer1.MultiAddr()}, + FetchAITestKeys[2], AgentsTestKeys[2], []string{peer1.MultiAddr()}, ) if err != nil { t.Fatal("Failed to initialize DHTClient:", err) @@ -1011,7 +985,7 @@ func TestRoutingDelegateClientToDHTClientIndirect(t *testing.T) { defer dhtClientCleanup() delegateClient, delegateClientCleanup, err := SetupDelegateClient( - AgentsTestAddresses[3], DefaultLocalHost, DefaultDelegatePort+1, + AgentsTestKeys[3], DefaultLocalHost, DefaultDelegatePort+1, FetchAITestPublicKeys[1], ) if err != nil { t.Fatal("Failed to initialize DelegateClient:", err) @@ -1066,7 +1040,7 @@ func TestRoutingAllToAll(t *testing.T) { // setup DHTPeers dhtPeer1, dhtPeerCleanup1, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestAddresses[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -1094,7 +1068,7 @@ func TestRoutingAllToAll(t *testing.T) { }) dhtPeer2, dhtPeerCleanup2, err := SetupLocalDHTPeer( - FetchAITestKeys[1], AgentsTestAddresses[1], DefaultLocalPort+1, DefaultDelegatePort+1, + FetchAITestKeys[1], AgentsTestKeys[1], DefaultLocalPort+1, DefaultDelegatePort+1, []string{dhtPeer1.MultiAddr()}, ) if err != nil { @@ -1122,7 +1096,7 @@ func TestRoutingAllToAll(t *testing.T) { }) dhtPeer3, dhtPeerCleanup3, err := SetupLocalDHTPeer( - FetchAITestKeys[2], AgentsTestAddresses[2], DefaultLocalPort+2, DefaultDelegatePort+2, + FetchAITestKeys[2], AgentsTestKeys[2], DefaultLocalPort+2, DefaultDelegatePort+2, []string{dhtPeer1.MultiAddr()}, ) if err != nil { @@ -1150,7 +1124,7 @@ func TestRoutingAllToAll(t *testing.T) { }) dhtPeer4, dhtPeerCleanup4, err := SetupLocalDHTPeer( - FetchAITestKeys[3], AgentsTestAddresses[3], DefaultLocalPort+3, DefaultDelegatePort+3, + FetchAITestKeys[3], AgentsTestKeys[3], DefaultLocalPort+3, DefaultDelegatePort+3, []string{dhtPeer2.MultiAddr()}, ) if err != nil { @@ -1180,7 +1154,7 @@ func TestRoutingAllToAll(t *testing.T) { // setup DHTClients dhtClient1, dhtClientCleanup1, err := SetupDHTClient( - FetchAITestKeys[4], AgentsTestAddresses[4], []string{dhtPeer3.MultiAddr()}, + FetchAITestKeys[4], AgentsTestKeys[4], []string{dhtPeer3.MultiAddr()}, ) if err != nil { t.Fatal("Failed to initialize DHTClient:", err) @@ -1207,7 +1181,7 @@ func TestRoutingAllToAll(t *testing.T) { }) dhtClient2, dhtClientCleanup2, err := SetupDHTClient( - FetchAITestKeys[5], AgentsTestAddresses[5], []string{dhtPeer3.MultiAddr()}, + FetchAITestKeys[5], AgentsTestKeys[5], []string{dhtPeer3.MultiAddr()}, ) if err != nil { t.Fatal("Failed to initialize DHTClient:", err) @@ -1234,7 +1208,7 @@ func TestRoutingAllToAll(t *testing.T) { }) dhtClient3, dhtClientCleanup3, err := SetupDHTClient( - FetchAITestKeys[6], AgentsTestAddresses[6], []string{dhtPeer4.MultiAddr()}, + FetchAITestKeys[6], AgentsTestKeys[6], []string{dhtPeer4.MultiAddr()}, ) if err != nil { t.Fatal("Failed to initialize DHTClient:", err) @@ -1263,7 +1237,7 @@ func TestRoutingAllToAll(t *testing.T) { // setup DelegateClients delegateClient1, delegateClientCleanup1, err := SetupDelegateClient( - AgentsTestAddresses[7], DefaultLocalHost, DefaultDelegatePort+2, + AgentsTestKeys[7], DefaultLocalHost, DefaultDelegatePort+2, FetchAITestPublicKeys[2], ) if err != nil { t.Fatal("Failed to initialize DelegateClient:", err) @@ -1290,7 +1264,7 @@ func TestRoutingAllToAll(t *testing.T) { }) delegateClient2, delegateClientCleanup2, err := SetupDelegateClient( - AgentsTestAddresses[8], DefaultLocalHost, DefaultDelegatePort+3, + AgentsTestKeys[8], DefaultLocalHost, DefaultDelegatePort+3, FetchAITestPublicKeys[3], ) if err != nil { t.Fatal("Failed to initialize DelegateClient:", err) @@ -1317,7 +1291,7 @@ func TestRoutingAllToAll(t *testing.T) { }) delegateClient3, delegateClientCleanup3, err := SetupDelegateClient( - AgentsTestAddresses[9], DefaultLocalHost, DefaultDelegatePort+3, + AgentsTestKeys[9], DefaultLocalHost, DefaultDelegatePort+3, FetchAITestPublicKeys[3], ) if err != nil { t.Fatal("Failed to initialize DelegateClient:", err) @@ -1382,39 +1356,38 @@ func TestRoutingAllToAll(t *testing.T) { } -/* - Helpers - TOFIX(LR) how to share test helpers between packages tests - without having circular dependencies -*/ +// Crypto operations -func SetupLocalDHTPeer(key string, addr string, dhtPort uint16, delegatePort uint16, entry []string) (*DHTPeer, func(), error) { - opts := []Option{ - LocalURI(DefaultLocalHost, dhtPort), - PublicURI(DefaultLocalHost, dhtPort), - IdentityFromFetchAIKey(key), - EnableRelayService(), - BootstrapFrom(entry), - } +func TestFetchAICrypto(t *testing.T) { + publicKey := "02358e3e42a6ba15cf6b2ba6eb05f02b8893acf82b316d7dd9cda702b0892b8c71" + address := "fetch19dq2mkcpp6x0aypxt9c9gz6n4fqvax0x9a7t5r" + peerPublicKey := "027af21aff853b9d9589867ea142b0a60a9611fc8e1fae04c2f7144113fa4e938e" + pySigStrCanonize := "N/GOa7/m3HU8/gpLJ88VCQ6vXsdrfiiYcqnNtF+c2N9VG9ZIiycykN4hdbpbOCGrChMYZQA3G1GpozsShrUBgg==" - if addr != "" { - opts = append(opts, RegisterAgentAddress(addr, func() bool { return true })) + addressFromPublicKey, _ := utils.FetchAIAddressFromPublicKey(publicKey) + if address != addressFromPublicKey { + t.Error("[ERR] Addresses don't match") + } else { + t.Log("[OK] Agent address matches its public key") } - if delegatePort != 0 { - opts = append(opts, EnableDelegateService(delegatePort)) + valid, err := utils.VerifyFetchAISignatureBTC([]byte(peerPublicKey), pySigStrCanonize, publicKey) + if !valid { + t.Errorf("Signature using BTC don't match %s", err.Error()) } - - dhtPeer, err := New(opts...) - if err != nil { - return nil, nil, err + valid, err = utils.VerifyFetchAISignatureLibp2p([]byte(peerPublicKey), pySigStrCanonize, publicKey) + if !valid { + t.Errorf("Signature using LPP don't match %s", err.Error()) } - - return dhtPeer, func() { dhtPeer.Close() }, nil - } -func SetupLocalDHTPeerWithPoR(key string, agentKey string, dhtPort uint16, delegatePort uint16, entry []string) (*DHTPeer, func(), error) { +/* + Helpers + TOFIX(LR) how to share test helpers between packages tests + without having circular dependencies +*/ + +func SetupLocalDHTPeer(key string, agentKey string, dhtPort uint16, delegatePort uint16, entry []string) (*DHTPeer, func(), error) { opts := []Option{ LocalURI(DefaultLocalHost, dhtPort), PublicURI(DefaultLocalHost, dhtPort), @@ -1439,7 +1412,7 @@ func SetupLocalDHTPeerWithPoR(key string, agentKey string, dhtPort uint16, deleg return nil, nil, err } - signature, err := signFetchAI([]byte(peerPubKey), agentKey) + signature, err := utils.SignFetchAI([]byte(peerPubKey), agentKey) if err != nil { return nil, nil, err } @@ -1450,7 +1423,7 @@ func SetupLocalDHTPeerWithPoR(key string, agentKey string, dhtPort uint16, deleg record.PeerPublicKey = peerPubKey record.Signature = signature - opts = append(opts, RegisterAgentAddressWithPoR(record, func() bool { return true })) + opts = append(opts, RegisterAgentAddress(record, func() bool { return true })) } if delegatePort != 0 { @@ -1468,22 +1441,7 @@ func SetupLocalDHTPeerWithPoR(key string, agentKey string, dhtPort uint16, deleg // DHTClient -func SetupDHTClient(key string, address string, entry []string) (*dhtclient.DHTClient, func(), error) { - opts := []dhtclient.Option{ - dhtclient.IdentityFromFetchAIKey(key), - dhtclient.RegisterAgentAddress(address, func() bool { return true }), - dhtclient.BootstrapFrom(entry), - } - - dhtClient, err := dhtclient.New(opts...) - if err != nil { - return nil, nil, err - } - - return dhtClient, func() { dhtClient.Close() }, nil -} - -func SetupDHTClientWithPoR(key string, agentKey string, entry []string) (*dhtclient.DHTClient, func(), error) { +func SetupDHTClient(key string, agentKey string, entry []string) (*dhtclient.DHTClient, func(), error) { agentPubKey, err := utils.FetchAIPublicKeyFromFetchAIPrivateKey(agentKey) if err != nil { @@ -1500,7 +1458,7 @@ func SetupDHTClientWithPoR(key string, agentKey string, entry []string) (*dhtcli return nil, nil, err } - signature, err := signFetchAI([]byte(peerPubKey), agentKey) + signature, err := utils.SignFetchAI([]byte(peerPubKey), agentKey) if err != nil { return nil, nil, err } @@ -1513,7 +1471,7 @@ func SetupDHTClientWithPoR(key string, agentKey string, entry []string) (*dhtcli opts := []dhtclient.Option{ dhtclient.IdentityFromFetchAIKey(key), - dhtclient.RegisterAgentAddressWithPoR(record, func() bool { return true }), + dhtclient.RegisterAgentAddress(record, func() bool { return true }), dhtclient.BootstrapFrom(entry), } @@ -1551,42 +1509,7 @@ func (client *DelegateClient) ProcessEnvelope(fn func(*aea.Envelope) error) { client.processEnvelope = fn } -func SetupDelegateClient(address string, host string, port uint16) (*DelegateClient, func(), error) { - var err error - client := &DelegateClient{} - client.AgentAddress = address - client.Rx = make(chan *aea.Envelope) - client.Conn, err = net.Dial("tcp", host+":"+strconv.FormatInt(int64(port), 10)) - if err != nil { - return nil, nil, err - } - - err = utils.WriteBytesConn(client.Conn, []byte(address)) - ignore(err) - _, err = utils.ReadBytesConn(client.Conn) - if err != nil { - return nil, nil, err - } - - go func() { - for { - envel, err := utils.ReadEnvelopeConn(client.Conn) - if err != nil { - break - } - if client.processEnvelope != nil { - err = client.processEnvelope(envel) - ignore(err) - } else { - client.Rx <- envel - } - } - }() - - return client, func() { client.Close() }, nil -} - -func SetupDelegateClientWithPoR(key string, host string, port uint16, peerPubKey string) (*DelegateClient, func(), error) { +func SetupDelegateClient(key string, host string, port uint16, peerPubKey string) (*DelegateClient, func(), error) { var err error client := &DelegateClient{} client.AgentKey = key @@ -1603,7 +1526,7 @@ func SetupDelegateClientWithPoR(key string, host string, port uint16, peerPubKey } client.AgentAddress = address - signature, err := signFetchAI([]byte(peerPubKey), key) + signature, err := utils.SignFetchAI([]byte(peerPubKey), key) if err != nil { return nil, nil, err } diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/options.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/options.go index 2dd4badca2..0377a0f64e 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/options.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/options.go @@ -48,15 +48,7 @@ func IdentityFromFetchAIKey(key string) Option { } // RegisterAgentAddress for dhtpeer.New -func RegisterAgentAddress(addr string, isReady func() bool) Option { - return func(dhtPeer *DHTPeer) error { - dhtPeer.myAgentAddress = addr - dhtPeer.myAgentReady = isReady - return nil - } -} - -func RegisterAgentAddressWithPoR(record *aea.AgentRecord, isReady func() bool) Option { +func RegisterAgentAddress(record *aea.AgentRecord, isReady func() bool) Option { return func(dhtPeer *DHTPeer) error { pbRecord := &dhtnode.AgentRecord{} pbRecord.Address = record.Address diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhttests/dhttests.go b/packages/fetchai/connections/p2p_libp2p/dht/dhttests/dhttests.go index ed99ab66d5..8a0bb842fc 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhttests/dhttests.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhttests/dhttests.go @@ -24,6 +24,7 @@ package dhttests import ( "libp2p_node/aea" "libp2p_node/dht/dhtpeer" + "libp2p_node/utils" "log" ) @@ -31,12 +32,14 @@ import ( const ( DHTPeerDefaultLocalHost = "127.0.0.1" DHTPeerDefaultLocalPort = 2000 - DHTPeerDefaultFetchAIKey = "5071fbef50ed1fa1061d84dbf8152c7811f9a3a992ca6c43ae70b80c5ceb56df" - DHTPeerDefaultAgentAddress = "2FRCqDBo7Yw3E2VJc1tAkggppWzLnCCYjPN9zHrQrj8Fupzmkr" DHTPeerDefaultDelegatePort = 3000 - DHTClientDefaultFetchAIKey = "3916b301d1a0ec09de1db4833b0c945531004290caee0b4a5d7b554caa39dbf1" - DHTClientDefaultAgentAddress = "2TsHmM9JXeFgK928LYc6HV96gi78pBv6sWprJAXaS6ydg9MTC6" + DHTPeerDefaultFetchAIKey = "34604436e55b0eb99b5e62508433e172dd3ee133cf7a2fecb705e69611147605" + DHTPeerDefaultFetchAIPublicKey = "039e883de988eededb9afaa4d3a6baec9ba74dd1cc237028e810569780b319940a" + + DHTPeerDefaultAgentKey = "719133dc740d76ff6d1d325e193f7cd63af4c8f3491bfe3010e58b0b58c77795" + DHTPeerDefaultAgentPublicKey = "039623e63ba1617404b2abbe7bd94d24eb788335f870fac1ae4519e9bc359b7833" + DHTPeerDefaultAgentAddress = "fetch134rg4n3wgmwctxsrm7gp6l65uwv6hxtxyfdwgw" ) // NewDHTPeerWithDefaults for testing @@ -45,11 +48,23 @@ func NewDHTPeerWithDefaults(inbox chan<- *aea.Envelope) (*dhtpeer.DHTPeer, func( dhtpeer.LocalURI(DHTPeerDefaultLocalHost, DHTPeerDefaultLocalPort), dhtpeer.PublicURI(DHTPeerDefaultLocalHost, DHTPeerDefaultLocalPort), dhtpeer.IdentityFromFetchAIKey(DHTPeerDefaultFetchAIKey), - dhtpeer.RegisterAgentAddress(DHTPeerDefaultAgentAddress, func() bool { return true }), dhtpeer.EnableRelayService(), dhtpeer.EnableDelegateService(DHTPeerDefaultDelegatePort), } + signature, err := utils.SignFetchAI([]byte(DHTPeerDefaultFetchAIPublicKey), DHTPeerDefaultAgentKey) + if err != nil { + return nil, nil, err + } + + record := &aea.AgentRecord{} + record.Address = DHTPeerDefaultAgentAddress + record.PublicKey = DHTPeerDefaultAgentPublicKey + record.PeerPublicKey = DHTPeerDefaultFetchAIPublicKey + record.Signature = signature + + opts = append(opts, dhtpeer.RegisterAgentAddress(record, func() bool { return true })) + dhtPeer, err := dhtpeer.New(opts...) if err != nil { return nil, nil, err @@ -58,7 +73,7 @@ func NewDHTPeerWithDefaults(inbox chan<- *aea.Envelope) (*dhtpeer.DHTPeer, func( cleanup := func() { errs := dhtPeer.Close() if len(errs) > 0 { - log.Println("ERROR while stoping DHTPeer:", errs) + log.Println("ERROR while stopping DHTPeer:", errs) } } diff --git a/packages/fetchai/connections/p2p_libp2p/go.mod b/packages/fetchai/connections/p2p_libp2p/go.mod index 1ae69cece4..dba9c4c8c5 100644 --- a/packages/fetchai/connections/p2p_libp2p/go.mod +++ b/packages/fetchai/connections/p2p_libp2p/go.mod @@ -3,20 +3,31 @@ module libp2p_node go 1.13 require ( + github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4 // indirect github.com/btcsuite/btcd v0.20.1-beta github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d + github.com/dave/dst v0.26.2 // indirect github.com/golang/protobuf v1.4.2 github.com/ipfs/go-cid v0.0.5 github.com/joho/godotenv v1.3.0 + github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect github.com/libp2p/go-libp2p v0.8.3 github.com/libp2p/go-libp2p-circuit v0.2.2 github.com/libp2p/go-libp2p-core v0.5.3 github.com/libp2p/go-libp2p-kad-dht v0.7.11 + github.com/mattn/go-colorable v0.1.8 // indirect + github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/multiformats/go-multiaddr v0.2.1 github.com/multiformats/go-multihash v0.0.13 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.7.1 github.com/rs/zerolog v1.19.0 - golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d + github.com/segmentio/golines v0.0.0-20200824192126-7f30d3046793 // indirect + github.com/sirupsen/logrus v1.7.0 // indirect + golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 + golang.org/x/mod v0.4.0 // indirect + golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e // indirect + golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect + golang.org/x/tools v0.0.0-20201215171152-6307297f4651 // indirect google.golang.org/protobuf v1.25.0 ) diff --git a/packages/fetchai/connections/p2p_libp2p/go.sum b/packages/fetchai/connections/p2p_libp2p/go.sum index e0ef651797..ba6266ffa8 100644 --- a/packages/fetchai/connections/p2p_libp2p/go.sum +++ b/packages/fetchai/connections/p2p_libp2p/go.sum @@ -7,9 +7,14 @@ github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETF github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4 h1:EBTWhcAX7rNQ80RLwLCpHZBBrJuzallFHnF+yMXo928= +github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -42,6 +47,14 @@ github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmf github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/dave/dst v0.23.1 h1:2obX6c3RqALrEOp6u01qsqPvwp0t+RpOp9O4Bf9KhXs= +github.com/dave/dst v0.23.1/go.mod h1:LjPcLEauK4jC5hQ1fE/wr05O41zK91Pr4Qs22Ljq7gs= +github.com/dave/dst v0.26.2 h1:lnxLAKI3tx7MgLNVDirFCsDTlTG9nKTk7GcptKcWSwY= +github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= +github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= +github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -58,6 +71,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= +github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= @@ -99,6 +114,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= +github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -121,6 +137,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -187,6 +204,9 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -351,11 +371,23 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= @@ -421,10 +453,12 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= @@ -459,8 +493,14 @@ github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.19.0 h1:hYz4ZVdUgjXTBUmrkrw55j1nHx68LfOKIQk5IYtyScg= github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/segmentio/golines v0.0.0-20200824192126-7f30d3046793 h1:rhR7esJSmty+9ST6Gsp7mlQHkpISw2DiYjuFaz3dRDg= +github.com/segmentio/golines v0.0.0-20200824192126-7f30d3046793/go.mod h1:bQSh5qdVR67XiCKbaVvYO41s50c5hQo+3cY/1CQQ3xQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= @@ -494,8 +534,11 @@ github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvX github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -514,8 +557,10 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -528,6 +573,9 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 h1:sYNJzB4J8toYPQTM6pAkcmBRgw9SnQKP9oXCHfgy604= +golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -536,6 +584,11 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -549,6 +602,9 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -557,7 +613,9 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -573,19 +631,34 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191024172528-b4ff53e7a1cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M= +golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181127232545-e782529d0ddd/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -593,16 +666,23 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191024220359-3d91e92cde03/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnfG5kSmgy9KZR9sW3W5QeA= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201215171152-6307297f4651 h1:bdfqbHwYVvhLEIkESR524rqSsmV06Og3Fgz60LE7vZc= +golang.org/x/tools v0.0.0-20201215171152-6307297f4651/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -626,6 +706,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -634,6 +715,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= diff --git a/packages/fetchai/connections/p2p_libp2p/libp2p_node.go b/packages/fetchai/connections/p2p_libp2p/libp2p_node.go index 020bb5a716..d3d89c1d56 100644 --- a/packages/fetchai/connections/p2p_libp2p/libp2p_node.go +++ b/packages/fetchai/connections/p2p_libp2p/libp2p_node.go @@ -106,11 +106,8 @@ func main() { dhtclient.IdentityFromFetchAIKey(key), dhtclient.BootstrapFrom(entryPeers), } - if aeaAddr != "" { - opts = append(opts, dhtclient.RegisterAgentAddress(aeaAddr, agent.Connected)) - } if record != nil { - opts = append(opts, dhtclient.RegisterAgentAddressWithPoR(record, agent.Connected)) + opts = append(opts, dhtclient.RegisterAgentAddress(record, agent.Connected)) } node, err = dhtclient.New(opts...) } else { @@ -122,11 +119,8 @@ func main() { dhtpeer.EnableDelegateService(nodePortDelegate), dhtpeer.BootstrapFrom(entryPeers), } - if aeaAddr != "" { - opts = append(opts, dhtpeer.RegisterAgentAddress(aeaAddr, agent.Connected)) - } if record != nil { - opts = append(opts, dhtPeer.RegisterAgentAddressWithPoR(record, agent.Connected)) + opts = append(opts, dhtpeer.RegisterAgentAddress(record, agent.Connected)) } if nodePortMonitoring != 0 { opts = append(opts, dhtpeer.EnablePrometheusMonitoring(nodePortMonitoring)) diff --git a/packages/fetchai/connections/p2p_libp2p/utils/utils.go b/packages/fetchai/connections/p2p_libp2p/utils/utils.go index b81d1e537c..89e641d669 100644 --- a/packages/fetchai/connections/p2p_libp2p/utils/utils.go +++ b/packages/fetchai/connections/p2p_libp2p/utils/utils.go @@ -321,6 +321,23 @@ func VerifyFetchAISignatureLibp2p(message []byte, signature string, pubkey strin return verifyKey.Verify(message, sigDER) } +func SignFetchAI(message []byte, privKey string) (string, error) { + signingKey, _, err := KeyPairFromFetchAIKey(privKey) + if err != nil { + return "", err + } + signature, err := signingKey.Sign(message) + if err != nil { + return "", err + } + strSignature, err := ConvertDEREncodedSignatureToStr(signature) + if err != nil { + return "", err + } + encodedSignature := base64.StdEncoding.EncodeToString(strSignature) + return encodedSignature, nil +} + // KeyPairFromFetchAIKey key pair from hex encoded secp256k1 private key func KeyPairFromFetchAIKey(key string) (crypto.PrivKey, crypto.PubKey, error) { pk_bytes, err := hex.DecodeString(key) diff --git a/packages/hashes.csv b/packages/hashes.csv index 8ac88c5715..5df3e179ba 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,7 +34,7 @@ fetchai/connections/http_server,QmP3nhFMyEi7N64oLV5EQDNowhS8be3jVefxkwir4PKpQ9 fetchai/connections/ledger,QmaViv1ZK4jLgQXycpYLjboFFcznN9zXUZT4JKdbeS2wQ8 fetchai/connections/local,QmNPNVkqtDtzENpv1XqM4LHZna6v6jJ7Hsk6RB14iSdtjG fetchai/connections/oef,QmVGcKDeDMEhtcBnDNVTWchHkA2YhHnDGoK8izobnDQKmw -fetchai/connections/p2p_libp2p,Qma5WZXDxEUnwmwcgDNKgXxT1TviZHHJSdShhjjzZhm3Nk +fetchai/connections/p2p_libp2p,QmbtMkbfnZjBfUfhSrC7arTMbssrdXBhar2S1uuoCNhE9T fetchai/connections/p2p_libp2p_client,QmPV4eyGijvdqUzMAfgBrTDnp5rhJhSH4YbyyZXmnar8hk fetchai/connections/p2p_stub,QmaHtQs9dJRnF27WDZSVW3FFEGbY1419NH8u67B8hgnteV fetchai/connections/scaffold,QmcjctNNxnFZ6xU2aDs7WvzNHMvvv4PuhnS739yWxHmJKz From 18028759d01e995fded67218d55d257b37b0f25a Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Tue, 15 Dec 2020 21:38:21 +0000 Subject: [PATCH 006/204] Address golangci-lint comments --- .../connections/p2p_libp2p/connection.yaml | 8 +++--- .../p2p_libp2p/dht/dhtclient/dhtclient.go | 6 ++--- .../p2p_libp2p/dht/dhtpeer/dhtpeer.go | 27 +++++++++++-------- .../p2p_libp2p/dht/dhtpeer/dhtpeer_test.go | 6 ++--- .../connections/p2p_libp2p/utils/utils.go | 2 +- packages/hashes.csv | 2 +- 6 files changed, 28 insertions(+), 23 deletions(-) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index eec20053ac..b5f693ffe5 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -17,7 +17,7 @@ fingerprint: aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK check_dependencies.py: QmY8EiWapBspyR3zKGEY3TnQayz7hF7DxZ2d9zozVfFhtA connection.py: QmVf4XX25S8FQHcZca6o1xC77Ez9dUNp66K3dGnVB724GC - dht/dhtclient/dhtclient.go: QmeKS6wgfeoYDaHRfb9FEDSCT2fDYnRXZxmEjc8yz7Zbec + dht/dhtclient/dhtclient.go: QmQSCmviTBg9GRAHkRWsyt3uppTfLYseHie6KVLSSeVd2L dht/dhtclient/dhtclient_test.go: QmYVnf5j8Jyahi5jZNswkGR2tdnM4D1r5Fjdrz654EcD1F dht/dhtclient/options.go: QmWs7fQqHRavjwe1PjihHFi85HeJu9qSEVyMdcRviZzaT2 dht/dhtnode/dhtnode.go: QmbyhgbCSAbQ1QsDw7FM7Nt5sZcvhbupA1jv5faxutbV7N @@ -26,8 +26,8 @@ fingerprint: dht/dhtnode/streams.go: Qmc2JcyiU4wHsgDj6aUunMAp4c5yMzo2ixeqRZHSW5PVwo dht/dhtnode/utils.go: QmZi91a8RWrUUM6LztpMWBhLKj47W7UwMoTvooptF6oNHU dht/dhtpeer/benchmarks_test.go: QmUj4zGAG8QgDVFf9PeP5kKdjUbqBi5nzdYmhrUfX2eZBD - dht/dhtpeer/dhtpeer.go: QmWDqiqMiz7MNZAiJLLZnNWCjgCSwyjwMbPPf6bQgSB26G - dht/dhtpeer/dhtpeer_test.go: QmVn5BRKtyBAqR52RH5frBGiynyPoaWr9737bZZniMGgex + dht/dhtpeer/dhtpeer.go: QmTYaRrrRphDfK3JUa52qFFZ1FgP7LSEnen4628me492RX + dht/dhtpeer/dhtpeer_test.go: QmbAKGzUvotDNrjk4QWAUUD4vhgTmeeTGrFjo5rJBwjfkw dht/dhtpeer/options.go: Qme5L9bRBQZaR5fciPJ1feFwqaQpkdYfW8TaiY1TDXs91H dht/dhttests/dhttests.go: QmQJxw2rJCGUFTXKxH3sf3Zaqta7H76EvaB7mWNNfikFkA dht/monitoring/file.go: QmZB8pNf7infMXWaC5A46rr3mghNfkuSd8UKH4Ceg8nDSM @@ -36,7 +36,7 @@ fingerprint: go.mod: QmScMYT69B7HbQApxXmZFpSwsKBZVPAJimVFYpTVYTvzbe go.sum: QmawFS81vDFYVZUZMKxQjNdKkLxDzfRRfqD1Tcpzze66yd libp2p_node.go: QmSMPBU6GMJ7gAHWMsYGVDSUv5sw48x2rvNKydcxVv7MqL - utils/utils.go: QmcgF9WCrP74kmQSSQWFyhbNBV9QwfXiF6J9BP7Yqnuyva + utils/utils.go: QmcXJwBg95H627cnzgkqgcEDmBCNum74KVnPEvPkhw4BDm fingerprint_ignore_patterns: [] build_entrypoint: check_dependencies.py connections: [] diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go index 9f1c335667..90f459dd46 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go @@ -109,9 +109,8 @@ func New(opts ...Option) (*DHTClient, error) { } // agent record is mandatory - // FIXME if dhtClient.myAgentRecord == nil { - // return nil, errors.New("missing agent record") + return nil, errors.New("missing agent record") } // check if the PoR is delivered for my public key @@ -542,8 +541,9 @@ func (dhtClient *DHTClient) registerAgentAddress() error { Msgf("registering addr and peerID to relay peer") registration := &dhtnode.Register{Record: dhtClient.myAgentRecord} - msg := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Register{registration}} + msg := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Register{Register: registration}} buf, err := proto.Marshal(msg) + ignore(err) err = utils.WriteBytes(stream, buf) if err != nil { diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go index f27346aec4..500770b162 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go @@ -35,8 +35,8 @@ import ( "sync" "time" - "github.com/golang/protobuf/proto" "github.com/rs/zerolog" + "google.golang.org/protobuf/proto" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p-core/crypto" @@ -526,8 +526,9 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { if err != nil { lerror(err).Msg("couldn't deserialize acn registration message") status := &dhtnode.Status{Code: dhtnode.Status_ERROR_GENERIC, Msgs: []string{err.Error()}} - response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} + response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) + ignore(err) err = utils.WriteBytesConn(conn, buf) ignore(err) @@ -544,9 +545,10 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { register = pl.Register default: err = errors.New("Unexpected payload") - status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNEXPECTED_PAYLOAD} - response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNEXPECTED_PAYLOAD, Msgs: []string{err.Error()}} + response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) + ignore(err) err = utils.WriteBytesConn(conn, buf) ignore(err) @@ -565,8 +567,9 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { status, err := dhtnode.IsValidProofOfRepresentation(record, myPubKey) if err != nil { lerror(err).Msg("PoR is not valid") - response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} + response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) + ignore(err) err = utils.WriteBytesConn(conn, buf) ignore(err) @@ -574,8 +577,9 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { return } - msg = &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} + msg = &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(msg) + ignore(err) err = utils.WriteBytesConn(conn, buf) if err != nil { nbrConns.Dec() @@ -1053,7 +1057,7 @@ func (dhtPeer *DHTPeer) handleAeaRegisterStream(stream network.Stream) { if err != nil { lerror(err).Msg("couldn't deserialize acn registration message") status := &dhtnode.Status{Code: dhtnode.Status_ERROR_GENERIC, Msgs: []string{err.Error()}} - response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} + response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) ignore(err) err = utils.WriteBytes(stream, buf) @@ -1072,8 +1076,8 @@ func (dhtPeer *DHTPeer) handleAeaRegisterStream(stream network.Stream) { register = pl.Register default: err = errors.New("Unexpected payload") - status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNEXPECTED_PAYLOAD} - response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNEXPECTED_PAYLOAD, Msgs: []string{err.Error()}} + response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) ignore(err) err = utils.WriteBytes(stream, buf) @@ -1094,7 +1098,7 @@ func (dhtPeer *DHTPeer) handleAeaRegisterStream(stream network.Stream) { status, err := dhtnode.IsValidProofOfRepresentation(record, clientPubKey) if err != nil { lerror(err).Msg("PoR is not valid") - response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} + response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) ignore(err) err = utils.WriteBytes(stream, buf) @@ -1104,8 +1108,9 @@ func (dhtPeer *DHTPeer) handleAeaRegisterStream(stream network.Stream) { return } - msg = &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{status}} + msg = &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(msg) + ignore(err) err = utils.WriteBytes(stream, buf) if err != nil { err = stream.Reset() diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go index cee1ab5e26..3ea6bd9659 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go @@ -32,8 +32,8 @@ import ( "libp2p_node/dht/dhtnode" "libp2p_node/utils" - "github.com/golang/protobuf/proto" "github.com/pkg/errors" + "google.golang.org/protobuf/proto" ) /* @@ -1544,9 +1544,9 @@ func SetupDelegateClient(key string, host string, port uint16, peerPubKey string record.PeerPublicKey = peerPubKey record.Signature = signature registration := &dhtnode.Register{Record: record} - msg := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Register{registration}} + msg := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Register{Register: registration}} data, err := proto.Marshal(msg) - + ignore(err) err = utils.WriteBytesConn(client.Conn, data) ignore(err) data, err = utils.ReadBytesConn(client.Conn) diff --git a/packages/fetchai/connections/p2p_libp2p/utils/utils.go b/packages/fetchai/connections/p2p_libp2p/utils/utils.go index 89e641d669..6ca810d3a5 100644 --- a/packages/fetchai/connections/p2p_libp2p/utils/utils.go +++ b/packages/fetchai/connections/p2p_libp2p/utils/utils.go @@ -42,7 +42,7 @@ import ( "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multihash" "github.com/rs/zerolog" - "golang.org/x/crypto/ripemd160" + "golang.org/x/crypto/ripemd160" // nolint:staticcheck host "github.com/libp2p/go-libp2p-core/host" peerstore "github.com/libp2p/go-libp2p-core/peerstore" diff --git a/packages/hashes.csv b/packages/hashes.csv index 5df3e179ba..f9fcd3ab76 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,7 +34,7 @@ fetchai/connections/http_server,QmP3nhFMyEi7N64oLV5EQDNowhS8be3jVefxkwir4PKpQ9 fetchai/connections/ledger,QmaViv1ZK4jLgQXycpYLjboFFcznN9zXUZT4JKdbeS2wQ8 fetchai/connections/local,QmNPNVkqtDtzENpv1XqM4LHZna6v6jJ7Hsk6RB14iSdtjG fetchai/connections/oef,QmVGcKDeDMEhtcBnDNVTWchHkA2YhHnDGoK8izobnDQKmw -fetchai/connections/p2p_libp2p,QmbtMkbfnZjBfUfhSrC7arTMbssrdXBhar2S1uuoCNhE9T +fetchai/connections/p2p_libp2p,QmYBHSwej3fTiPkf5prigYn9sA3J8YdxKkYCMFVUwGFbdZ fetchai/connections/p2p_libp2p_client,QmPV4eyGijvdqUzMAfgBrTDnp5rhJhSH4YbyyZXmnar8hk fetchai/connections/p2p_stub,QmaHtQs9dJRnF27WDZSVW3FFEGbY1419NH8u67B8hgnteV fetchai/connections/scaffold,QmcjctNNxnFZ6xU2aDs7WvzNHMvvv4PuhnS739yWxHmJKz From 153581a76659cb14f260c65f75b18e887639bb42 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Wed, 16 Dec 2020 12:10:06 +0000 Subject: [PATCH 007/204] Add PoR support to agent lookup operation --- .../connections/p2p_libp2p/connection.yaml | 14 +- .../p2p_libp2p/dht/dhtclient/dhtclient.go | 178 +++++++++-- .../p2p_libp2p/dht/dhtnode/message.pb.go | 281 ++++++++++++++---- .../p2p_libp2p/dht/dhtnode/message.proto | 13 +- .../p2p_libp2p/dht/dhtnode/utils.go | 13 +- .../p2p_libp2p/dht/dhtpeer/benchmarks_test.go | 2 +- .../p2p_libp2p/dht/dhtpeer/dhtpeer.go | 274 +++++++++++++---- .../p2p_libp2p/dht/dhtpeer/dhtpeer_test.go | 2 +- packages/hashes.csv | 2 +- 9 files changed, 624 insertions(+), 155 deletions(-) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index b5f693ffe5..31cabef787 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -17,17 +17,17 @@ fingerprint: aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK check_dependencies.py: QmY8EiWapBspyR3zKGEY3TnQayz7hF7DxZ2d9zozVfFhtA connection.py: QmVf4XX25S8FQHcZca6o1xC77Ez9dUNp66K3dGnVB724GC - dht/dhtclient/dhtclient.go: QmQSCmviTBg9GRAHkRWsyt3uppTfLYseHie6KVLSSeVd2L + dht/dhtclient/dhtclient.go: QmajtiksLNCp5YthvVdD9QvuLyudYecpv13sioxbnXdmjZ dht/dhtclient/dhtclient_test.go: QmYVnf5j8Jyahi5jZNswkGR2tdnM4D1r5Fjdrz654EcD1F dht/dhtclient/options.go: QmWs7fQqHRavjwe1PjihHFi85HeJu9qSEVyMdcRviZzaT2 dht/dhtnode/dhtnode.go: QmbyhgbCSAbQ1QsDw7FM7Nt5sZcvhbupA1jv5faxutbV7N - dht/dhtnode/message.pb.go: QmZDrnCfXTLWRqmenTHVKwiuywypuZz5VMXvGjaWqtR6np - dht/dhtnode/message.proto: QmaMTai7UMwAfoLHYib5FBuqXa8XWNht6uhHGZ1bbLtzTo + dht/dhtnode/message.pb.go: QmfVCzkS2p2DZbtt8gfBqW3G2oXdPShHGkhME9zbVNfW88 + dht/dhtnode/message.proto: QmVZrdF8RzhoCKcNdNSzHh8vDwDjA9A7adonXFHrDY1Vpg dht/dhtnode/streams.go: Qmc2JcyiU4wHsgDj6aUunMAp4c5yMzo2ixeqRZHSW5PVwo - dht/dhtnode/utils.go: QmZi91a8RWrUUM6LztpMWBhLKj47W7UwMoTvooptF6oNHU - dht/dhtpeer/benchmarks_test.go: QmUj4zGAG8QgDVFf9PeP5kKdjUbqBi5nzdYmhrUfX2eZBD - dht/dhtpeer/dhtpeer.go: QmTYaRrrRphDfK3JUa52qFFZ1FgP7LSEnen4628me492RX - dht/dhtpeer/dhtpeer_test.go: QmbAKGzUvotDNrjk4QWAUUD4vhgTmeeTGrFjo5rJBwjfkw + dht/dhtnode/utils.go: Qmd5f5JCA41kutBE4953JA4XzRuVofYJJ8g4dTsJETtXcd + dht/dhtpeer/benchmarks_test.go: QmQcy8WN5DY8Nfp5nkDgRaS6NjQbMzc6DSMbdY8QDPx9ao + dht/dhtpeer/dhtpeer.go: QmVkVBXrPhWTZ4wEU2jhjpUwsYyJYhbA25CuCpYSY69x6U + dht/dhtpeer/dhtpeer_test.go: Qmc3FqVoq8wx8vJ7Q9NEpRbCVCvpAjiwTNwjWMeE91HRGd dht/dhtpeer/options.go: Qme5L9bRBQZaR5fciPJ1feFwqaQpkdYfW8TaiY1TDXs91H dht/dhttests/dhttests.go: QmQJxw2rJCGUFTXKxH3sf3Zaqta7H76EvaB7mWNNfikFkA dht/monitoring/file.go: QmZB8pNf7infMXWaC5A46rr3mghNfkuSd8UKH4Ceg8nDSM diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go index 90f459dd46..8f5eaba9d5 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go @@ -25,12 +25,13 @@ package dhtclient import ( "context" - "errors" "log" "math/rand" "strings" "time" + "github.com/pkg/errors" + "github.com/rs/zerolog" "google.golang.org/protobuf/proto" @@ -115,7 +116,7 @@ func New(opts ...Option) (*DHTClient, error) { // check if the PoR is delivered for my public key myPublicKey, err := utils.FetchAIPublicKeyFromPubKey(dhtClient.publicKey) - status, errPoR := dhtnode.IsValidProofOfRepresentation(dhtClient.myAgentRecord, myPublicKey) + status, errPoR := dhtnode.IsValidProofOfRepresentation(dhtClient.myAgentRecord, dhtClient.myAgentRecord.Address, myPublicKey) if status.Code != dhtnode.Status_SUCCESS { msg := "Invalid AgentRecord" if err != nil { @@ -352,10 +353,11 @@ func (dhtClient *DHTClient) RouteEnvelope(envel *aea.Envelope) error { } } - ldebug(). - Str("op", "route"). - Str("target", target). - Msg("looking up peer ID for agent Address") + //ldebug(). + // Str("op", "route"). + // Str("target", target). + // Msg("looking up peer ID for agent Address") + // client can get addresses only through bootstrap peer stream, err := dhtClient.newStreamLoopUntilTimeout(dhtClient.relayPeer, dhtnode.AeaAddressStream, newStreamTimeoutRelayPeer) if err != nil { @@ -365,34 +367,99 @@ func (dhtClient *DHTClient) RouteEnvelope(envel *aea.Envelope) error { ldebug(). Str("op", "route"). Str("target", target). - Msg("requesting peer ID from relay...") + Msg("requesting agent record from relay...") - err = utils.WriteBytes(stream, []byte(target)) + lookupRequest := &dhtnode.LookupRequest{AgentAddress: target} + msg := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_LookupRequest{LookupRequest: lookupRequest}, + } + buf, err := proto.Marshal(msg) if err != nil { lerror(err). Str("op", "route"). Str("target", target). - Msg("while sending address to relay") - return errors.New("ERROR route - While sending address to relay:" + err.Error()) + Msg("while serializing LookupRequest message") + errReset := stream.Reset() + ignore(errReset) + return err + } + err = utils.WriteBytes(stream, buf) + if err != nil { + lerror(err). + Str("op", "route"). + Str("target", target). + Msg("while sending LookupRequest message") + errReset := stream.Reset() + ignore(errReset) + return err } - msg, err := utils.ReadString(stream) + buf, err = utils.ReadBytes(stream) if err != nil { lerror(err). Str("op", "route"). Str("target", target). - Msgf("while reading target peer id from relay") - return errors.New("ERROR route - While reading target peer id from relay:" + err.Error()) + Msg("while receiving response acn message") + errReset := stream.Reset() + ignore(errReset) + return err + } + + response := &dhtnode.AcnMessage{} + err = proto.Unmarshal(buf, response) + if err != nil { + lerror(err).Str("op", "route").Str("target", target). + Msgf("couldn't deserialize agent lookup response") + return err } + + // response is either a LookupResponse or Status + var lookupResponse *dhtnode.LookupResponse = nil + var status *dhtnode.Status = nil + switch pl := response.Payload.(type) { + case *dhtnode.AcnMessage_LookupResponse: + lookupResponse = pl.LookupResponse + case *dhtnode.AcnMessage_Status: + status = pl.Status + default: + err = errors.New("Unexpected Acn Message") + lerror(err).Str("op", "route").Str("target", target). + Msgf("couldn't deserialize agent lookup response") + return err + } + + if status != nil { + err = errors.New(status.Code.String() + " : " + strings.Join(status.Msgs, ",")) + lerror(err).Str("op", "route").Str("target", target). + Msgf("failed agent lookup") + return err + } + + // lookupResponse must be set + record := lookupResponse.AgentRecord + valid, err := dhtnode.IsValidProofOfRepresentation(record, target, record.PeerPublicKey) + if err != nil || valid.Code != dhtnode.Status_SUCCESS { + errMsg := status.Code.String() + " : " + strings.Join(status.Msgs, ",") + if err == nil { + err = errors.New(errMsg) + } else { + err = errors.Wrap(err, status.Code.String()+" : "+strings.Join(status.Msgs, ",")) + } + lerror(err).Str("op", "route").Str("target", target). + Msgf("invalid agent record") + return err + } + stream.Close() - peerID, err := peer.Decode(msg) + peerID, err := utils.IDFromFetchAIPublicKey(record.PeerPublicKey) if err != nil { lerror(err). Str("op", "route"). Str("target", target). Msgf("CRITICAL couldn't get peer ID from message %s", msg) - return errors.New("CRITICAL route - couldn't get peer ID from message:" + err.Error()) + return errors.New("CRITICAL route - couldn't get peer ID from record peerID:" + err.Error()) } ldebug(). @@ -432,6 +499,7 @@ func (dhtClient *DHTClient) RouteEnvelope(envel *aea.Envelope) error { Str("op", "route"). Str("target", target). Msgf("opening stream to target %s", peerID) + // FIXME-ENVELOPE stream, err = dhtClient.newStreamLoopUntilTimeout(peerID, dhtnode.AeaEnvelopeStream, newStreamTimeout) if err != nil { return err @@ -455,7 +523,7 @@ func (dhtClient *DHTClient) RouteEnvelope(envel *aea.Envelope) error { func (dhtClient *DHTClient) handleAeaEnvelopeStream(stream network.Stream) { lerror, lwarn, _, ldebug := dhtClient.getLoggers() - ldebug().Msgf("Got a new aea envelope stream") + //ldebug().Msgf("Got a new aea envelope stream") envel, err := utils.ReadEnvelope(stream) if err != nil { @@ -481,19 +549,53 @@ func (dhtClient *DHTClient) handleAeaEnvelopeStream(stream network.Stream) { func (dhtClient *DHTClient) handleAeaAddressStream(stream network.Stream) { lerror, _, _, ldebug := dhtClient.getLoggers() - ldebug().Msg("Got a new aea address stream") + //ldebug().Msg("Got a new aea address stream") - reqAddress, err := utils.ReadString(stream) + // get LookupRequest + buf, err := utils.ReadBytes(stream) if err != nil { - lerror(err). - Str("op", "resolve"). - Str("target", reqAddress). - Msg("while reading Address from stream") + lerror(err).Str("op", "resolve"). + Msg("while reading message from stream") + err = stream.Reset() + ignore(err) + return + } + + msg := &dhtnode.AcnMessage{} + err = proto.Unmarshal(buf, msg) + if err != nil { + lerror(err).Str("op", "resolve").Msg("couldn't deserialize acn registration message") + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_GENERIC, Msgs: []string{err.Error()}} + response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Reset() + ignore(err) + return + } + + // Get LookupRequest message + var lookupRequest *dhtnode.LookupRequest + switch pl := msg.Payload.(type) { + case *dhtnode.AcnMessage_LookupRequest: + lookupRequest = pl.LookupRequest + default: + err = errors.New("Unexpected payload") + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNEXPECTED_PAYLOAD, Msgs: []string{err.Error()}} + response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) err = stream.Reset() ignore(err) return } + reqAddress := lookupRequest.AgentAddress + ldebug(). Str("op", "resolve"). Str("target", reqAddress). @@ -503,14 +605,32 @@ func (dhtClient *DHTClient) handleAeaAddressStream(stream network.Stream) { Str("op", "resolve"). Str("target", reqAddress). Msgf("requested address different from advertised one %s", dhtClient.myAgentAddress) - stream.Close() + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNKNOWN_AGENT_ADDRESS} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_Status{Status: status}, + } + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) // TODO(LR) stream.Reset() if err + err = stream.Close() + ignore(err) + return } else { - err = utils.WriteBytes(stream, []byte(dhtClient.routedHost.ID().Pretty())) + lookupResponse := &dhtnode.LookupResponse{AgentRecord: dhtClient.myAgentRecord} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_LookupResponse{LookupResponse: lookupResponse}, + } + buf, err := proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) if err != nil { - lerror(err). - Str("op", "resolve"). - Str("target", reqAddress). - Msg("While sending peerID to peer") + lerror(err).Str("op", "resolve").Str("addr", reqAddress). + Msg("while sending agent record to peer") + err = stream.Reset() + ignore(err) } } @@ -541,7 +661,7 @@ func (dhtClient *DHTClient) registerAgentAddress() error { Msgf("registering addr and peerID to relay peer") registration := &dhtnode.Register{Record: dhtClient.myAgentRecord} - msg := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Register{Register: registration}} + msg := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Register{Register: registration}} buf, err := proto.Marshal(msg) ignore(err) diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go index 5b6e5f4ada..e7cfff0fd4 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go @@ -36,7 +36,9 @@ const ( // register (1x) Status_ERROR_INVALID_AGENT_ADDRESS Status_ErrCode = 10 Status_ERROR_INVALID_PUBLIC_KEY Status_ErrCode = 11 - Status_ERROR_INVALID_PROOF Status_ErrCode = 12 // + Status_ERROR_INVALID_PROOF Status_ErrCode = 12 + // lookup (2x) + Status_ERROR_UNKNOWN_AGENT_ADDRESS Status_ErrCode = 20 ) // Enum value maps for Status_ErrCode. @@ -49,6 +51,7 @@ var ( 10: "ERROR_INVALID_AGENT_ADDRESS", 11: "ERROR_INVALID_PUBLIC_KEY", 12: "ERROR_INVALID_PROOF", + 20: "ERROR_UNKNOWN_AGENT_ADDRESS", } Status_ErrCode_value = map[string]int32{ "SUCCESS": 0, @@ -58,6 +61,7 @@ var ( "ERROR_INVALID_AGENT_ADDRESS": 10, "ERROR_INVALID_PUBLIC_KEY": 11, "ERROR_INVALID_PROOF": 12, + "ERROR_UNKNOWN_AGENT_ADDRESS": 20, } ) @@ -85,7 +89,7 @@ func (x Status_ErrCode) Number() protoreflect.EnumNumber { // Deprecated: Use Status_ErrCode.Descriptor instead. func (Status_ErrCode) EnumDescriptor() ([]byte, []int) { - return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{2, 0} + return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{4, 0} } type AgentRecord struct { @@ -214,6 +218,100 @@ func (x *Register) GetRecord() *AgentRecord { return nil } +type LookupRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + AgentAddress string `protobuf:"bytes,1,opt,name=agent_address,json=agentAddress,proto3" json:"agent_address,omitempty"` +} + +func (x *LookupRequest) Reset() { + *x = LookupRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_dht_dhtnode_message_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LookupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupRequest) ProtoMessage() {} + +func (x *LookupRequest) ProtoReflect() protoreflect.Message { + mi := &file_dht_dhtnode_message_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LookupRequest.ProtoReflect.Descriptor instead. +func (*LookupRequest) Descriptor() ([]byte, []int) { + return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{2} +} + +func (x *LookupRequest) GetAgentAddress() string { + if x != nil { + return x.AgentAddress + } + return "" +} + +type LookupResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + AgentRecord *AgentRecord `protobuf:"bytes,1,opt,name=agent_record,json=agentRecord,proto3" json:"agent_record,omitempty"` +} + +func (x *LookupResponse) Reset() { + *x = LookupResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_dht_dhtnode_message_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LookupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupResponse) ProtoMessage() {} + +func (x *LookupResponse) ProtoReflect() protoreflect.Message { + mi := &file_dht_dhtnode_message_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LookupResponse.ProtoReflect.Descriptor instead. +func (*LookupResponse) Descriptor() ([]byte, []int) { + return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{3} +} + +func (x *LookupResponse) GetAgentRecord() *AgentRecord { + if x != nil { + return x.AgentRecord + } + return nil +} + type Status struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -226,7 +324,7 @@ type Status struct { func (x *Status) Reset() { *x = Status{} if protoimpl.UnsafeEnabled { - mi := &file_dht_dhtnode_message_proto_msgTypes[2] + mi := &file_dht_dhtnode_message_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -239,7 +337,7 @@ func (x *Status) String() string { func (*Status) ProtoMessage() {} func (x *Status) ProtoReflect() protoreflect.Message { - mi := &file_dht_dhtnode_message_proto_msgTypes[2] + mi := &file_dht_dhtnode_message_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -252,7 +350,7 @@ func (x *Status) ProtoReflect() protoreflect.Message { // Deprecated: Use Status.ProtoReflect.Descriptor instead. func (*Status) Descriptor() ([]byte, []int) { - return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{2} + return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{4} } func (x *Status) GetCode() Status_ErrCode { @@ -278,13 +376,15 @@ type AcnMessage struct { // Types that are assignable to Payload: // *AcnMessage_Status // *AcnMessage_Register + // *AcnMessage_LookupRequest + // *AcnMessage_LookupResponse Payload isAcnMessage_Payload `protobuf_oneof:"payload"` } func (x *AcnMessage) Reset() { *x = AcnMessage{} if protoimpl.UnsafeEnabled { - mi := &file_dht_dhtnode_message_proto_msgTypes[3] + mi := &file_dht_dhtnode_message_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -297,7 +397,7 @@ func (x *AcnMessage) String() string { func (*AcnMessage) ProtoMessage() {} func (x *AcnMessage) ProtoReflect() protoreflect.Message { - mi := &file_dht_dhtnode_message_proto_msgTypes[3] + mi := &file_dht_dhtnode_message_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -310,7 +410,7 @@ func (x *AcnMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use AcnMessage.ProtoReflect.Descriptor instead. func (*AcnMessage) Descriptor() ([]byte, []int) { - return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{3} + return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{5} } func (x *AcnMessage) GetVersion() string { @@ -341,6 +441,20 @@ func (x *AcnMessage) GetRegister() *Register { return nil } +func (x *AcnMessage) GetLookupRequest() *LookupRequest { + if x, ok := x.GetPayload().(*AcnMessage_LookupRequest); ok { + return x.LookupRequest + } + return nil +} + +func (x *AcnMessage) GetLookupResponse() *LookupResponse { + if x, ok := x.GetPayload().(*AcnMessage_LookupResponse); ok { + return x.LookupResponse + } + return nil +} + type isAcnMessage_Payload interface { isAcnMessage_Payload() } @@ -353,10 +467,22 @@ type AcnMessage_Register struct { Register *Register `protobuf:"bytes,3,opt,name=register,proto3,oneof"` } +type AcnMessage_LookupRequest struct { + LookupRequest *LookupRequest `protobuf:"bytes,4,opt,name=lookup_request,json=lookupRequest,proto3,oneof"` +} + +type AcnMessage_LookupResponse struct { + LookupResponse *LookupResponse `protobuf:"bytes,5,opt,name=lookup_response,json=lookupResponse,proto3,oneof"` +} + func (*AcnMessage_Status) isAcnMessage_Payload() {} func (*AcnMessage_Register) isAcnMessage_Payload() {} +func (*AcnMessage_LookupRequest) isAcnMessage_Payload() {} + +func (*AcnMessage_LookupResponse) isAcnMessage_Payload() {} + var File_dht_dhtnode_message_proto protoreflect.FileDescriptor var file_dht_dhtnode_message_proto_rawDesc = []byte{ @@ -376,34 +502,52 @@ var file_dht_dhtnode_message_proto_rawDesc = []byte{ 0x72, 0x65, 0x22, 0x38, 0x0a, 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x2c, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, - 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0x8a, 0x02, 0x0a, - 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x45, 0x72, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, - 0x63, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x07, 0x45, 0x72, 0x72, - 0x43, 0x6f, 0x64, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, - 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x53, 0x55, 0x50, - 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x10, 0x01, - 0x12, 0x1c, 0x0a, 0x18, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x45, 0x58, 0x50, 0x45, - 0x43, 0x54, 0x45, 0x44, 0x5f, 0x50, 0x41, 0x59, 0x4c, 0x4f, 0x41, 0x44, 0x10, 0x02, 0x12, 0x11, - 0x0a, 0x0d, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x10, - 0x03, 0x12, 0x1f, 0x0a, 0x1b, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, - 0x49, 0x44, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, - 0x10, 0x0a, 0x12, 0x1c, 0x0a, 0x18, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, - 0x4c, 0x49, 0x44, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x4b, 0x45, 0x59, 0x10, 0x0b, - 0x12, 0x17, 0x0a, 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, - 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x4f, 0x46, 0x10, 0x0c, 0x22, 0x8d, 0x01, 0x0a, 0x0a, 0x41, 0x63, - 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x48, 0x00, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a, - 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x11, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, - 0x65, 0x72, 0x48, 0x00, 0x52, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x42, 0x09, - 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0x34, 0x0a, 0x0d, + 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, + 0x0d, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, + 0x73, 0x73, 0x22, 0x49, 0x0a, 0x0e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, + 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x68, 0x74, + 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x52, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0xab, 0x02, + 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, + 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x45, 0x72, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x52, + 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x22, 0xdf, 0x01, 0x0a, 0x07, 0x45, 0x72, + 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, + 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x53, 0x55, + 0x50, 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x10, + 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x45, 0x58, 0x50, + 0x45, 0x43, 0x54, 0x45, 0x44, 0x5f, 0x50, 0x41, 0x59, 0x4c, 0x4f, 0x41, 0x44, 0x10, 0x02, 0x12, + 0x11, 0x0a, 0x0d, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, + 0x10, 0x03, 0x12, 0x1f, 0x0a, 0x1b, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, + 0x4c, 0x49, 0x44, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, + 0x53, 0x10, 0x0a, 0x12, 0x1c, 0x0a, 0x18, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, + 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x4b, 0x45, 0x59, 0x10, + 0x0b, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, + 0x49, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x4f, 0x46, 0x10, 0x0c, 0x12, 0x1f, 0x0a, 0x1b, 0x45, 0x52, + 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x41, 0x47, 0x45, 0x4e, + 0x54, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x14, 0x22, 0x92, 0x02, 0x0a, 0x0a, + 0x41, 0x63, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x48, 0x00, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, + 0x2f, 0x0a, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x52, 0x65, 0x67, 0x69, + 0x73, 0x74, 0x65, 0x72, 0x48, 0x00, 0x52, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, + 0x12, 0x3f, 0x0a, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, + 0x64, 0x65, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x48, 0x00, 0x52, 0x0d, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x42, 0x0a, 0x0f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x72, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x68, 0x74, + 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -419,24 +563,29 @@ func file_dht_dhtnode_message_proto_rawDescGZIP() []byte { } var file_dht_dhtnode_message_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_dht_dhtnode_message_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_dht_dhtnode_message_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_dht_dhtnode_message_proto_goTypes = []interface{}{ - (Status_ErrCode)(0), // 0: dhtnode.Status.ErrCode - (*AgentRecord)(nil), // 1: dhtnode.AgentRecord - (*Register)(nil), // 2: dhtnode.Register - (*Status)(nil), // 3: dhtnode.Status - (*AcnMessage)(nil), // 4: dhtnode.AcnMessage + (Status_ErrCode)(0), // 0: dhtnode.Status.ErrCode + (*AgentRecord)(nil), // 1: dhtnode.AgentRecord + (*Register)(nil), // 2: dhtnode.Register + (*LookupRequest)(nil), // 3: dhtnode.LookupRequest + (*LookupResponse)(nil), // 4: dhtnode.LookupResponse + (*Status)(nil), // 5: dhtnode.Status + (*AcnMessage)(nil), // 6: dhtnode.AcnMessage } var file_dht_dhtnode_message_proto_depIdxs = []int32{ 1, // 0: dhtnode.Register.record:type_name -> dhtnode.AgentRecord - 0, // 1: dhtnode.Status.code:type_name -> dhtnode.Status.ErrCode - 3, // 2: dhtnode.AcnMessage.status:type_name -> dhtnode.Status - 2, // 3: dhtnode.AcnMessage.register:type_name -> dhtnode.Register - 4, // [4:4] is the sub-list for method output_type - 4, // [4:4] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 1, // 1: dhtnode.LookupResponse.agent_record:type_name -> dhtnode.AgentRecord + 0, // 2: dhtnode.Status.code:type_name -> dhtnode.Status.ErrCode + 5, // 3: dhtnode.AcnMessage.status:type_name -> dhtnode.Status + 2, // 4: dhtnode.AcnMessage.register:type_name -> dhtnode.Register + 3, // 5: dhtnode.AcnMessage.lookup_request:type_name -> dhtnode.LookupRequest + 4, // 6: dhtnode.AcnMessage.lookup_response:type_name -> dhtnode.LookupResponse + 7, // [7:7] is the sub-list for method output_type + 7, // [7:7] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name } func init() { file_dht_dhtnode_message_proto_init() } @@ -470,7 +619,7 @@ func file_dht_dhtnode_message_proto_init() { } } file_dht_dhtnode_message_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Status); i { + switch v := v.(*LookupRequest); i { case 0: return &v.state case 1: @@ -482,6 +631,30 @@ func file_dht_dhtnode_message_proto_init() { } } file_dht_dhtnode_message_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LookupResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_dht_dhtnode_message_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Status); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_dht_dhtnode_message_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*AcnMessage); i { case 0: return &v.state @@ -494,9 +667,11 @@ func file_dht_dhtnode_message_proto_init() { } } } - file_dht_dhtnode_message_proto_msgTypes[3].OneofWrappers = []interface{}{ + file_dht_dhtnode_message_proto_msgTypes[5].OneofWrappers = []interface{}{ (*AcnMessage_Status)(nil), (*AcnMessage_Register)(nil), + (*AcnMessage_LookupRequest)(nil), + (*AcnMessage_LookupResponse)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -504,7 +679,7 @@ func file_dht_dhtnode_message_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_dht_dhtnode_message_proto_rawDesc, NumEnums: 1, - NumMessages: 4, + NumMessages: 6, NumExtensions: 0, NumServices: 0, }, diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto index ff1c7aad1d..3a409b7c5e 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto @@ -14,6 +14,14 @@ message Register { AgentRecord record = 1; } +message LookupRequest { + string agent_address = 1; +} + +message LookupResponse { + AgentRecord agent_record = 1; +} + message Status { enum ErrCode { // common (0x) @@ -25,7 +33,8 @@ message Status { ERROR_INVALID_AGENT_ADDRESS = 10; ERROR_INVALID_PUBLIC_KEY = 11; ERROR_INVALID_PROOF = 12; - // + // lookup (2x) + ERROR_UNKNOWN_AGENT_ADDRESS = 20; } ErrCode code = 1; @@ -37,5 +46,7 @@ message AcnMessage { oneof payload { Status status = 2; Register register = 3; + LookupRequest lookup_request = 4; + LookupResponse lookup_response = 5; } } \ No newline at end of file diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/utils.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/utils.go index 621992703b..44ce2680dc 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/utils.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/utils.go @@ -26,7 +26,18 @@ import ( utils "libp2p_node/utils" ) -func IsValidProofOfRepresentation(record *AgentRecord, representativePeerPubKey string) (*Status, error) { +const ( + CurrentVersion = "0.1.0" +) + +func IsValidProofOfRepresentation(record *AgentRecord, agentAddress string, representativePeerPubKey string) (*Status, error) { + // check agent address matches + if record.Address != agentAddress { + err := errors.New("Wrong agent address, expected " + agentAddress) + response := &Status{Code: Status_ERROR_INVALID_AGENT_ADDRESS, Msgs: []string{err.Error()}} + return response, err + } + // check public key matches if record.PeerPublicKey != representativePeerPubKey { err := errors.New("Wrong peer public key, expected " + representativePeerPubKey) diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/benchmarks_test.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/benchmarks_test.go index f63bde1556..026d19fdf1 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/benchmarks_test.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/benchmarks_test.go @@ -321,7 +321,7 @@ func benchmarkAgentLookup(npeers uint16, b *testing.B) { for i := 0; i < b.N; i++ { b.ResetTimer() - _, err = peer.lookupAddressDHT(addrs[len(peers)-1-i%len(peers)]) + _, _, err = peer.lookupAddressDHT(addrs[len(peers)-1-i%len(peers)]) if err != nil { b.Fail() } diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go index 500770b162..265fc25eeb 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go @@ -25,7 +25,6 @@ package dhtpeer import ( "context" - "errors" "fmt" "io" "log" @@ -35,6 +34,8 @@ import ( "sync" "time" + "github.com/pkg/errors" + "github.com/rs/zerolog" "google.golang.org/protobuf/proto" @@ -177,7 +178,9 @@ func New(opts ...Option) (*DHTPeer, error) { // check if the PoR is delivered for my public key if dhtPeer.myAgentRecord != nil { myPublicKey, err := utils.FetchAIPublicKeyFromPubKey(dhtPeer.publicKey) - status, errPoR := dhtnode.IsValidProofOfRepresentation(dhtPeer.myAgentRecord, myPublicKey) + status, errPoR := dhtnode.IsValidProofOfRepresentation( + dhtPeer.myAgentRecord, dhtPeer.myAgentRecord.Address, myPublicKey, + ) if status.Code != dhtnode.Status_SUCCESS { return nil, errors.New("Invalid AgentRecord - " + err.Error() + " - " + errPoR.Error()) } @@ -526,7 +529,7 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { if err != nil { lerror(err).Msg("couldn't deserialize acn registration message") status := &dhtnode.Status{Code: dhtnode.Status_ERROR_GENERIC, Msgs: []string{err.Error()}} - response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} + response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) ignore(err) err = utils.WriteBytesConn(conn, buf) @@ -546,7 +549,7 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { default: err = errors.New("Unexpected payload") status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNEXPECTED_PAYLOAD, Msgs: []string{err.Error()}} - response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} + response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) ignore(err) err = utils.WriteBytesConn(conn, buf) @@ -564,10 +567,10 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { // check if the PoR is valid myPubKey, err := utils.FetchAIPublicKeyFromPubKey(dhtPeer.publicKey) ignore(err) - status, err := dhtnode.IsValidProofOfRepresentation(record, myPubKey) + status, err := dhtnode.IsValidProofOfRepresentation(record, addr, myPubKey) if err != nil { lerror(err).Msg("PoR is not valid") - response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} + response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) ignore(err) err = utils.WriteBytesConn(conn, buf) @@ -577,7 +580,7 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { return } - msg = &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} + msg = &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(msg) ignore(err) err = utils.WriteBytesConn(conn, buf) @@ -669,6 +672,24 @@ func (dhtPeer *DHTPeer) RouteEnvelope(envel *aea.Envelope) error { routeCountAll.Inc() start := timer.NewTimer() + // get sender agent record + // TOFIX can change function signature to force the caller to provide the record + /* + var record *dhtnode.AgentRecord + sender := envel.Sender + if sender == dhtPeer.myAgentAddress { + record = dhtPeer.myAgentRecord + } else if val, exists := dhtPeer.agentRecords[sender]; exists { + // TOFIX(LR) should acquire RLock + record = val + } else { + err := errors.New("Envelope sender is not registered locally " + sender) + lerror(err).Str("op", "route").Str("addr", envel.To). + Msg("") + return err + } + */ + target := envel.To if target == dhtPeer.myAgentAddress { @@ -719,7 +740,7 @@ func (dhtPeer *DHTPeer) RouteEnvelope(envel *aea.Envelope) error { } else { linfo().Str("op", "route").Str("addr", target). Msg("did NOT find destination address locally, looking for it in the DHT...") - peerID, err = dhtPeer.lookupAddressDHT(target) + peerID, _, err = dhtPeer.lookupAddressDHT(target) // FIXME-ENVELOPE if err != nil { lerror(err).Str("op", "route").Str("addr", target). Msg("while looking up address on the DHT") @@ -764,7 +785,8 @@ func (dhtPeer *DHTPeer) RouteEnvelope(envel *aea.Envelope) error { return nil } -func (dhtPeer *DHTPeer) lookupAddressDHT(address string) (peer.ID, error) { +/// TOFIX(LR) should return (*dhtnode) +func (dhtPeer *DHTPeer) lookupAddressDHT(address string) (peer.ID, *dhtnode.AgentRecord, error) { lerror, lwarn, linfo, _ := dhtPeer.getLoggers() var err error @@ -773,7 +795,7 @@ func (dhtPeer *DHTPeer) lookupAddressDHT(address string) (peer.ID, error) { addressCID, err := utils.ComputeCID(address) if err != nil { - return "", err + return "", nil, err } linfo().Str("op", "lookup").Str("addr", address). @@ -782,13 +804,12 @@ func (dhtPeer *DHTPeer) lookupAddressDHT(address string) (peer.ID, error) { defer cancel() //var elapsed time.Duration var provider peer.AddrInfo - var connected bool = false var stream network.Stream start := timer.NewTimer() noProvider := false - for !connected { + for { providers := dhtPeer.dht.FindProvidersAsync(ctx, addressCID, 0) for provider = range providers { @@ -805,13 +826,93 @@ func (dhtPeer *DHTPeer) lookupAddressDHT(address string) (peer.ID, error) { // Msgf("opening stream to the address provider %s...", provider) ctxConnect := context.Background() stream, err = dhtPeer.routedHost.NewStream(ctxConnect, provider.ID, dhtnode.AeaAddressStream) - if err == nil { - connected = true - break + if err != nil { + lwarn().Str("op", "lookup").Str("addr", address). + Msgf("couldn't open stream to address provider %s: %s, looking up other providers...", provider, err.Error()) + dhtPeer.routedHost.Peerstore().ClearAddrs(provider.ID) + continue + } + + linfo().Str("op", "lookup").Str("addr", address). + Msgf("getting agent record from provider %s...", provider) + + // prepare LookupRequest + lookupRequest := &dhtnode.LookupRequest{AgentAddress: address} + msg := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_LookupRequest{LookupRequest: lookupRequest}} + buf, err := proto.Marshal(msg) + ignore(err) + + err = utils.WriteBytes(stream, []byte(buf)) + if err != nil { + lwarn().Str("op", "lookup").Str("addr", address). + Msgf("couldn't send agent lookup request to provider %s (%s), looking up other providers...", provider, err.Error()) + err = stream.Reset() + ignore(err) + continue + } + + buf, err = utils.ReadBytes(stream) + if err != nil { + lwarn().Str("op", "lookup").Str("addr", address). + Msgf("couldn't receive agent lookup response from provider %s (%s), looking up other providers...", provider, err.Error()) + err = stream.Reset() + ignore(err) + continue + } + + stream.Close() + + response := &dhtnode.AcnMessage{} + err = proto.Unmarshal(buf, response) + if err != nil { + lwarn().Str("op", "lookup").Str("addr", address). + Msgf("couldn't deserialize agent lookup response from provider %s (%s), looking up other providers...", provider, err.Error()) + continue + } + + // Response is either a LookupResponse or Status + var lookupResponse *dhtnode.LookupResponse = nil + var status *dhtnode.Status = nil + switch pl := response.Payload.(type) { + case *dhtnode.AcnMessage_LookupResponse: + lookupResponse = pl.LookupResponse + case *dhtnode.AcnMessage_Status: + status = pl.Status + default: + err = errors.New("Unexpected Acn Message") + lwarn().Str("op", "lookup").Str("addr", address). + Msgf("couldn't deserialize agent lookup response from provider %s (%s), looking up other providers...", provider, err.Error()) + continue + } + + if status != nil { + err = errors.New(status.Code.String() + " : " + strings.Join(status.Msgs, ",")) + lwarn().Str("op", "lookup").Str("addr", address). + Msgf("Failed agent lookup response from provider %s (%s), looking up other providers...", provider, err.Error()) + continue } - dhtPeer.routedHost.Peerstore().ClearAddrs(provider.ID) - lwarn().Str("op", "lookup").Str("addr", address).Msgf("couldn't open stream to address provider %s: %s, looking up for other providers...", provider, err.Error()) + // lookupResponse must be set + record := lookupResponse.AgentRecord + valid, err := dhtnode.IsValidProofOfRepresentation(record, address, record.PeerPublicKey) + if err != nil || valid.Code != dhtnode.Status_SUCCESS { + errMsg := status.Code.String() + " : " + strings.Join(status.Msgs, ",") + if err == nil { + err = errors.New(errMsg) + } else { + err = errors.Wrap(err, status.Code.String()+" : "+strings.Join(status.Msgs, ",")) + } + lwarn().Str("op", "lookup").Str("addr", address). + Msgf("invalid agent record from provider %s (%s), looking up other providers...", provider, err.Error()) + continue + } + + peerid, err := utils.IDFromFetchAIPublicKey(record.PeerPublicKey) + if err != nil { + return "", nil, errors.New("CRITICAL couldn't get peer ID from message:" + err.Error()) + } + + return peerid, record, nil } if provider.ID == "" { @@ -826,33 +927,12 @@ func (dhtPeer *DHTPeer) lookupAddressDHT(address string) (peer.ID, error) { case <-ctx.Done(): err = errors.New(msg + " " + address + " within timeout") lerror(err).Str("op", "lookup").Str("addr", address).Msg("") - return "", err + return "", nil, err } - } else if !connected { - return "", err + } else { + return "", nil, err } } - - linfo().Str("op", "lookup").Str("addr", address). - Msgf("reading peer ID from provider %s...", provider) - - err = utils.WriteBytes(stream, []byte(address)) - if err != nil { - return "", errors.New("ERROR while sending address to peer:" + err.Error()) - } - - msg, err := utils.ReadString(stream) - if err != nil { - return "", errors.New("ERROR while reading target peer id from peer:" + err.Error()) - } - stream.Close() - - peerid, err := peer.Decode(msg) - if err != nil { - return "", errors.New("CRITICAL couldn't get peer ID from message:" + err.Error()) - } - - return peerid, nil } func (dhtPeer *DHTPeer) handleAeaEnvelopeStream(stream network.Stream) { @@ -894,18 +974,56 @@ func (dhtPeer *DHTPeer) handleAeaAddressStream(stream network.Stream) { //linfo().Msgf("Got a new aea address stream") - reqAddress, err := utils.ReadString(stream) + // get LookupRequest + buf, err := utils.ReadBytes(stream) if err != nil { - lerror(err).Str("op", "resolve").Str("addr", reqAddress). - Msg("while reading Address from stream") + lerror(err).Str("op", "resolve"). + Msg("while reading message from stream") err = stream.Reset() ignore(err) return } + msg := &dhtnode.AcnMessage{} + err = proto.Unmarshal(buf, msg) + if err != nil { + lerror(err).Str("op", "resolve").Msg("couldn't deserialize acn registration message") + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_GENERIC, Msgs: []string{err.Error()}} + response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Reset() + ignore(err) + return + } + + // Get LookupRequest message + var lookupRequest *dhtnode.LookupRequest + switch pl := msg.Payload.(type) { + case *dhtnode.AcnMessage_LookupRequest: + lookupRequest = pl.LookupRequest + default: + err = errors.New("Unexpected payload") + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNEXPECTED_PAYLOAD, Msgs: []string{err.Error()}} + response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Reset() + ignore(err) + return + } + + reqAddress := lookupRequest.AgentAddress + //linfo().Str("op", "resolve").Str("addr", reqAddress). // Msg("Received query for addr") var sPeerID string + var sRecord *dhtnode.AgentRecord = nil + dhtPeer.dhtAddressesLock.RLock() idRelay, existsRelay := dhtPeer.dhtAddresses[reqAddress] dhtPeer.dhtAddressesLock.RUnlock() @@ -918,13 +1036,15 @@ func (dhtPeer *DHTPeer) handleAeaAddressStream(stream network.Stream) { if err != nil { lerror(err).Str("op", "resolve").Str("addr", reqAddress). Msgf("CRITICAL could not get peer ID from public key %s", dhtPeer.publicKey) - return + } else { + sPeerID = peerID.Pretty() + sRecord = dhtPeer.myAgentRecord } - sPeerID = peerID.Pretty() } else if existsRelay { linfo().Str("op", "resolve").Str("addr", reqAddress). Msg("found address in my relay clients map") sPeerID = idRelay + sRecord = dhtPeer.agentRecords[reqAddress] } else if existsDelegate { linfo().Str("op", "resolve").Str("addr", reqAddress). Msgf("found address in my delegate clients map") @@ -932,32 +1052,65 @@ func (dhtPeer *DHTPeer) handleAeaAddressStream(stream network.Stream) { if err != nil { lerror(err).Str("op", "resolve").Str("addr", reqAddress). Msgf("CRITICAL could not get peer ID from public key %s", dhtPeer.publicKey) - return + } else { + sPeerID = peerID.Pretty() + sRecord = dhtPeer.agentRecords[reqAddress] } - sPeerID = peerID.Pretty() } else { // needed when a relay client queries for a peer ID //linfo().Str("op", "resolve").Str("addr", reqAddress). // Msg("did NOT found the address locally, looking for it in the DHT...") - peerID, err := dhtPeer.lookupAddressDHT(reqAddress) + peerID, peerRecord, err := dhtPeer.lookupAddressDHT(reqAddress) if err == nil { linfo().Str("op", "resolve").Str("addr", reqAddress). Msg("found address on the DHT") sPeerID = peerID.Pretty() + sRecord = peerRecord } else { lerror(err).Str("op", "resolve").Str("addr", reqAddress). Msgf("did NOT find address locally or on the DHT.") + + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNKNOWN_AGENT_ADDRESS} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_Status{Status: status}, + } + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) // TODO(LR) stream.Reset() if err + err = stream.Close() + ignore(err) return } } - linfo().Str("op", "resolve").Str("addr", reqAddress). - Msgf("sending peer id %s", sPeerID) - err = utils.WriteBytes(stream, []byte(sPeerID)) - if err != nil { - lerror(err).Str("op", "resolve").Str("addr", reqAddress). - Msg("While sending peerID to peer") + if sRecord != nil { + linfo().Str("op", "resolve").Str("addr", reqAddress). + Msgf("sending agent record (%s) %s", sPeerID, sRecord) + + lookupResponse := &dhtnode.LookupResponse{AgentRecord: sRecord} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_LookupResponse{LookupResponse: lookupResponse}, + } + buf, err := proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + if err != nil { + lerror(err).Str("op", "resolve").Str("addr", reqAddress). + Msg("While sending agent record to peer") + err = stream.Reset() + ignore(err) + } } + + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_GENERIC, Msgs: []string{"Internal error: Couldn't get AgentRecord"}} + response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) } func (dhtPeer *DHTPeer) handleAeaNotifStream(stream network.Stream) { @@ -1050,14 +1203,13 @@ func (dhtPeer *DHTPeer) handleAeaRegisterStream(stream network.Stream) { ignore(err) return } - /////////// msg := &dhtnode.AcnMessage{} err = proto.Unmarshal(buf, msg) if err != nil { lerror(err).Msg("couldn't deserialize acn registration message") status := &dhtnode.Status{Code: dhtnode.Status_ERROR_GENERIC, Msgs: []string{err.Error()}} - response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} + response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) ignore(err) err = utils.WriteBytes(stream, buf) @@ -1077,7 +1229,7 @@ func (dhtPeer *DHTPeer) handleAeaRegisterStream(stream network.Stream) { default: err = errors.New("Unexpected payload") status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNEXPECTED_PAYLOAD, Msgs: []string{err.Error()}} - response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} + response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) ignore(err) err = utils.WriteBytes(stream, buf) @@ -1095,10 +1247,10 @@ func (dhtPeer *DHTPeer) handleAeaRegisterStream(stream network.Stream) { // check if the PoR is valid clientPubKey, err := utils.FetchAIPublicKeyFromPubKey(stream.Conn().RemotePublicKey()) ignore(err) - status, err := dhtnode.IsValidProofOfRepresentation(record, clientPubKey) + status, err := dhtnode.IsValidProofOfRepresentation(record, record.Address, clientPubKey) if err != nil { lerror(err).Msg("PoR is not valid") - response := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} + response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) ignore(err) err = utils.WriteBytes(stream, buf) @@ -1108,7 +1260,7 @@ func (dhtPeer *DHTPeer) handleAeaRegisterStream(stream network.Stream) { return } - msg = &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Status{Status: status}} + msg = &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(msg) ignore(err) err = utils.WriteBytes(stream, buf) diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go index 3ea6bd9659..a394d823cd 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go @@ -1544,7 +1544,7 @@ func SetupDelegateClient(key string, host string, port uint16, peerPubKey string record.PeerPublicKey = peerPubKey record.Signature = signature registration := &dhtnode.Register{Record: record} - msg := &dhtnode.AcnMessage{Version: "0.1.0", Payload: &dhtnode.AcnMessage_Register{Register: registration}} + msg := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Register{Register: registration}} data, err := proto.Marshal(msg) ignore(err) err = utils.WriteBytesConn(client.Conn, data) diff --git a/packages/hashes.csv b/packages/hashes.csv index f9fcd3ab76..8bcf828603 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,7 +34,7 @@ fetchai/connections/http_server,QmP3nhFMyEi7N64oLV5EQDNowhS8be3jVefxkwir4PKpQ9 fetchai/connections/ledger,QmaViv1ZK4jLgQXycpYLjboFFcznN9zXUZT4JKdbeS2wQ8 fetchai/connections/local,QmNPNVkqtDtzENpv1XqM4LHZna6v6jJ7Hsk6RB14iSdtjG fetchai/connections/oef,QmVGcKDeDMEhtcBnDNVTWchHkA2YhHnDGoK8izobnDQKmw -fetchai/connections/p2p_libp2p,QmYBHSwej3fTiPkf5prigYn9sA3J8YdxKkYCMFVUwGFbdZ +fetchai/connections/p2p_libp2p,QmWhtpMTzqEV8ezSc2qVoDpCSpGw5MUaQpVt2comhw99Wi fetchai/connections/p2p_libp2p_client,QmPV4eyGijvdqUzMAfgBrTDnp5rhJhSH4YbyyZXmnar8hk fetchai/connections/p2p_stub,QmaHtQs9dJRnF27WDZSVW3FFEGbY1419NH8u67B8hgnteV fetchai/connections/scaffold,QmcjctNNxnFZ6xU2aDs7WvzNHMvvv4PuhnS739yWxHmJKz From e17de2df8c7246a98616d88a65f8a7d3d79a637c Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Wed, 16 Dec 2020 14:27:21 +0000 Subject: [PATCH 008/204] Add PoR support to envelope routing --- .../fetchai/connections/p2p_libp2p/aea/api.go | 7 + .../connections/p2p_libp2p/aea/envelope.proto | 1 + .../connections/p2p_libp2p/connection.yaml | 18 +- .../p2p_libp2p/dht/dhtclient/dhtclient.go | 236 ++++++++++- .../dht/dhtclient/dhtclient_test.go | 8 +- .../p2p_libp2p/dht/dhtnode/message.pb.go | 228 ++++++++--- .../p2p_libp2p/dht/dhtnode/message.proto | 13 +- .../p2p_libp2p/dht/dhtpeer/dhtpeer.go | 380 +++++++++++++++--- .../p2p_libp2p/dht/dhtpeer/dhtpeer_test.go | 75 ++-- .../connections/p2p_libp2p/utils/utils.go | 1 - packages/hashes.csv | 2 +- 11 files changed, 777 insertions(+), 192 deletions(-) diff --git a/packages/fetchai/connections/p2p_libp2p/aea/api.go b/packages/fetchai/connections/p2p_libp2p/aea/api.go index e900b55465..57861d028f 100644 --- a/packages/fetchai/connections/p2p_libp2p/aea/api.go +++ b/packages/fetchai/connections/p2p_libp2p/aea/api.go @@ -365,6 +365,13 @@ func (aea *AeaApi) listen_for_envelopes() { } return } + if envel.Sender != aea.agent_record.Address { + logger.Error(). + Str("err", "Sender ("+envel.Sender+") must match registered address"). + Msg("while processing envelope") + // TODO send error back to agent + continue + } logger.Debug().Msgf("received envelope from agent") aea.out_queue <- envel if aea.closing { diff --git a/packages/fetchai/connections/p2p_libp2p/aea/envelope.proto b/packages/fetchai/connections/p2p_libp2p/aea/envelope.proto index 4af92e3e8e..0c7438e7a6 100644 --- a/packages/fetchai/connections/p2p_libp2p/aea/envelope.proto +++ b/packages/fetchai/connections/p2p_libp2p/aea/envelope.proto @@ -1,5 +1,6 @@ syntax = "proto3"; +//package libp2p_node.aea; package aea; message Envelope{ diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index 31cabef787..f2d3a65517 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -10,24 +10,24 @@ aea_version: '>=0.7.0, <0.8.0' fingerprint: README.md: QmWRXWUuSawx6DoLjYTPHFk7peMKfnY6gKTiGzAtrEXRp9 __init__.py: QmYQuLNyQ8WTjgRYAoKAzoJEb7ocKXvM2hTyK4hsGch5D6 - aea/api.go: QmfXwePWUCCcYpKNYe9Uz7dfnZUVSLN66aaBHXQ9jWZb3J + aea/api.go: QmTMrqwSJhManituQr23P4mwtbW5ssbcwtL46L5a6MDP6L aea/envelope.pb.go: QmRfUNGpCeVJfsW3H1MzCN4pwDWgumfyWufVFp6xvUjjug - aea/envelope.proto: QmSC8EGCKiNFR2vf5bSWymSzYDFMipQW9aQVMwPzQoKb4n + aea/envelope.proto: QmVuvesmfgzj5aKnbFoCocoGEv3T9MR7u6KWn7CT5yfjGi aea/pipe_unix.go: QmSzbAexrSUSA9ZE6VT6MmcWF5MhEcmceQjAbnfRoK7Ruv aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK check_dependencies.py: QmY8EiWapBspyR3zKGEY3TnQayz7hF7DxZ2d9zozVfFhtA connection.py: QmVf4XX25S8FQHcZca6o1xC77Ez9dUNp66K3dGnVB724GC - dht/dhtclient/dhtclient.go: QmajtiksLNCp5YthvVdD9QvuLyudYecpv13sioxbnXdmjZ - dht/dhtclient/dhtclient_test.go: QmYVnf5j8Jyahi5jZNswkGR2tdnM4D1r5Fjdrz654EcD1F + dht/dhtclient/dhtclient.go: QmZCgMowWw4ueCSsyYY12om3RMQA3Ydp98ti65Eqa5ccuz + dht/dhtclient/dhtclient_test.go: QmbTvSmNU1uqjLQo9LdD7zWmDcLuzEacDeg7AG4VXhmriV dht/dhtclient/options.go: QmWs7fQqHRavjwe1PjihHFi85HeJu9qSEVyMdcRviZzaT2 dht/dhtnode/dhtnode.go: QmbyhgbCSAbQ1QsDw7FM7Nt5sZcvhbupA1jv5faxutbV7N - dht/dhtnode/message.pb.go: QmfVCzkS2p2DZbtt8gfBqW3G2oXdPShHGkhME9zbVNfW88 - dht/dhtnode/message.proto: QmVZrdF8RzhoCKcNdNSzHh8vDwDjA9A7adonXFHrDY1Vpg + dht/dhtnode/message.pb.go: QmPnnc1KLiqm82H99YbWQbPWHoYbhHDEZvEaMAYWJsRdB2 + dht/dhtnode/message.proto: QmRvtWkeS9i1szhYWe3StnK1t4zhQjEGJhwZuLPrGbBsfX dht/dhtnode/streams.go: Qmc2JcyiU4wHsgDj6aUunMAp4c5yMzo2ixeqRZHSW5PVwo dht/dhtnode/utils.go: Qmd5f5JCA41kutBE4953JA4XzRuVofYJJ8g4dTsJETtXcd dht/dhtpeer/benchmarks_test.go: QmQcy8WN5DY8Nfp5nkDgRaS6NjQbMzc6DSMbdY8QDPx9ao - dht/dhtpeer/dhtpeer.go: QmVkVBXrPhWTZ4wEU2jhjpUwsYyJYhbA25CuCpYSY69x6U - dht/dhtpeer/dhtpeer_test.go: Qmc3FqVoq8wx8vJ7Q9NEpRbCVCvpAjiwTNwjWMeE91HRGd + dht/dhtpeer/dhtpeer.go: QmRkKAj1uJhNQYwAdPH5eKPRykHyhutwwXJ2yZLbn4c7pb + dht/dhtpeer/dhtpeer_test.go: QmQvoVABqb3oDsH1uqF22d5Tjb2a8Hyf7U7q8jaYQfrjfa dht/dhtpeer/options.go: Qme5L9bRBQZaR5fciPJ1feFwqaQpkdYfW8TaiY1TDXs91H dht/dhttests/dhttests.go: QmQJxw2rJCGUFTXKxH3sf3Zaqta7H76EvaB7mWNNfikFkA dht/monitoring/file.go: QmZB8pNf7infMXWaC5A46rr3mghNfkuSd8UKH4Ceg8nDSM @@ -36,7 +36,7 @@ fingerprint: go.mod: QmScMYT69B7HbQApxXmZFpSwsKBZVPAJimVFYpTVYTvzbe go.sum: QmawFS81vDFYVZUZMKxQjNdKkLxDzfRRfqD1Tcpzze66yd libp2p_node.go: QmSMPBU6GMJ7gAHWMsYGVDSUv5sw48x2rvNKydcxVv7MqL - utils/utils.go: QmcXJwBg95H627cnzgkqgcEDmBCNum74KVnPEvPkhw4BDm + utils/utils.go: QmeVAETLySSzRpkDh8FRGAKBuqeurLTpfufw4CRYL2dW49 fingerprint_ignore_patterns: [] build_entrypoint: check_dependencies.py connections: [] diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go index 8f5eaba9d5..890dff2b07 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient.go @@ -117,7 +117,7 @@ func New(opts ...Option) (*DHTClient, error) { // check if the PoR is delivered for my public key myPublicKey, err := utils.FetchAIPublicKeyFromPubKey(dhtClient.publicKey) status, errPoR := dhtnode.IsValidProofOfRepresentation(dhtClient.myAgentRecord, dhtClient.myAgentRecord.Address, myPublicKey) - if status.Code != dhtnode.Status_SUCCESS { + if err != nil || errPoR != nil || status.Code != dhtnode.Status_SUCCESS { msg := "Invalid AgentRecord" if err != nil { msg += " - " + err.Error() @@ -325,8 +325,17 @@ func (dhtClient *DHTClient) MultiAddr() string { func (dhtClient *DHTClient) RouteEnvelope(envel *aea.Envelope) error { lerror, lwarn, _, ldebug := dhtClient.getLoggers() + // + if envel.Sender != dhtClient.myAgentAddress { + err := errors.New("Sender (" + envel.Sender + ") must match registered address") + lerror(err).Str("addr", dhtClient.myAgentAddress). + Msgf("while routing envelope") + return err + } + target := envel.To + // TODO(LR) check if the record is valid if target == dhtClient.myAgentAddress { ldebug(). Str("op", "route"). @@ -430,7 +439,7 @@ func (dhtClient *DHTClient) RouteEnvelope(envel *aea.Envelope) error { } if status != nil { - err = errors.New(status.Code.String() + " : " + strings.Join(status.Msgs, ",")) + err = errors.New(status.Code.String() + " : " + strings.Join(status.Msgs, ":")) lerror(err).Str("op", "route").Str("target", target). Msgf("failed agent lookup") return err @@ -440,11 +449,11 @@ func (dhtClient *DHTClient) RouteEnvelope(envel *aea.Envelope) error { record := lookupResponse.AgentRecord valid, err := dhtnode.IsValidProofOfRepresentation(record, target, record.PeerPublicKey) if err != nil || valid.Code != dhtnode.Status_SUCCESS { - errMsg := status.Code.String() + " : " + strings.Join(status.Msgs, ",") + errMsg := status.Code.String() + " : " + strings.Join(status.Msgs, ":") if err == nil { err = errors.New(errMsg) } else { - err = errors.Wrap(err, status.Code.String()+" : "+strings.Join(status.Msgs, ",")) + err = errors.Wrap(err, status.Code.String()+" : "+strings.Join(status.Msgs, ":")) } lerror(err).Str("op", "route").Str("target", target). Msgf("invalid agent record") @@ -499,40 +508,194 @@ func (dhtClient *DHTClient) RouteEnvelope(envel *aea.Envelope) error { Str("op", "route"). Str("target", target). Msgf("opening stream to target %s", peerID) - // FIXME-ENVELOPE stream, err = dhtClient.newStreamLoopUntilTimeout(peerID, dhtnode.AeaEnvelopeStream, newStreamTimeout) if err != nil { return err } + + envelBytes, err := proto.Marshal(envel) + if err != nil { + lerror(err). + Str("op", "route"). + Str("target", target). + Msg("couldn't serialize envelope") + errReset := stream.Reset() + ignore(errReset) + return err + } + // TODO(LR) check if source is my agent + aeaEnvelope := &dhtnode.AeaEnvelope{ + Envel: envelBytes, + Record: dhtClient.myAgentRecord, + } + msg = &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_AeaEnvelope{AeaEnvelope: aeaEnvelope}, + } + buf, err = proto.Marshal(msg) + if err != nil { + lerror(err). + Str("op", "route"). + Str("target", target). + Msg("couldn't serialize envelope") + errReset := stream.Reset() + ignore(errReset) + return err + } + ldebug(). Str("op", "route"). Str("target", target). Msg("sending envelope to target...") - err = utils.WriteEnvelope(envel, stream) + err = utils.WriteBytes(stream, buf) if err != nil { + lerror(err). + Str("op", "route"). + Str("target", target). + Msg("couldn't send envelope") errReset := stream.Reset() ignore(errReset) - } else { - stream.Close() + return err } - return err + // wait for response + buf, err = utils.ReadBytes(stream) + if err != nil { + lerror(err). + Str("op", "route"). + Str("target", target). + Msg("while getting confirmation") + errReset := stream.Reset() + ignore(errReset) + return err + } + stream.Close() + + response = &dhtnode.AcnMessage{} + err = proto.Unmarshal(buf, response) + if err != nil { + lerror(err). + Str("op", "route"). + Str("target", target). + Msg("while deserializing acn confirmation message") + return err + } + + // response is expected to be a Status + status = nil + switch pl := response.Payload.(type) { + case *dhtnode.AcnMessage_Status: + status = pl.Status + default: + err = errors.New("Unexpected Acn Message") + lerror(err). + Str("op", "route"). + Str("target", target). + Msg("while deserializing acn confirmation message") + return err + } + + if status.Code != dhtnode.Status_SUCCESS { + err = errors.New(status.Code.String() + " : " + strings.Join(status.Msgs, ":")) + lerror(err). + Str("op", "route"). + Str("target", target). + Msg("failed to deliver envelope") + return err + } + + return err } func (dhtClient *DHTClient) handleAeaEnvelopeStream(stream network.Stream) { lerror, lwarn, _, ldebug := dhtClient.getLoggers() //ldebug().Msgf("Got a new aea envelope stream") - - envel, err := utils.ReadEnvelope(stream) + buf, err := utils.ReadBytes(stream) if err != nil { lerror(err).Msg("while reading envelope from stream") err = stream.Reset() ignore(err) return } - stream.Close() + + // get envelope + msg := &dhtnode.AcnMessage{} + err = proto.Unmarshal(buf, msg) + if err != nil { + lerror(err).Msg("while deserializing acn aea envelope message") + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_SERIALIZATION} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_Status{Status: status}, + } + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Close() + ignore(err) + return + } + + // payload is expected to be AeaEnvelope + var aeaEnvelope *dhtnode.AeaEnvelope + switch pl := msg.Payload.(type) { + case *dhtnode.AcnMessage_AeaEnvelope: + aeaEnvelope = pl.AeaEnvelope + default: + err = errors.New("Unexpected payload") + lerror(err).Msg("while deserializing acn aea envelope message") + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNEXPECTED_PAYLOAD} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_Status{Status: status}, + } + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Close() + ignore(err) + return + } + + envel := &aea.Envelope{} + err = proto.Unmarshal(aeaEnvelope.Envel, envel) + if err != nil { + lerror(err).Msg("while deserializing acn aea envelope message") + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_SERIALIZATION, Msgs: []string{err.Error()}} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_Status{Status: status}, + } + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Close() + ignore(err) + return + } + + remotePubkey, err := utils.FetchAIPublicKeyFromPubKey(stream.Conn().RemotePublicKey()) + ignore(err) + status, err := dhtnode.IsValidProofOfRepresentation(aeaEnvelope.Record, aeaEnvelope.Record.Address, remotePubkey) + if err != nil || status.Code != dhtnode.Status_SUCCESS { + if err == nil { + err = errors.New(status.Code.String() + ":" + strings.Join(status.Msgs, ":")) + } + lerror(err).Msg("incoming envelope PoR is not valid") + response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Close() + ignore(err) + return + } ldebug().Msgf("Received envelope from peer %s", envel.String()) @@ -540,10 +703,46 @@ func (dhtClient *DHTClient) handleAeaEnvelopeStream(stream network.Stream) { err = dhtClient.processEnvelope(envel) if err != nil { lerror(err).Msgf("while processing envelope by agent") + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_AGENT_NOT_READY} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_Status{Status: status}, + } + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Close() + ignore(err) + return } } else { lwarn().Msgf("ignored envelope %s", envel.String()) + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNKNOWN_AGENT_ADDRESS} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_Status{Status: status}, + } + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Close() + ignore(err) + return + } + + status = &dhtnode.Status{Code: dhtnode.Status_SUCCESS} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_Status{Status: status}, } + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Close() + ignore(err) } func (dhtClient *DHTClient) handleAeaAddressStream(stream network.Stream) { @@ -565,13 +764,14 @@ func (dhtClient *DHTClient) handleAeaAddressStream(stream network.Stream) { err = proto.Unmarshal(buf, msg) if err != nil { lerror(err).Str("op", "resolve").Msg("couldn't deserialize acn registration message") - status := &dhtnode.Status{Code: dhtnode.Status_ERROR_GENERIC, Msgs: []string{err.Error()}} + // TOFIX(LR) setting Msgs to err.Error is potentially a security vulnerability + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_SERIALIZATION, Msgs: []string{err.Error()}} response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) ignore(err) err = utils.WriteBytes(stream, buf) ignore(err) - err = stream.Reset() + err = stream.Close() ignore(err) return } @@ -589,7 +789,7 @@ func (dhtClient *DHTClient) handleAeaAddressStream(stream network.Stream) { ignore(err) err = utils.WriteBytes(stream, buf) ignore(err) - err = stream.Reset() + err = stream.Close() ignore(err) return } @@ -691,15 +891,15 @@ func (dhtClient *DHTClient) registerAgentAddress() error { case *dhtnode.AcnMessage_Status: status = pl.Status default: - errReset := stream.Reset() + errReset := stream.Close() ignore(errReset) return err } if status.Code != dhtnode.Status_SUCCESS { - errReset := stream.Reset() + errReset := stream.Close() ignore(errReset) - return errors.New("Registration failed: " + strings.Join(status.Msgs, ",")) + return errors.New("Registration failed: " + strings.Join(status.Msgs, ":")) } stream.Close() diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient_test.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient_test.go index ccf4c961c1..4e3fb10056 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient_test.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/dhtclient_test.go @@ -43,7 +43,7 @@ const ( // TestNew dht client peer func TestNew(t *testing.T) { - rxEnvelopesPeer := make(chan *aea.Envelope) + rxEnvelopesPeer := make(chan *aea.Envelope, 2) dhtPeer, cleanup, err := dhttests.NewDHTPeerWithDefaults(rxEnvelopesPeer) if err != nil { t.Fatal("Failed to create DHTPeer (required for DHTClient):", err) @@ -75,7 +75,7 @@ func TestNew(t *testing.T) { } defer dhtClient.Close() - rxEnvelopesClient := make(chan *aea.Envelope) + rxEnvelopesClient := make(chan *aea.Envelope, 2) dhtClient.ProcessEnvelope(func(envel *aea.Envelope) error { rxEnvelopesClient <- envel return nil @@ -86,7 +86,7 @@ func TestNew(t *testing.T) { // TestRouteEnvelopeToPeerAgent send envelope from DHTClient agent to DHTPeer agent func TestRouteEnvelopeToPeerAgent(t *testing.T) { - rxEnvelopesPeer := make(chan *aea.Envelope) + rxEnvelopesPeer := make(chan *aea.Envelope, 2) dhtPeer, cleanup, err := dhttests.NewDHTPeerWithDefaults(rxEnvelopesPeer) if err != nil { t.Fatal("Failed to create DHTPeer (required for DHTClient):", err) @@ -118,7 +118,7 @@ func TestRouteEnvelopeToPeerAgent(t *testing.T) { } defer dhtClient.Close() - rxEnvelopesClient := make(chan *aea.Envelope) + rxEnvelopesClient := make(chan *aea.Envelope, 2) dhtClient.ProcessEnvelope(func(envel *aea.Envelope) error { rxEnvelopesClient <- envel return nil diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go index e7cfff0fd4..cd3cbfd4c9 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go @@ -33,12 +33,14 @@ const ( Status_ERROR_UNSUPPORTED_VERSION Status_ErrCode = 1 Status_ERROR_UNEXPECTED_PAYLOAD Status_ErrCode = 2 Status_ERROR_GENERIC Status_ErrCode = 3 + Status_ERROR_SERIALIZATION Status_ErrCode = 4 // register (1x) Status_ERROR_INVALID_AGENT_ADDRESS Status_ErrCode = 10 Status_ERROR_INVALID_PUBLIC_KEY Status_ErrCode = 11 Status_ERROR_INVALID_PROOF Status_ErrCode = 12 - // lookup (2x) + // lookup & delivery (2x) Status_ERROR_UNKNOWN_AGENT_ADDRESS Status_ErrCode = 20 + Status_ERROR_AGENT_NOT_READY Status_ErrCode = 21 ) // Enum value maps for Status_ErrCode. @@ -48,20 +50,24 @@ var ( 1: "ERROR_UNSUPPORTED_VERSION", 2: "ERROR_UNEXPECTED_PAYLOAD", 3: "ERROR_GENERIC", + 4: "ERROR_SERIALIZATION", 10: "ERROR_INVALID_AGENT_ADDRESS", 11: "ERROR_INVALID_PUBLIC_KEY", 12: "ERROR_INVALID_PROOF", 20: "ERROR_UNKNOWN_AGENT_ADDRESS", + 21: "ERROR_AGENT_NOT_READY", } Status_ErrCode_value = map[string]int32{ "SUCCESS": 0, "ERROR_UNSUPPORTED_VERSION": 1, "ERROR_UNEXPECTED_PAYLOAD": 2, "ERROR_GENERIC": 3, + "ERROR_SERIALIZATION": 4, "ERROR_INVALID_AGENT_ADDRESS": 10, "ERROR_INVALID_PUBLIC_KEY": 11, "ERROR_INVALID_PROOF": 12, "ERROR_UNKNOWN_AGENT_ADDRESS": 20, + "ERROR_AGENT_NOT_READY": 21, } ) @@ -89,7 +95,7 @@ func (x Status_ErrCode) Number() protoreflect.EnumNumber { // Deprecated: Use Status_ErrCode.Descriptor instead. func (Status_ErrCode) EnumDescriptor() ([]byte, []int) { - return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{4, 0} + return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{5, 0} } type AgentRecord struct { @@ -312,6 +318,62 @@ func (x *LookupResponse) GetAgentRecord() *AgentRecord { return nil } +type AeaEnvelope struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // TOFIX(LR) import aea.Envelop type + Envel []byte `protobuf:"bytes,1,opt,name=envel,proto3" json:"envel,omitempty"` + Record *AgentRecord `protobuf:"bytes,2,opt,name=record,proto3" json:"record,omitempty"` +} + +func (x *AeaEnvelope) Reset() { + *x = AeaEnvelope{} + if protoimpl.UnsafeEnabled { + mi := &file_dht_dhtnode_message_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AeaEnvelope) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AeaEnvelope) ProtoMessage() {} + +func (x *AeaEnvelope) ProtoReflect() protoreflect.Message { + mi := &file_dht_dhtnode_message_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AeaEnvelope.ProtoReflect.Descriptor instead. +func (*AeaEnvelope) Descriptor() ([]byte, []int) { + return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{4} +} + +func (x *AeaEnvelope) GetEnvel() []byte { + if x != nil { + return x.Envel + } + return nil +} + +func (x *AeaEnvelope) GetRecord() *AgentRecord { + if x != nil { + return x.Record + } + return nil +} + type Status struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -324,7 +386,7 @@ type Status struct { func (x *Status) Reset() { *x = Status{} if protoimpl.UnsafeEnabled { - mi := &file_dht_dhtnode_message_proto_msgTypes[4] + mi := &file_dht_dhtnode_message_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -337,7 +399,7 @@ func (x *Status) String() string { func (*Status) ProtoMessage() {} func (x *Status) ProtoReflect() protoreflect.Message { - mi := &file_dht_dhtnode_message_proto_msgTypes[4] + mi := &file_dht_dhtnode_message_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -350,7 +412,7 @@ func (x *Status) ProtoReflect() protoreflect.Message { // Deprecated: Use Status.ProtoReflect.Descriptor instead. func (*Status) Descriptor() ([]byte, []int) { - return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{4} + return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{5} } func (x *Status) GetCode() Status_ErrCode { @@ -378,13 +440,14 @@ type AcnMessage struct { // *AcnMessage_Register // *AcnMessage_LookupRequest // *AcnMessage_LookupResponse + // *AcnMessage_AeaEnvelope Payload isAcnMessage_Payload `protobuf_oneof:"payload"` } func (x *AcnMessage) Reset() { *x = AcnMessage{} if protoimpl.UnsafeEnabled { - mi := &file_dht_dhtnode_message_proto_msgTypes[5] + mi := &file_dht_dhtnode_message_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -397,7 +460,7 @@ func (x *AcnMessage) String() string { func (*AcnMessage) ProtoMessage() {} func (x *AcnMessage) ProtoReflect() protoreflect.Message { - mi := &file_dht_dhtnode_message_proto_msgTypes[5] + mi := &file_dht_dhtnode_message_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -410,7 +473,7 @@ func (x *AcnMessage) ProtoReflect() protoreflect.Message { // Deprecated: Use AcnMessage.ProtoReflect.Descriptor instead. func (*AcnMessage) Descriptor() ([]byte, []int) { - return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{5} + return file_dht_dhtnode_message_proto_rawDescGZIP(), []int{6} } func (x *AcnMessage) GetVersion() string { @@ -455,6 +518,13 @@ func (x *AcnMessage) GetLookupResponse() *LookupResponse { return nil } +func (x *AcnMessage) GetAeaEnvelope() *AeaEnvelope { + if x, ok := x.GetPayload().(*AcnMessage_AeaEnvelope); ok { + return x.AeaEnvelope + } + return nil +} + type isAcnMessage_Payload interface { isAcnMessage_Payload() } @@ -475,6 +545,10 @@ type AcnMessage_LookupResponse struct { LookupResponse *LookupResponse `protobuf:"bytes,5,opt,name=lookup_response,json=lookupResponse,proto3,oneof"` } +type AcnMessage_AeaEnvelope struct { + AeaEnvelope *AeaEnvelope `protobuf:"bytes,6,opt,name=aea_envelope,json=aeaEnvelope,proto3,oneof"` +} + func (*AcnMessage_Status) isAcnMessage_Payload() {} func (*AcnMessage_Register) isAcnMessage_Payload() {} @@ -483,6 +557,8 @@ func (*AcnMessage_LookupRequest) isAcnMessage_Payload() {} func (*AcnMessage_LookupResponse) isAcnMessage_Payload() {} +func (*AcnMessage_AeaEnvelope) isAcnMessage_Payload() {} + var File_dht_dhtnode_message_proto protoreflect.FileDescriptor var file_dht_dhtnode_message_proto_rawDesc = []byte{ @@ -510,44 +586,56 @@ var file_dht_dhtnode_message_proto_rawDesc = []byte{ 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, - 0x52, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0xab, 0x02, - 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, - 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x45, 0x72, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x52, - 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x22, 0xdf, 0x01, 0x0a, 0x07, 0x45, 0x72, - 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, - 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x53, 0x55, - 0x50, 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x10, - 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x45, 0x58, 0x50, - 0x45, 0x43, 0x54, 0x45, 0x44, 0x5f, 0x50, 0x41, 0x59, 0x4c, 0x4f, 0x41, 0x44, 0x10, 0x02, 0x12, - 0x11, 0x0a, 0x0d, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, - 0x10, 0x03, 0x12, 0x1f, 0x0a, 0x1b, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, - 0x4c, 0x49, 0x44, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, - 0x53, 0x10, 0x0a, 0x12, 0x1c, 0x0a, 0x18, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, - 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x4b, 0x45, 0x59, 0x10, - 0x0b, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, - 0x49, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x4f, 0x46, 0x10, 0x0c, 0x12, 0x1f, 0x0a, 0x1b, 0x45, 0x52, - 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x41, 0x47, 0x45, 0x4e, - 0x54, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x14, 0x22, 0x92, 0x02, 0x0a, 0x0a, - 0x41, 0x63, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x48, 0x00, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, - 0x2f, 0x0a, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x52, 0x65, 0x67, 0x69, - 0x73, 0x74, 0x65, 0x72, 0x48, 0x00, 0x52, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, - 0x12, 0x3f, 0x0a, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, - 0x64, 0x65, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x48, 0x00, 0x52, 0x0d, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x42, 0x0a, 0x0f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x72, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x68, 0x74, - 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0x51, 0x0a, + 0x0b, 0x41, 0x65, 0x61, 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x65, 0x6e, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x65, 0x6e, 0x76, + 0x65, 0x6c, 0x12, 0x2c, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x67, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x22, 0xdf, 0x02, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x0a, 0x04, 0x63, + 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x64, 0x68, 0x74, 0x6e, + 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x45, 0x72, 0x72, 0x43, 0x6f, + 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x73, 0x67, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x22, 0x93, 0x02, 0x0a, + 0x07, 0x45, 0x72, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, + 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, + 0x4e, 0x53, 0x55, 0x50, 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, + 0x4f, 0x4e, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, + 0x45, 0x58, 0x50, 0x45, 0x43, 0x54, 0x45, 0x44, 0x5f, 0x50, 0x41, 0x59, 0x4c, 0x4f, 0x41, 0x44, + 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x47, 0x45, 0x4e, 0x45, + 0x52, 0x49, 0x43, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x53, + 0x45, 0x52, 0x49, 0x41, 0x4c, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x04, 0x12, 0x1f, + 0x0a, 0x1b, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, + 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x0a, 0x12, + 0x1c, 0x0a, 0x18, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, + 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x4b, 0x45, 0x59, 0x10, 0x0b, 0x12, 0x17, 0x0a, + 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x50, + 0x52, 0x4f, 0x4f, 0x46, 0x10, 0x0c, 0x12, 0x1f, 0x0a, 0x1b, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, + 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, + 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x14, 0x12, 0x19, 0x0a, 0x15, 0x45, 0x52, 0x52, 0x4f, 0x52, + 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x4e, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x59, + 0x10, 0x15, 0x22, 0xcd, 0x02, 0x0a, 0x0a, 0x41, 0x63, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x64, 0x68, + 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x48, 0x00, 0x52, 0x06, + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, + 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, + 0x64, 0x65, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x48, 0x00, 0x52, 0x08, 0x72, + 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x3f, 0x0a, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, + 0x70, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x16, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, + 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x0f, 0x6c, 0x6f, 0x6f, 0x6b, + 0x75, 0x70, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, + 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x6c, 0x6f, + 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x0c, + 0x61, 0x65, 0x61, 0x5f, 0x65, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x65, 0x61, + 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x65, 0x61, 0x45, + 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, + 0x61, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -563,29 +651,32 @@ func file_dht_dhtnode_message_proto_rawDescGZIP() []byte { } var file_dht_dhtnode_message_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_dht_dhtnode_message_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_dht_dhtnode_message_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_dht_dhtnode_message_proto_goTypes = []interface{}{ (Status_ErrCode)(0), // 0: dhtnode.Status.ErrCode (*AgentRecord)(nil), // 1: dhtnode.AgentRecord (*Register)(nil), // 2: dhtnode.Register (*LookupRequest)(nil), // 3: dhtnode.LookupRequest (*LookupResponse)(nil), // 4: dhtnode.LookupResponse - (*Status)(nil), // 5: dhtnode.Status - (*AcnMessage)(nil), // 6: dhtnode.AcnMessage + (*AeaEnvelope)(nil), // 5: dhtnode.AeaEnvelope + (*Status)(nil), // 6: dhtnode.Status + (*AcnMessage)(nil), // 7: dhtnode.AcnMessage } var file_dht_dhtnode_message_proto_depIdxs = []int32{ 1, // 0: dhtnode.Register.record:type_name -> dhtnode.AgentRecord 1, // 1: dhtnode.LookupResponse.agent_record:type_name -> dhtnode.AgentRecord - 0, // 2: dhtnode.Status.code:type_name -> dhtnode.Status.ErrCode - 5, // 3: dhtnode.AcnMessage.status:type_name -> dhtnode.Status - 2, // 4: dhtnode.AcnMessage.register:type_name -> dhtnode.Register - 3, // 5: dhtnode.AcnMessage.lookup_request:type_name -> dhtnode.LookupRequest - 4, // 6: dhtnode.AcnMessage.lookup_response:type_name -> dhtnode.LookupResponse - 7, // [7:7] is the sub-list for method output_type - 7, // [7:7] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name + 1, // 2: dhtnode.AeaEnvelope.record:type_name -> dhtnode.AgentRecord + 0, // 3: dhtnode.Status.code:type_name -> dhtnode.Status.ErrCode + 6, // 4: dhtnode.AcnMessage.status:type_name -> dhtnode.Status + 2, // 5: dhtnode.AcnMessage.register:type_name -> dhtnode.Register + 3, // 6: dhtnode.AcnMessage.lookup_request:type_name -> dhtnode.LookupRequest + 4, // 7: dhtnode.AcnMessage.lookup_response:type_name -> dhtnode.LookupResponse + 5, // 8: dhtnode.AcnMessage.aea_envelope:type_name -> dhtnode.AeaEnvelope + 9, // [9:9] is the sub-list for method output_type + 9, // [9:9] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_dht_dhtnode_message_proto_init() } @@ -643,7 +734,7 @@ func file_dht_dhtnode_message_proto_init() { } } file_dht_dhtnode_message_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Status); i { + switch v := v.(*AeaEnvelope); i { case 0: return &v.state case 1: @@ -655,6 +746,18 @@ func file_dht_dhtnode_message_proto_init() { } } file_dht_dhtnode_message_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Status); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_dht_dhtnode_message_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*AcnMessage); i { case 0: return &v.state @@ -667,11 +770,12 @@ func file_dht_dhtnode_message_proto_init() { } } } - file_dht_dhtnode_message_proto_msgTypes[5].OneofWrappers = []interface{}{ + file_dht_dhtnode_message_proto_msgTypes[6].OneofWrappers = []interface{}{ (*AcnMessage_Status)(nil), (*AcnMessage_Register)(nil), (*AcnMessage_LookupRequest)(nil), (*AcnMessage_LookupResponse)(nil), + (*AcnMessage_AeaEnvelope)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -679,7 +783,7 @@ func file_dht_dhtnode_message_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_dht_dhtnode_message_proto_rawDesc, NumEnums: 1, - NumMessages: 6, + NumMessages: 7, NumExtensions: 0, NumServices: 0, }, diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto index 3a409b7c5e..60e26f9e39 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto @@ -22,6 +22,12 @@ message LookupResponse { AgentRecord agent_record = 1; } +message AeaEnvelope { + // TOFIX(LR) import aea.Envelop type + bytes envel = 1; + AgentRecord record = 2; +} + message Status { enum ErrCode { // common (0x) @@ -29,12 +35,14 @@ message Status { ERROR_UNSUPPORTED_VERSION = 1; ERROR_UNEXPECTED_PAYLOAD = 2; ERROR_GENERIC = 3; + ERROR_SERIALIZATION = 4; // register (1x) ERROR_INVALID_AGENT_ADDRESS = 10; ERROR_INVALID_PUBLIC_KEY = 11; ERROR_INVALID_PROOF = 12; - // lookup (2x) - ERROR_UNKNOWN_AGENT_ADDRESS = 20; + // lookup & delivery (2x) + ERROR_UNKNOWN_AGENT_ADDRESS = 20; + ERROR_AGENT_NOT_READY = 21; } ErrCode code = 1; @@ -48,5 +56,6 @@ message AcnMessage { Register register = 3; LookupRequest lookup_request = 4; LookupResponse lookup_response = 5; + AeaEnvelope aea_envelope = 6; } } \ No newline at end of file diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go index 265fc25eeb..4706fbf3d9 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer.go @@ -181,8 +181,17 @@ func New(opts ...Option) (*DHTPeer, error) { status, errPoR := dhtnode.IsValidProofOfRepresentation( dhtPeer.myAgentRecord, dhtPeer.myAgentRecord.Address, myPublicKey, ) - if status.Code != dhtnode.Status_SUCCESS { - return nil, errors.New("Invalid AgentRecord - " + err.Error() + " - " + errPoR.Error()) + if err != nil || errPoR != nil || status.Code != dhtnode.Status_SUCCESS { + errMsg := "Invalid AgentRecord" + if err == nil { + err = errors.New(errMsg) + } else { + err = errors.Wrap(err, errMsg) + } + if errPoR != nil { + err = errors.Wrap(err, errPoR.Error()) + } + return nil, err } } @@ -528,7 +537,8 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { err = proto.Unmarshal(buf, msg) if err != nil { lerror(err).Msg("couldn't deserialize acn registration message") - status := &dhtnode.Status{Code: dhtnode.Status_ERROR_GENERIC, Msgs: []string{err.Error()}} + // TOFIX(LR) setting Msgs to err.Error is potentially a security vulnerability + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_SERIALIZATION, Msgs: []string{err.Error()}} response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) ignore(err) @@ -568,7 +578,7 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { myPubKey, err := utils.FetchAIPublicKeyFromPubKey(dhtPeer.publicKey) ignore(err) status, err := dhtnode.IsValidProofOfRepresentation(record, addr, myPubKey) - if err != nil { + if err != nil || status.Code != dhtnode.Status_SUCCESS { lerror(err).Msg("PoR is not valid") response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) @@ -580,6 +590,7 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { return } + // TOFIX(LR) post-pone answer until address successfully registered msg = &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(msg) ignore(err) @@ -629,8 +640,18 @@ func (dhtPeer *DHTPeer) handleNewDelegationConnection(conn net.Conn) { dhtPeer.goroutines.Add(1) go func() { defer dhtPeer.goroutines.Done() - err := dhtPeer.RouteEnvelope(envel) - ignore(err) + if envel.Sender != addr { + err = errors.New("Sender (" + envel.Sender + ") must match registered address") + lerror(err).Str("addr", addr). + Msg("while routing delegate client envelope") + } else { + err := dhtPeer.RouteEnvelope(envel) + if err != nil { + lerror(err).Str("addr", addr). + Msg("while routing delegate client envelope") + // TODO() send error back + } + } }() } @@ -660,7 +681,7 @@ func (dhtPeer *DHTPeer) MultiAddr() string { // RouteEnvelope to its destination func (dhtPeer *DHTPeer) RouteEnvelope(envel *aea.Envelope) error { - lerror, lwarn, linfo, _ := dhtPeer.getLoggers() + lerror, lwarn, linfo, ldebug := dhtPeer.getLoggers() routeCount, _ := dhtPeer.monitor.GetGauge(metricOpRouteCount) routeCountAll, _ := dhtPeer.monitor.GetCounter(metricOpRouteCountAll) @@ -672,29 +693,39 @@ func (dhtPeer *DHTPeer) RouteEnvelope(envel *aea.Envelope) error { routeCountAll.Inc() start := timer.NewTimer() - // get sender agent record - // TOFIX can change function signature to force the caller to provide the record - /* - var record *dhtnode.AgentRecord - sender := envel.Sender - if sender == dhtPeer.myAgentAddress { - record = dhtPeer.myAgentRecord - } else if val, exists := dhtPeer.agentRecords[sender]; exists { - // TOFIX(LR) should acquire RLock - record = val - } else { - err := errors.New("Envelope sender is not registered locally " + sender) - lerror(err).Str("op", "route").Str("addr", envel.To). - Msg("") - return err - } - */ + println("-> Routing envelope:", envel.String()) + + // get sender agent envelRec + // TODO can change function signature to force the caller to provide the envelRec + var envelRec *dhtnode.AgentRecord + sender := envel.Sender + + dhtPeer.agentRecordsLock.RLock() + localRec, existsLocal := dhtPeer.agentRecords[sender] + dhtPeer.agentRecordsLock.RUnlock() + + if sender == dhtPeer.myAgentAddress { + envelRec = dhtPeer.myAgentRecord + } else if existsLocal { + // TOFIX(LR) should acquire RLock + envelRec = localRec + } else { + err := errors.New("Envelope sender is not registered locally " + sender) + lerror(err).Str("op", "route").Str("addr", envel.To). + Msg("") + return err + } target := envel.To + dhtPeer.tcpAddressesLock.RLock() + connDelegate, existsDelegate := dhtPeer.tcpAddresses[target] + dhtPeer.tcpAddressesLock.RUnlock() + if target == dhtPeer.myAgentAddress { linfo().Str("op", "route").Str("addr", target). Msg("route envelope destinated to my local agent...") + // TOFIX(LR) risk of infinite loop for dhtPeer.myAgentReady != nil && !dhtPeer.myAgentReady() { lwarn().Str("op", "route").Str("addr", target). Msg("agent not ready yet, sleeping for some time ...") @@ -712,15 +743,16 @@ func (dhtPeer *DHTPeer) RouteEnvelope(envel *aea.Envelope) error { } else { lwarn().Str("op", "route").Str("addr", target). Msgf("ProcessEnvelope not set, ignoring envelope %s", envel.String()) + return errors.New("Agent not ready") } - } else if conn, exists := dhtPeer.tcpAddresses[target]; exists { + } else if existsDelegate { linfo().Str("op", "route").Str("addr", target). - Msgf("destination is a delegate client %s", conn.RemoteAddr().String()) + Msgf("destination is a delegate client %s", connDelegate.RemoteAddr().String()) routeCount.Dec() routeCountSuccess.Inc() duration := timer.GetTimer(start) opLatencyRoute.Observe(float64(duration.Microseconds())) - return utils.WriteEnvelopeConn(conn, envel) + return utils.WriteEnvelopeConn(connDelegate, envel) } else { var peerID peer.ID var err error @@ -740,7 +772,7 @@ func (dhtPeer *DHTPeer) RouteEnvelope(envel *aea.Envelope) error { } else { linfo().Str("op", "route").Str("addr", target). Msg("did NOT find destination address locally, looking for it in the DHT...") - peerID, _, err = dhtPeer.lookupAddressDHT(target) // FIXME-ENVELOPE + peerID, _, err = dhtPeer.lookupAddressDHT(target) // guarantees peerID has a valid PoR if err != nil { lerror(err).Str("op", "route").Str("addr", target). Msg("while looking up address on the DHT") @@ -769,17 +801,109 @@ func (dhtPeer *DHTPeer) RouteEnvelope(envel *aea.Envelope) error { linfo().Str("op", "route").Str("addr", target). Msgf("sending envelope to target peer %s...", peerID.Pretty()) - err = utils.WriteEnvelope(envel, stream) + + envelBytes, err := proto.Marshal(envel) if err != nil { + lerror(err). + Str("op", "route"). + Str("addr", target). + Msg("couldn't serialize envelope") errReset := stream.Reset() ignore(errReset) - } else { - routeCountSuccess.Inc() - stream.Close() + routeCount.Dec() + return err + } + aeaEnvelope := &dhtnode.AeaEnvelope{ + Envel: envelBytes, + Record: envelRec, + } + msg := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_AeaEnvelope{AeaEnvelope: aeaEnvelope}, + } + buf, err := proto.Marshal(msg) + if err != nil { + lerror(err). + Str("op", "route"). + Str("addr", target). + Msg("couldn't serialize envelope") + errReset := stream.Reset() + ignore(errReset) + routeCount.Dec() + return err + } + + ldebug(). + Str("op", "route"). + Str("target", target). + Msg("sending envelope to target...") + err = utils.WriteBytes(stream, buf) + if err != nil { + lerror(err). + Str("op", "route"). + Str("addr", target). + Msg("couldn't send envelope") + errReset := stream.Reset() + ignore(errReset) + routeCount.Dec() + return err + } + + // wait for response + linfo().Str("op", "route").Str("addr", target). + Msgf("waiting fro envelope delivery confirmation from target peer %s...", peerID.Pretty()) + buf, err = utils.ReadBytes(stream) + if err != nil { + lerror(err). + Str("op", "route"). + Str("addr", target). + Msg("while getting confirmation") + errReset := stream.Reset() + ignore(errReset) + routeCount.Dec() + return err + } + + stream.Close() + + response := &dhtnode.AcnMessage{} + err = proto.Unmarshal(buf, response) + if err != nil { + lerror(err). + Str("op", "route"). + Str("addr", target). + Msg("while deserializing acn confirmation message") + routeCount.Dec() + return err + } + + // response is expected to be a Status + var status *dhtnode.Status + switch pl := response.Payload.(type) { + case *dhtnode.AcnMessage_Status: + status = pl.Status + default: + err = errors.New("Unexpected Acn Message") + lerror(err). + Str("op", "route"). + Str("addr", target). + Msg("while deserializing acn confirmation message") + routeCount.Dec() + return err + } + + if status.Code != dhtnode.Status_SUCCESS { + err = errors.New(status.Code.String() + " : " + strings.Join(status.Msgs, ":")) + lerror(err). + Str("op", "route"). + Str("addr", target). + Msg("failed to deliver envelope") + routeCount.Dec() + return err } routeCount.Dec() - return err + return nil } return nil @@ -886,7 +1010,7 @@ func (dhtPeer *DHTPeer) lookupAddressDHT(address string) (peer.ID, *dhtnode.Agen } if status != nil { - err = errors.New(status.Code.String() + " : " + strings.Join(status.Msgs, ",")) + err = errors.New(status.Code.String() + " : " + strings.Join(status.Msgs, ":")) lwarn().Str("op", "lookup").Str("addr", address). Msgf("Failed agent lookup response from provider %s (%s), looking up other providers...", provider, err.Error()) continue @@ -896,11 +1020,11 @@ func (dhtPeer *DHTPeer) lookupAddressDHT(address string) (peer.ID, *dhtnode.Agen record := lookupResponse.AgentRecord valid, err := dhtnode.IsValidProofOfRepresentation(record, address, record.PeerPublicKey) if err != nil || valid.Code != dhtnode.Status_SUCCESS { - errMsg := status.Code.String() + " : " + strings.Join(status.Msgs, ",") + errMsg := status.Code.String() + " : " + strings.Join(status.Msgs, ":") if err == nil { err = errors.New(errMsg) } else { - err = errors.Wrap(err, status.Code.String()+" : "+strings.Join(status.Msgs, ",")) + err = errors.Wrap(err, status.Code.String()+" : "+strings.Join(status.Msgs, ":")) } lwarn().Str("op", "lookup").Str("addr", address). Msgf("invalid agent record from provider %s (%s), looking up other providers...", provider, err.Error()) @@ -940,33 +1064,165 @@ func (dhtPeer *DHTPeer) handleAeaEnvelopeStream(stream network.Stream) { //linfo().Msg("Got a new aea envelope stream") - envel, err := utils.ReadEnvelope(stream) + buf, err := utils.ReadBytes(stream) if err != nil { lerror(err).Msg("while reading envelope from stream") err = stream.Reset() ignore(err) return } - stream.Close() + + // get envelope + msg := &dhtnode.AcnMessage{} + err = proto.Unmarshal(buf, msg) + if err != nil { + lerror(err).Msg("while deserializing acn aea envelope message") + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_SERIALIZATION} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_Status{Status: status}, + } + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Close() + ignore(err) + return + } + + // payload is expected to be AeaEnvelope + var aeaEnvelope *dhtnode.AeaEnvelope + switch pl := msg.Payload.(type) { + case *dhtnode.AcnMessage_AeaEnvelope: + aeaEnvelope = pl.AeaEnvelope + default: + err = errors.New("Unexpected payload") + lerror(err).Msg("while deserializing acn aea envelope message") + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNEXPECTED_PAYLOAD} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_Status{Status: status}, + } + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Close() + ignore(err) + return + } + + envel := &aea.Envelope{} + err = proto.Unmarshal(aeaEnvelope.Envel, envel) + if err != nil { + lerror(err).Msg("while deserializing acn aea envelope message") + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_SERIALIZATION, Msgs: []string{err.Error()}} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_Status{Status: status}, + } + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Close() + ignore(err) + return + } + + remotePubkey, err := utils.FetchAIPublicKeyFromPubKey(stream.Conn().RemotePublicKey()) + ignore(err) + status, err := dhtnode.IsValidProofOfRepresentation(aeaEnvelope.Record, aeaEnvelope.Record.Address, remotePubkey) + if err != nil || status.Code != dhtnode.Status_SUCCESS { + if err == nil { + err = errors.New(status.Code.String() + ":" + strings.Join(status.Msgs, ":")) + } + lerror(err).Msg("incoming envelope PoR is not valid") + response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Close() + ignore(err) + return + } linfo().Msgf("Received envelope from peer %s", envel.String()) // check if destination is a tcp client - if conn, exists := dhtPeer.tcpAddresses[envel.To]; exists { - linfo().Msgf("Sending envelope to tcp delegate client %s...", conn.RemoteAddr().String()) - err = utils.WriteEnvelopeConn(conn, envel) + + dhtPeer.tcpAddressesLock.RLock() + connDelegate, existsDelegate := dhtPeer.tcpAddresses[envel.To] + dhtPeer.tcpAddressesLock.RUnlock() + + if existsDelegate { + linfo().Msgf("Sending envelope to tcp delegate client %s...", connDelegate.RemoteAddr().String()) + err = utils.WriteEnvelopeConn(connDelegate, envel) if err != nil { - lerror(err).Msgf("while sending envelope to tcp client %s", conn.RemoteAddr().String()) + lerror(err).Msgf("while sending envelope to tcp client %s", connDelegate.RemoteAddr().String()) + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_AGENT_NOT_READY} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_Status{Status: status}, + } + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Close() + ignore(err) + return } - } else if envel.To == dhtPeer.myAgentAddress && dhtPeer.processEnvelope != nil { - linfo().Msg("Processing envelope by local agent...") - err = dhtPeer.processEnvelope(envel) - if err != nil { + } else if envel.To == dhtPeer.myAgentAddress { + if dhtPeer.processEnvelope == nil { lerror(err).Msgf("while processing envelope by agent") + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_AGENT_NOT_READY} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_Status{Status: status}, + } + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Close() + ignore(err) + return } + linfo().Msg("Processing envelope by local agent...") + err = dhtPeer.processEnvelope(envel) + ignore(err) } else { lwarn().Msgf("ignored envelope %s", envel.String()) + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_UNKNOWN_AGENT_ADDRESS} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_Status{Status: status}, + } + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Close() + ignore(err) + return + } + + // all good + status = &dhtnode.Status{Code: dhtnode.Status_SUCCESS} + response := &dhtnode.AcnMessage{ + Version: dhtnode.CurrentVersion, + Payload: &dhtnode.AcnMessage_Status{Status: status}, } + buf, err = proto.Marshal(response) + ignore(err) + err = utils.WriteBytes(stream, buf) + ignore(err) + err = stream.Close() + ignore(err) + } func (dhtPeer *DHTPeer) handleAeaAddressStream(stream network.Stream) { @@ -988,13 +1244,14 @@ func (dhtPeer *DHTPeer) handleAeaAddressStream(stream network.Stream) { err = proto.Unmarshal(buf, msg) if err != nil { lerror(err).Str("op", "resolve").Msg("couldn't deserialize acn registration message") - status := &dhtnode.Status{Code: dhtnode.Status_ERROR_GENERIC, Msgs: []string{err.Error()}} + // TOFIX(LR) setting Msgs to err.Error is potentially a security vulnerability + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_SERIALIZATION, Msgs: []string{err.Error()}} response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) ignore(err) err = utils.WriteBytes(stream, buf) ignore(err) - err = stream.Reset() + err = stream.Close() ignore(err) return } @@ -1012,7 +1269,7 @@ func (dhtPeer *DHTPeer) handleAeaAddressStream(stream network.Stream) { ignore(err) err = utils.WriteBytes(stream, buf) ignore(err) - err = stream.Reset() + err = stream.Close() ignore(err) return } @@ -1030,6 +1287,9 @@ func (dhtPeer *DHTPeer) handleAeaAddressStream(stream network.Stream) { dhtPeer.tcpAddressesLock.RLock() _, existsDelegate := dhtPeer.tcpAddresses[reqAddress] dhtPeer.tcpAddressesLock.RUnlock() + dhtPeer.agentRecordsLock.RLock() + localRec := dhtPeer.agentRecords[reqAddress] + dhtPeer.agentRecordsLock.RUnlock() if reqAddress == dhtPeer.myAgentAddress { peerID, err := peer.IDFromPublicKey(dhtPeer.publicKey) @@ -1044,7 +1304,7 @@ func (dhtPeer *DHTPeer) handleAeaAddressStream(stream network.Stream) { linfo().Str("op", "resolve").Str("addr", reqAddress). Msg("found address in my relay clients map") sPeerID = idRelay - sRecord = dhtPeer.agentRecords[reqAddress] + sRecord = localRec } else if existsDelegate { linfo().Str("op", "resolve").Str("addr", reqAddress). Msgf("found address in my delegate clients map") @@ -1054,7 +1314,7 @@ func (dhtPeer *DHTPeer) handleAeaAddressStream(stream network.Stream) { Msgf("CRITICAL could not get peer ID from public key %s", dhtPeer.publicKey) } else { sPeerID = peerID.Pretty() - sRecord = dhtPeer.agentRecords[reqAddress] + sRecord = localRec } } else { // needed when a relay client queries for a peer ID @@ -1208,13 +1468,14 @@ func (dhtPeer *DHTPeer) handleAeaRegisterStream(stream network.Stream) { err = proto.Unmarshal(buf, msg) if err != nil { lerror(err).Msg("couldn't deserialize acn registration message") - status := &dhtnode.Status{Code: dhtnode.Status_ERROR_GENERIC, Msgs: []string{err.Error()}} + // TOFIX(LR) setting Msgs to err.Error is potentially a security vulnerability + status := &dhtnode.Status{Code: dhtnode.Status_ERROR_SERIALIZATION, Msgs: []string{err.Error()}} response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) ignore(err) err = utils.WriteBytes(stream, buf) ignore(err) - err = stream.Reset() + err = stream.Close() ignore(err) return } @@ -1234,7 +1495,7 @@ func (dhtPeer *DHTPeer) handleAeaRegisterStream(stream network.Stream) { ignore(err) err = utils.WriteBytes(stream, buf) ignore(err) - err = stream.Reset() + err = stream.Close() ignore(err) return } @@ -1248,18 +1509,22 @@ func (dhtPeer *DHTPeer) handleAeaRegisterStream(stream network.Stream) { clientPubKey, err := utils.FetchAIPublicKeyFromPubKey(stream.Conn().RemotePublicKey()) ignore(err) status, err := dhtnode.IsValidProofOfRepresentation(record, record.Address, clientPubKey) - if err != nil { + if err != nil || status.Code != dhtnode.Status_SUCCESS { + if err == nil { + err = errors.New(status.Code.String() + ":" + strings.Join(status.Msgs, ":")) + } lerror(err).Msg("PoR is not valid") response := &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(response) ignore(err) err = utils.WriteBytes(stream, buf) ignore(err) - err = stream.Reset() + err = stream.Close() ignore(err) return } + // TOFIX(LR) post-pone answer until address successfully registered msg = &dhtnode.AcnMessage{Version: dhtnode.CurrentVersion, Payload: &dhtnode.AcnMessage_Status{Status: status}} buf, err = proto.Marshal(msg) ignore(err) @@ -1270,6 +1535,7 @@ func (dhtPeer *DHTPeer) handleAeaRegisterStream(stream network.Stream) { return } + stream.Close() nbrClients.Inc() //linfo().Str("op", "register"). @@ -1292,8 +1558,6 @@ func (dhtPeer *DHTPeer) handleAeaRegisterStream(stream network.Stream) { lerror(err).Str("op", "register"). Str("addr", clientAddr). Msg("while announcing client address to the dht") - err = stream.Reset() - ignore(err) return } } diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go index a394d823cd..85d3a062ab 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtpeer/dhtpeer_test.go @@ -148,7 +148,8 @@ func TestRoutingDHTPeerToSelf(t *testing.T) { }) err = dhtPeer.RouteEnvelope(&aea.Envelope{ - To: AgentsTestAddresses[0], + To: AgentsTestAddresses[0], + Sender: AgentsTestAddresses[0], }) if err != nil { t.Error("Failed to Route envelope to local Agent") @@ -180,7 +181,7 @@ func TestRoutingDHTPeerToDHTPeerDirect(t *testing.T) { } defer cleanup2() - rxPeer1 := make(chan *aea.Envelope) + rxPeer1 := make(chan *aea.Envelope, 2) dhtPeer1.ProcessEnvelope(func(envel *aea.Envelope) error { rxPeer1 <- envel err := dhtPeer1.RouteEnvelope(&aea.Envelope{ @@ -190,7 +191,7 @@ func TestRoutingDHTPeerToDHTPeerDirect(t *testing.T) { return err }) - rxPeer2 := make(chan *aea.Envelope) + rxPeer2 := make(chan *aea.Envelope, 2) dhtPeer2.ProcessEnvelope(func(envel *aea.Envelope) error { rxPeer2 <- envel return nil @@ -239,7 +240,7 @@ func TestRoutingDHTPeerToDHTPeerIndirect(t *testing.T) { } defer cleanup2() - rxPeer1 := make(chan *aea.Envelope) + rxPeer1 := make(chan *aea.Envelope, 2) dhtPeer1.ProcessEnvelope(func(envel *aea.Envelope) error { rxPeer1 <- envel err := dhtPeer1.RouteEnvelope(&aea.Envelope{ @@ -249,7 +250,7 @@ func TestRoutingDHTPeerToDHTPeerIndirect(t *testing.T) { return err }) - rxPeer2 := make(chan *aea.Envelope) + rxPeer2 := make(chan *aea.Envelope, 2) dhtPeer2.ProcessEnvelope(func(envel *aea.Envelope) error { rxPeer2 <- envel return nil @@ -272,7 +273,7 @@ func TestRoutingDHTPeerToDHTPeerIndirect(t *testing.T) { // TestRoutingDHTPeerToDHTPeerIndirectTwoHops two dht peers connected to different peers func TestRoutingDHTPeerToDHTPeerIndirectTwoHops(t *testing.T) { entryPeer1, cleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], "", DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -308,7 +309,7 @@ func TestRoutingDHTPeerToDHTPeerIndirectTwoHops(t *testing.T) { } defer cleanup2() - rxPeer1 := make(chan *aea.Envelope) + rxPeer1 := make(chan *aea.Envelope, 2) dhtPeer1.ProcessEnvelope(func(envel *aea.Envelope) error { rxPeer1 <- envel err := dhtPeer1.RouteEnvelope(&aea.Envelope{ @@ -318,7 +319,7 @@ func TestRoutingDHTPeerToDHTPeerIndirectTwoHops(t *testing.T) { return err }) - rxPeer2 := make(chan *aea.Envelope) + rxPeer2 := make(chan *aea.Envelope, 2) dhtPeer2.ProcessEnvelope(func(envel *aea.Envelope) error { rxPeer2 <- envel return nil @@ -359,7 +360,7 @@ func TestRoutingDHTPeerToDHTPeerFullConnectivity(t *testing.T) { t.Fatal("Failed to initialize DHTPeer", i, ":", err) } - rx := make(chan *aea.Envelope) + rx := make(chan *aea.Envelope, 2) peer.ProcessEnvelope(func(envel *aea.Envelope) error { rx <- envel if string(envel.Message) == "ping" { @@ -429,7 +430,7 @@ func TestRoutingDHTClientToDHTPeerX(t *testing.T) { } defer clientCleanup() - rxPeer := make(chan *aea.Envelope) + rxPeer := make(chan *aea.Envelope, 2) peer.ProcessEnvelope(func(envel *aea.Envelope) error { rxPeer <- envel return peer.RouteEnvelope(&aea.Envelope{ @@ -438,7 +439,7 @@ func TestRoutingDHTClientToDHTPeerX(t *testing.T) { }) }) - rxClient := make(chan *aea.Envelope) + rxClient := make(chan *aea.Envelope, 2) client.ProcessEnvelope(func(envel *aea.Envelope) error { rxClient <- envel return nil @@ -487,7 +488,7 @@ func TestRoutingDHTClientToDHTPeerIndirect(t *testing.T) { } defer clientCleanup() - rxPeer := make(chan *aea.Envelope) + rxPeer := make(chan *aea.Envelope, 2) peer.ProcessEnvelope(func(envel *aea.Envelope) error { rxPeer <- envel return peer.RouteEnvelope(&aea.Envelope{ @@ -496,7 +497,7 @@ func TestRoutingDHTClientToDHTPeerIndirect(t *testing.T) { }) }) - rxClient := make(chan *aea.Envelope) + rxClient := make(chan *aea.Envelope, 2) client.ProcessEnvelope(func(envel *aea.Envelope) error { rxClient <- envel return nil @@ -544,7 +545,7 @@ func TestRoutingDHTClientToDHTClient(t *testing.T) { } defer clientCleanup2() - rxClient1 := make(chan *aea.Envelope) + rxClient1 := make(chan *aea.Envelope, 2) client1.ProcessEnvelope(func(envel *aea.Envelope) error { rxClient1 <- envel return client1.RouteEnvelope(&aea.Envelope{ @@ -553,7 +554,7 @@ func TestRoutingDHTClientToDHTClient(t *testing.T) { }) }) - rxClient2 := make(chan *aea.Envelope) + rxClient2 := make(chan *aea.Envelope, 2) client2.ProcessEnvelope(func(envel *aea.Envelope) error { rxClient2 <- envel return nil @@ -576,7 +577,7 @@ func TestRoutingDHTClientToDHTClient(t *testing.T) { // TestRoutingDHTClientToDHTClientIndirect dht client to dht client connected to a different peer func TestRoutingDHTClientToDHTClientIndirect(t *testing.T) { peer1, peerCleanup1, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], "", DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -609,7 +610,7 @@ func TestRoutingDHTClientToDHTClientIndirect(t *testing.T) { } defer clientCleanup2() - rxClient1 := make(chan *aea.Envelope) + rxClient1 := make(chan *aea.Envelope, 2) client1.ProcessEnvelope(func(envel *aea.Envelope) error { rxClient1 <- envel return client1.RouteEnvelope(&aea.Envelope{ @@ -618,7 +619,7 @@ func TestRoutingDHTClientToDHTClientIndirect(t *testing.T) { }) }) - rxClient2 := make(chan *aea.Envelope) + rxClient2 := make(chan *aea.Envelope, 2) client2.ProcessEnvelope(func(envel *aea.Envelope) error { rxClient2 <- envel return nil @@ -661,7 +662,7 @@ func TestRoutingDelegateClientToDHTPeerX(t *testing.T) { } defer clientCleanup() - rxPeer := make(chan *aea.Envelope) + rxPeer := make(chan *aea.Envelope, 2) peer.ProcessEnvelope(func(envel *aea.Envelope) error { rxPeer <- envel return nil @@ -715,7 +716,7 @@ func TestRoutingDelegateClientToDHTPeerIndirect(t *testing.T) { } defer clientCleanup() - rxPeer1 := make(chan *aea.Envelope) + rxPeer1 := make(chan *aea.Envelope, 2) peer1.ProcessEnvelope(func(envel *aea.Envelope) error { rxPeer1 <- envel return nil @@ -780,7 +781,7 @@ func TestRoutingDelegateClientToDHTPeerIndirectTwoHops(t *testing.T) { } defer clientCleanup() - rxPeer1 := make(chan *aea.Envelope) + rxPeer1 := make(chan *aea.Envelope, 2) peer1.ProcessEnvelope(func(envel *aea.Envelope) error { rxPeer1 <- envel return nil @@ -857,7 +858,7 @@ func TestRoutingDelegateClientToDelegateClient(t *testing.T) { // TestRoutingDelegateClientToDelegateClientIndirect func TestRoutingDelegateClientToDelegateClientIndirect(t *testing.T) { peer1, peer1Cleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], "", DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -912,7 +913,7 @@ func TestRoutingDelegateClientToDelegateClientIndirect(t *testing.T) { // TestRoutingDelegateClientToDHTClientDirect func TestRoutingDelegateClientToDHTClientDirect(t *testing.T) { peer, peerCleanup, err := SetupLocalDHTPeer( - FetchAITestKeys[0], AgentsTestKeys[0], DefaultLocalPort, DefaultDelegatePort, + FetchAITestKeys[0], "", DefaultLocalPort, DefaultDelegatePort, []string{}, ) if err != nil { @@ -934,7 +935,7 @@ func TestRoutingDelegateClientToDHTClientDirect(t *testing.T) { } defer delegateClientCleanup() - rxClientDHT := make(chan *aea.Envelope) + rxClientDHT := make(chan *aea.Envelope, 2) dhtClient.ProcessEnvelope(func(envel *aea.Envelope) error { rxClientDHT <- envel return dhtClient.RouteEnvelope(&aea.Envelope{ @@ -992,7 +993,7 @@ func TestRoutingDelegateClientToDHTClientIndirect(t *testing.T) { } defer delegateClientCleanup() - rxClientDHT := make(chan *aea.Envelope) + rxClientDHT := make(chan *aea.Envelope, 2) dhtClient.ProcessEnvelope(func(envel *aea.Envelope) error { rxClientDHT <- envel return dhtClient.RouteEnvelope(&aea.Envelope{ @@ -1048,7 +1049,7 @@ func TestRoutingAllToAll(t *testing.T) { } defer dhtPeerCleanup1() - rxPeerDHT1 := make(chan *aea.Envelope) + rxPeerDHT1 := make(chan *aea.Envelope, 100) dhtPeer1.ProcessEnvelope(func(envel *aea.Envelope) error { rxPeerDHT1 <- envel if string(envel.Message) == "ping" { @@ -1076,7 +1077,7 @@ func TestRoutingAllToAll(t *testing.T) { } defer dhtPeerCleanup2() - rxPeerDHT2 := make(chan *aea.Envelope) + rxPeerDHT2 := make(chan *aea.Envelope, 100) dhtPeer2.ProcessEnvelope(func(envel *aea.Envelope) error { rxPeerDHT2 <- envel if string(envel.Message) == "ping" { @@ -1104,7 +1105,7 @@ func TestRoutingAllToAll(t *testing.T) { } defer dhtPeerCleanup3() - rxPeerDHT3 := make(chan *aea.Envelope) + rxPeerDHT3 := make(chan *aea.Envelope, 100) dhtPeer3.ProcessEnvelope(func(envel *aea.Envelope) error { rxPeerDHT3 <- envel if string(envel.Message) == "ping" { @@ -1132,7 +1133,7 @@ func TestRoutingAllToAll(t *testing.T) { } defer dhtPeerCleanup4() - rxPeerDHT4 := make(chan *aea.Envelope) + rxPeerDHT4 := make(chan *aea.Envelope, 100) dhtPeer4.ProcessEnvelope(func(envel *aea.Envelope) error { rxPeerDHT4 <- envel if string(envel.Message) == "ping" { @@ -1161,7 +1162,7 @@ func TestRoutingAllToAll(t *testing.T) { } defer dhtClientCleanup1() - rxClientDHT1 := make(chan *aea.Envelope) + rxClientDHT1 := make(chan *aea.Envelope, 100) dhtClient1.ProcessEnvelope(func(envel *aea.Envelope) error { rxClientDHT1 <- envel if string(envel.Message) == "ping" { @@ -1188,7 +1189,7 @@ func TestRoutingAllToAll(t *testing.T) { } defer dhtClientCleanup2() - rxClientDHT2 := make(chan *aea.Envelope) + rxClientDHT2 := make(chan *aea.Envelope, 100) dhtClient2.ProcessEnvelope(func(envel *aea.Envelope) error { rxClientDHT2 <- envel if string(envel.Message) == "ping" { @@ -1215,7 +1216,7 @@ func TestRoutingAllToAll(t *testing.T) { } defer dhtClientCleanup3() - rxClientDHT3 := make(chan *aea.Envelope) + rxClientDHT3 := make(chan *aea.Envelope, 100) dhtClient3.ProcessEnvelope(func(envel *aea.Envelope) error { rxClientDHT3 <- envel if string(envel.Message) == "ping" { @@ -1244,7 +1245,7 @@ func TestRoutingAllToAll(t *testing.T) { } defer delegateClientCleanup1() - rxClientDelegate1 := make(chan *aea.Envelope) + rxClientDelegate1 := make(chan *aea.Envelope, 100) delegateClient1.ProcessEnvelope(func(envel *aea.Envelope) error { rxClientDelegate1 <- envel if string(envel.Message) == "ping" { @@ -1271,7 +1272,7 @@ func TestRoutingAllToAll(t *testing.T) { } defer delegateClientCleanup2() - rxClientDelegate2 := make(chan *aea.Envelope) + rxClientDelegate2 := make(chan *aea.Envelope, 100) delegateClient2.ProcessEnvelope(func(envel *aea.Envelope) error { rxClientDelegate2 <- envel if string(envel.Message) == "ping" { @@ -1298,7 +1299,7 @@ func TestRoutingAllToAll(t *testing.T) { } defer delegateClientCleanup3() - rxClientDelegate3 := make(chan *aea.Envelope) + rxClientDelegate3 := make(chan *aea.Envelope, 100) delegateClient3.ProcessEnvelope(func(envel *aea.Envelope) error { rxClientDelegate3 <- envel if string(envel.Message) == "ping" { @@ -1477,7 +1478,7 @@ func SetupDHTClient(key string, agentKey string, entry []string) (*dhtclient.DHT dhtClient, err := dhtclient.New(opts...) if err != nil { - println(err) + println("dhtclient.New:", err.Error()) return nil, nil, err } @@ -1532,7 +1533,7 @@ func SetupDelegateClient(key string, host string, port uint16, peerPubKey string } client.PoR = signature - client.Rx = make(chan *aea.Envelope) + client.Rx = make(chan *aea.Envelope, 2) client.Conn, err = net.Dial("tcp", host+":"+strconv.FormatInt(int64(port), 10)) if err != nil { return nil, nil, err diff --git a/packages/fetchai/connections/p2p_libp2p/utils/utils.go b/packages/fetchai/connections/p2p_libp2p/utils/utils.go index 6ca810d3a5..8ee6fb0bef 100644 --- a/packages/fetchai/connections/p2p_libp2p/utils/utils.go +++ b/packages/fetchai/connections/p2p_libp2p/utils/utils.go @@ -248,7 +248,6 @@ func ConvertStrEncodedSignatureToDER(signature []byte) []byte { sigDER[offset] = 0x02 sigDER[offset+1] = byte(len(sb)) copy(sigDER[offset+2:], sb) - println(len(sigDER)) return sigDER } diff --git a/packages/hashes.csv b/packages/hashes.csv index 8bcf828603..67efe1a161 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,7 +34,7 @@ fetchai/connections/http_server,QmP3nhFMyEi7N64oLV5EQDNowhS8be3jVefxkwir4PKpQ9 fetchai/connections/ledger,QmaViv1ZK4jLgQXycpYLjboFFcznN9zXUZT4JKdbeS2wQ8 fetchai/connections/local,QmNPNVkqtDtzENpv1XqM4LHZna6v6jJ7Hsk6RB14iSdtjG fetchai/connections/oef,QmVGcKDeDMEhtcBnDNVTWchHkA2YhHnDGoK8izobnDQKmw -fetchai/connections/p2p_libp2p,QmWhtpMTzqEV8ezSc2qVoDpCSpGw5MUaQpVt2comhw99Wi +fetchai/connections/p2p_libp2p,QmVjnho2ehRqLmYMYyDSkBg3zdWAP85AatmgzP3ryEFKa4 fetchai/connections/p2p_libp2p_client,QmPV4eyGijvdqUzMAfgBrTDnp5rhJhSH4YbyyZXmnar8hk fetchai/connections/p2p_stub,QmaHtQs9dJRnF27WDZSVW3FFEGbY1419NH8u67B8hgnteV fetchai/connections/scaffold,QmcjctNNxnFZ6xU2aDs7WvzNHMvvv4PuhnS739yWxHmJKz From 921bf43e64de1dc673e230db16a7a6861073f8fb Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Thu, 17 Dec 2020 08:23:35 +0000 Subject: [PATCH 009/204] Add PoR support to p2p_libp2p python connection --- .../connections/p2p_libp2p/connection.py | 85 ++++++++++++++++++- .../connections/p2p_libp2p/connection.yaml | 2 +- packages/hashes.csv | 2 +- tests/conftest.py | 61 ++++++++++++- 4 files changed, 143 insertions(+), 7 deletions(-) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index 2d62bd0954..982260e030 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -70,6 +70,8 @@ LIBP2P_SUCCESS_MESSAGE = "Peer running in " +POR_DEFAULT_SERVICE_ID = "" + def _ip_all_private_or_all_public(addrs: List[str]) -> bool: if len(addrs) == 0: @@ -203,12 +205,69 @@ def port(self) -> int: return self._port +class AgentRecord: + """Agent Proof-of-Representation to peer""" + + def __init__( + self, address: str, public_key: str, peer_public_key: str, signature: str, + ): + """Initialize the AgentRecord""" + self._service_id = POR_DEFAULT_SERVICE_ID + self._address = address + self._public_key = public_key + self._peer_public_key = peer_public_key + self._signature = signature + + @property + def address(self) -> str: + """Get agent address""" + return self._address + + @property + def public_key(self) -> str: + """Get agent public key""" + return self._public_key + + @property + def peer_public_key(self) -> str: + """Get agent's representative peer public key""" + return self._peer_public_key + + @property + def signature(self) -> str: + """Get record signature""" + return self._signature + + @property + def service_id(self) -> str: + """Get record service id""" + return self._service_id + + def __str__(self): + """Get string representation.""" + return f"(address={self.address}, pubkey={self.public_key}, peer_pubkey={self.peer_public_key}, signature={self.signature})" + + def is_valid_for(self, addr: str, public_key: str): + """ + Check if the agent record is valid + + """ + + if self._address != addr: + return False + if self._peer_public_key != public_key: + return False + # TODO(LR) check if agent address and public key match + # TODO(LR) check if signature is valid + return True + + class Libp2pNode: """Libp2p p2p node as a subprocess with named pipes interface.""" def __init__( self, - agent_addr: Address, + agent_record: AgentRecord, key: Crypto, module_path: str, clargs: Optional[List[str]] = None, @@ -225,7 +284,7 @@ def __init__( """ Initialize a p2p libp2p node. - :param agent_addr: the agent address. + :param agent_record: the agent proof-of-representation for peer. :param key: secp256k1 curve private key. :param module_path: the module path. :param clargs: the command line arguments for the libp2p node @@ -240,7 +299,8 @@ def __init__( :param peer_registration_delay: add artificial delay to agent registration in seconds """ - self.address = agent_addr + self.record = agent_record + self.address = self.record.address # node id in the p2p network self.key = key.private_key @@ -349,6 +409,13 @@ async def start(self) -> None: self._config += "AEA_P2P_URI_MONITORING={}\n".format( str(self.monitoring_uri) if self.monitoring_uri is not None else "" ) + self._config += "AEA_P2P_POR_ADDRESS={}\n".format(self.record.address) + self._config += "AEA_P2P_POR_PUBKEY={}\n".format(self.record.public_key) + self._config += "AEA_P2P_POR_PEER_PUBKEY={}\n".format( + self.record.peer_public_key + ) + self._config += "AEA_P2P_POR_SIGNATURE={}\n".format(self.record.signature) + self._config += "AEA_P2P_POR_SERVICE_ID={}\n".format(self.record.service_id) self._config += "AEA_P2P_CFG_REGISTRATION_DELAY={}\n".format( str(self.peer_registration_delay) if self.peer_registration_delay is not None @@ -511,6 +578,12 @@ def stop(self) -> None: os.remove(LIBP2P_NODE_ENV_FILE) +def _get_por_from_configuration(key: Crypto) -> AgentRecord: + signature = key.sign_message(key.public_key.encode("utf-8")) + record = AgentRecord(key.address, key.public_key, key.public_key, signature) + return record + + class P2PLibp2pConnection(Connection): """A libp2p p2p node connection.""" @@ -616,6 +689,10 @@ def __init__(self, **kwargs): "Node's public ip and entry peers ip addresses are not in the same ip address space (private/public)" ) + record = _get_por_from_configuration(key) + if not record.is_valid_for(self.address, key.public_key): + raise ValueError("Invalid Proof-of-Representation {}".format(str(record))) + # libp2p local node self.logger.debug("Public key used by libp2p node: {}".format(key.public_key)) temp_dir = tempfile.mkdtemp() @@ -623,7 +700,7 @@ def __init__(self, **kwargs): shutil.copytree(LIBP2P_NODE_MODULE, self.libp2p_workdir) self.node = Libp2pNode( - self.address, + record, key, self.libp2p_workdir, LIBP2P_NODE_CLARGS, diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index f2d3a65517..c192249680 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -16,7 +16,7 @@ fingerprint: aea/pipe_unix.go: QmSzbAexrSUSA9ZE6VT6MmcWF5MhEcmceQjAbnfRoK7Ruv aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK check_dependencies.py: QmY8EiWapBspyR3zKGEY3TnQayz7hF7DxZ2d9zozVfFhtA - connection.py: QmVf4XX25S8FQHcZca6o1xC77Ez9dUNp66K3dGnVB724GC + connection.py: QmdcL8EQ49Ecx1imT8XoRi1RDGXqaFMNxUCfHsNL4X78dd dht/dhtclient/dhtclient.go: QmZCgMowWw4ueCSsyYY12om3RMQA3Ydp98ti65Eqa5ccuz dht/dhtclient/dhtclient_test.go: QmbTvSmNU1uqjLQo9LdD7zWmDcLuzEacDeg7AG4VXhmriV dht/dhtclient/options.go: QmWs7fQqHRavjwe1PjihHFi85HeJu9qSEVyMdcRviZzaT2 diff --git a/packages/hashes.csv b/packages/hashes.csv index 67efe1a161..f741911f16 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,7 +34,7 @@ fetchai/connections/http_server,QmP3nhFMyEi7N64oLV5EQDNowhS8be3jVefxkwir4PKpQ9 fetchai/connections/ledger,QmaViv1ZK4jLgQXycpYLjboFFcznN9zXUZT4JKdbeS2wQ8 fetchai/connections/local,QmNPNVkqtDtzENpv1XqM4LHZna6v6jJ7Hsk6RB14iSdtjG fetchai/connections/oef,QmVGcKDeDMEhtcBnDNVTWchHkA2YhHnDGoK8izobnDQKmw -fetchai/connections/p2p_libp2p,QmVjnho2ehRqLmYMYyDSkBg3zdWAP85AatmgzP3ryEFKa4 +fetchai/connections/p2p_libp2p,QmRpTgnHJftmWLGgaa9raUzWm12gTFKyNdQVCdggU6PkiF fetchai/connections/p2p_libp2p_client,QmPV4eyGijvdqUzMAfgBrTDnp5rhJhSH4YbyyZXmnar8hk fetchai/connections/p2p_stub,QmaHtQs9dJRnF27WDZSVW3FFEGbY1419NH8u67B8hgnteV fetchai/connections/scaffold,QmcjctNNxnFZ6xU2aDs7WvzNHMvvv4PuhnS739yWxHmJKz diff --git a/tests/conftest.py b/tests/conftest.py index 03e7832317..06941306ee 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -773,7 +773,7 @@ def _make_stub_connection(input_file_path: str, output_file_path: str): return connection -def _make_libp2p_connection( +def _make_libp2p_connection_to_restore( port: int = 10234, host: str = "127.0.0.1", relay: bool = True, @@ -821,6 +821,65 @@ def _make_libp2p_connection( return P2PLibp2pConnection(configuration=configuration, identity=identity) +def _make_libp2p_connection( + port: int = 10234, + host: str = "127.0.0.1", + relay: bool = True, + delegate: bool = False, + entry_peers: Optional[Sequence[MultiAddr]] = None, + delegate_port: int = 11234, + delegate_host: str = "127.0.0.1", + node_key_file: Optional[str] = None, + agent_address: Optional[Address] = None, +) -> P2PLibp2pConnection: + log_file = "libp2p_node_{}.log".format(port) + if os.path.exists(log_file): + os.remove(log_file) + + shared_key = make_crypto(DEFAULT_LEDGER) + shared_key_path = f"./{shared_key.public_key}.txt" + with open(shared_key_path, "wb") as f: + shared_key.dump(f) + + address = agent_address + if address is None: + address = shared_key.address + identity = Identity("", address=address) + + key_file = node_key_file + if key_file is None: + key_file = shared_key_path + + if relay and delegate: + configuration = ConnectionConfig( + node_key_file=key_file, + local_uri="{}:{}".format(host, port), + public_uri="{}:{}".format(host, port), + entry_peers=entry_peers, + log_file=log_file, + delegate_uri="{}:{}".format(delegate_host, delegate_port), + connection_id=P2PLibp2pConnection.connection_id, + ) + elif relay and not delegate: + configuration = ConnectionConfig( + node_key_file=key_file, + local_uri="{}:{}".format(host, port), + public_uri="{}:{}".format(host, port), + entry_peers=entry_peers, + log_file=log_file, + connection_id=P2PLibp2pConnection.connection_id, + ) + else: + configuration = ConnectionConfig( + node_key_file=key_file, + local_uri="{}:{}".format(host, port), + entry_peers=entry_peers, + log_file=log_file, + connection_id=P2PLibp2pConnection.connection_id, + ) + return P2PLibp2pConnection(configuration=configuration, identity=identity) + + def _make_libp2p_client_connection( node_port: int = 11234, node_host: str = "127.0.0.1", uri: Optional[str] = None, ) -> P2PLibp2pClientConnection: From 2d27d824aecb1bd995b0fd0b331821f2d05869a6 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Thu, 17 Dec 2020 11:31:37 +0000 Subject: [PATCH 010/204] Add PoR validation to AgentRecord python class --- .../connections/p2p_libp2p/connection.py | 42 +++++++++++++++---- .../connections/p2p_libp2p/connection.yaml | 2 +- packages/hashes.csv | 2 +- tests/conftest.py | 9 ++++ 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index 7ac0bdcdd7..f7123d817b 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -18,6 +18,7 @@ # ------------------------------------------------------------------------------ """This module contains the p2p libp2p connection.""" import asyncio +import base64 import logging import os import platform @@ -25,17 +26,19 @@ import subprocess # nosec import tempfile from asyncio import AbstractEventLoop, CancelledError +from ecdsa import SECP256k1, util as ecdsa_util, VerifyingKey +from hashlib import sha256 from ipaddress import ip_address from pathlib import Path from random import randint from socket import gethostbyname from typing import IO, List, Optional, Sequence, cast -from aea.common import Address from aea.configurations.base import PublicId from aea.configurations.constants import DEFAULT_LEDGER from aea.connections.base import Connection, ConnectionStates from aea.crypto.base import Crypto +from aea.crypto.fetchai import FetchAIHelper from aea.crypto.registries import make_crypto from aea.exceptions import enforce from aea.helpers.multiaddr.base import MultiAddr @@ -170,6 +173,24 @@ def port(self) -> int: return self._port +def verify_message(msg: bytes, signature: str, public_key: str) -> bool: + """ + Verifies signature generate by Crypto.sign_message() + + :param msg: original bytes + :param signature: signature previousely generated by sign_message() + :param public_key: verifying key + :return: True if signature is valid + """ + key = VerifyingKey.from_string( + bytes.fromhex(public_key), curve=SECP256k1, hashfunc=sha256 + ) + signature_bytes = base64.b64decode(signature) + return key.verify( + signature_bytes, msg, hashfunc=sha256, sigdecode=ecdsa_util.sigdecode_string + ) + + class AgentRecord: """Agent Proof-of-Representation to peer""" @@ -212,18 +233,25 @@ def __str__(self): """Get string representation.""" return f"(address={self.address}, pubkey={self.public_key}, peer_pubkey={self.peer_public_key}, signature={self.signature})" - def is_valid_for(self, addr: str, public_key: str): + def is_valid_for(self, address: str, peer_public_key: str) -> bool: """ - Check if the agent record is valid + Check if the agent record is valid for `address` and `peer_public_key` + :param address: the expected agent address concerned by the record + :param peer_public_key: the expected representative peer public key + :return: True if record is valid """ - if self._address != addr: + if self._address != address: + return False + if self._peer_public_key != peer_public_key: + return False + if self._address != FetchAIHelper.get_address_from_public_key(self._public_key): return False - if self._peer_public_key != public_key: + if not verify_message( + self._peer_public_key.encode("utf-8"), self._signature, self._public_key + ): return False - # TODO(LR) check if agent address and public key match - # TODO(LR) check if signature is valid return True diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index 978ae82bda..9d23bed4a5 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -16,7 +16,7 @@ fingerprint: aea/pipe_unix.go: QmSzbAexrSUSA9ZE6VT6MmcWF5MhEcmceQjAbnfRoK7Ruv aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK check_dependencies.py: QmYDCgddhBGunicdN5z1397LC53sKv7oFjTuqwz5V8WLT6 - connection.py: QmcQysNWgGmPJSztZ1qVog9ssLjjsp1yMAy4ubWejGGJPx + connection.py: QmRky1tHapy2XQH5dQP81eFhCKvjHq6QzkcEGZHjAhGaaE dht/dhtclient/dhtclient.go: QmZCgMowWw4ueCSsyYY12om3RMQA3Ydp98ti65Eqa5ccuz dht/dhtclient/dhtclient_test.go: QmbTvSmNU1uqjLQo9LdD7zWmDcLuzEacDeg7AG4VXhmriV dht/dhtclient/options.go: QmWs7fQqHRavjwe1PjihHFi85HeJu9qSEVyMdcRviZzaT2 diff --git a/packages/hashes.csv b/packages/hashes.csv index b4aa0855df..95d17daff9 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,7 +34,7 @@ fetchai/connections/http_server,QmQ8hNfEts6th9zfnocYHzyYw4dXkDNDj8zoKtov1wB49n fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 -fetchai/connections/p2p_libp2p,QmdkA2iF2KT2PfcC6ivuQkNLckKynfmVsNvfaxdD58SJG6 +fetchai/connections/p2p_libp2p,QmWm4rMxh3CeogLdwsqrUUsnQdP2fBp7gDEwb66MUofXon fetchai/connections/p2p_libp2p_client,QmQndwAPhGHTpe1J8c6Q6jabV2KfkND7jsYiEF9Ysmyby3 fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T diff --git a/tests/conftest.py b/tests/conftest.py index d47bf3fd8c..11f5d754c3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -841,6 +841,7 @@ def _make_libp2p_connection( delegate_host: str = "127.0.0.1", node_key_file: Optional[str] = None, agent_address: Optional[Address] = None, + build_directory: Optional[str] = None, ) -> P2PLibp2pConnection: log_file = "libp2p_node_{}.log".format(port) if os.path.exists(log_file): @@ -860,6 +861,9 @@ def _make_libp2p_connection( if key_file is None: key_file = shared_key_path + if not build_directory: + build_directory = os.getcwd() + if relay and delegate: configuration = ConnectionConfig( node_key_file=key_file, @@ -869,6 +873,7 @@ def _make_libp2p_connection( log_file=log_file, delegate_uri="{}:{}".format(delegate_host, delegate_port), connection_id=P2PLibp2pConnection.connection_id, + build_directory=build_directory, ) elif relay and not delegate: configuration = ConnectionConfig( @@ -878,6 +883,7 @@ def _make_libp2p_connection( entry_peers=entry_peers, log_file=log_file, connection_id=P2PLibp2pConnection.connection_id, + build_directory=build_directory, ) else: configuration = ConnectionConfig( @@ -886,7 +892,10 @@ def _make_libp2p_connection( entry_peers=entry_peers, log_file=log_file, connection_id=P2PLibp2pConnection.connection_id, + build_directory=build_directory, ) + if not os.path.exists(os.path.join(build_directory, LIBP2P_NODE_MODULE_NAME)): + build_node(build_directory) return P2PLibp2pConnection(configuration=configuration, identity=identity) From 0af4f85fd35db314e0ba34a139e879f335a71769 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 17 Dec 2020 11:49:38 +0000 Subject: [PATCH 011/204] post release fixes --- SECURITY.md | 4 ++-- aea/cli/scaffold.py | 2 +- scripts/RELEASE_PROCESS.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index a334d82966..8d1e34bedc 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,8 +8,8 @@ The following table shows which versions of `aea` are currently being supported | Version | Supported | | ------- | ------------------ | -| 0.7.x | :white_check_mark: | -| < 0.7.0 | :x: | +| 0.8.x | :white_check_mark: | +| < 0.8.0 | :x: | ## Reporting a Vulnerability diff --git a/aea/cli/scaffold.py b/aea/cli/scaffold.py index 449bd675be..c61ed0bebd 100644 --- a/aea/cli/scaffold.py +++ b/aea/cli/scaffold.py @@ -250,7 +250,7 @@ def _scaffold_non_package_item( try: # copy the item package into the agent project. src = Path(os.path.join(AEA_DIR, aea_dir, "scaffold.py")) - logger.debug(f"Copying error handler. src={src} dst={dest}") + logger.debug(f"Copying {type_name}. src={src} dst={dest}") shutil.copyfile(src, dest) # add the item to the configurations. diff --git a/scripts/RELEASE_PROCESS.md b/scripts/RELEASE_PROCESS.md index 9428c713eb..7adfce0ad1 100644 --- a/scripts/RELEASE_PROCESS.md +++ b/scripts/RELEASE_PROCESS.md @@ -13,7 +13,7 @@ 6. Check the docs are up-to-date by running `python scripts/generate_api_docs.py` and `python scripts/check_doc_links.py`. Ensure all links are configured `mkdocs serve`. Commit if satisfied. -7. Write release notes and place them in `HISTORY.md`. Add upgrading tips in `upgrading.md`. Commit if satisfied. +7. Write release notes and place them in `HISTORY.md`. Add upgrading tips in `upgrading.md`. If necessary, adjust version references in `SECURITY.md`. Commit if satisfied. 8. Open PRs and merge into master. @@ -28,6 +28,6 @@ 13. Release packages into registry: `python scripts/deploy_to_registry.py`. -14. If necessary, adjust version references in `SECURITY.md`. Commit if satisfied. +14. Create and push Docker images `user-image` and `deploy-image`. If something goes wrong and only needs a small fix do `LAST_VERSION.post1` as version, apply fixes, push again to PyPI. From 7b9969d0abb6b336ec21a8a0e82f426ef3424b08 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Thu, 17 Dec 2020 15:44:47 +0300 Subject: [PATCH 012/204] build command fix --- aea/aea_builder.py | 8 +------- .../p2p_libp2p/check_dependencies.py | 19 ++++++++++++------- .../connections/p2p_libp2p/connection.yaml | 2 +- packages/hashes.csv | 2 +- scripts/whitelist.py | 1 + 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/aea/aea_builder.py b/aea/aea_builder.py index 18362a51ba..9a86db0ff0 100644 --- a/aea/aea_builder.py +++ b/aea/aea_builder.py @@ -78,12 +78,7 @@ from aea.decision_maker.base import DecisionMakerHandler from aea.error_handler.base import AbstractErrorHandler from aea.exceptions import AEAException, AEAValidationError, enforce -from aea.helpers.base import ( - ensure_dir, - find_topological_order, - load_env_file, - load_module, -) +from aea.helpers.base import find_topological_order, load_env_file, load_module from aea.helpers.exception_policy import ExceptionPolicyEnum from aea.helpers.install_dependency import install_dependency from aea.helpers.logging import AgentLoggerAdapter, WithLogger, get_logger @@ -780,7 +775,6 @@ def _set_component_build_directory( configuration.author, configuration.name, ) - ensure_dir(configuration.build_directory) def add_component_instance(self, component: Component) -> "AEABuilder": """ diff --git a/packages/fetchai/connections/p2p_libp2p/check_dependencies.py b/packages/fetchai/connections/p2p_libp2p/check_dependencies.py index d02789d6d1..f4debc3180 100644 --- a/packages/fetchai/connections/p2p_libp2p/check_dependencies.py +++ b/packages/fetchai/connections/p2p_libp2p/check_dependencies.py @@ -25,6 +25,7 @@ import shutil import subprocess # nosec import sys +import tempfile from distutils.dir_util import copy_tree from itertools import islice from subprocess import Popen, TimeoutExpired # nosec @@ -182,7 +183,6 @@ def main(): # pragma: nocover if len(sys.argv) < 2: raise ValueError("Please provide build directory path as an argument!") build_dir = sys.argv[1] - ensure_dir(build_dir) check_versions() build_node(build_dir) @@ -217,12 +217,17 @@ def _golang_module_build( def build_node(build_dir: str) -> None: """Build node placed inside build_dir.""" - copy_tree(LIBP2P_NODE_MODULE, build_dir) - - err_str = _golang_module_build(build_dir) - if err_str: # pragma: nocover - raise Exception(f"Node build failed: {err_str}") - print("libp2p_node built successfully!") + with tempfile.TemporaryDirectory() as dirname: + copy_tree(LIBP2P_NODE_MODULE, dirname) + err_str = _golang_module_build(dirname) + if err_str: # pragma: nocover + raise Exception(f"Node build failed: {err_str}") + ensure_dir(build_dir) + shutil.copy( + os.path.join(dirname, LIBP2P_NODE_MODULE_NAME), + os.path.join(build_dir, LIBP2P_NODE_MODULE_NAME), + ) + print(f"{LIBP2P_NODE_MODULE_NAME} built successfully!") if __name__ == "__main__": diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index a26cf1e495..6c4b4e5c36 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -15,7 +15,7 @@ fingerprint: aea/envelope.proto: QmSC8EGCKiNFR2vf5bSWymSzYDFMipQW9aQVMwPzQoKb4n aea/pipe_unix.go: QmSzbAexrSUSA9ZE6VT6MmcWF5MhEcmceQjAbnfRoK7Ruv aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK - check_dependencies.py: QmYDCgddhBGunicdN5z1397LC53sKv7oFjTuqwz5V8WLT6 + check_dependencies.py: QmcZATMqEThMQBdsud1AwJEYtKq86VfPSXhnfdE7TSWcsc connection.py: QmP53cuL1Z9h51Ji34kPQi8S41qGdC2mYe5JdCT3pSJYuG dht/dhtclient/dhtclient.go: QmeHnMoBdff59vSqRRsYUGGJPUMi5Euk8pKiKa5jTzHHQD dht/dhtclient/dhtclient_test.go: QmPfnHSHXtbaW5VYuq1QsKQWey64pUEvLEaKKkT9eAcmws diff --git a/packages/hashes.csv b/packages/hashes.csv index 02ba043792..52aff649a2 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,7 +34,7 @@ fetchai/connections/http_server,QmQ8hNfEts6th9zfnocYHzyYw4dXkDNDj8zoKtov1wB49n fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 -fetchai/connections/p2p_libp2p,QmWmjTisTaNBjdriV3D6oxaVBqw4NVaNWTtrpsKX5A2bnF +fetchai/connections/p2p_libp2p,QmV4fWq2ePnDP7r4JmSb51nUNHaEj9zMvg15nsx1FD2GHX fetchai/connections/p2p_libp2p_client,QmQndwAPhGHTpe1J8c6Q6jabV2KfkND7jsYiEF9Ysmyby3 fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T diff --git a/scripts/whitelist.py b/scripts/whitelist.py index 4df86b3388..a82f4c269e 100644 --- a/scripts/whitelist.py +++ b/scripts/whitelist.py @@ -236,3 +236,4 @@ _.unsupported_skill_count _.decoding_error_count _.ErrorHandler # unused class (aea/error_handler/scaffold.py:27) +ensure_dir # unused function (aea/helpers/base.py:561) From 689674936ec5849483acff271c827ff1e86f360d Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 17 Dec 2020 14:15:39 +0000 Subject: [PATCH 013/204] Improve verbosity of the build command. --- aea/aea_builder.py | 25 ++++++++++++------- .../connections/p2p_libp2p/connection.py | 2 +- .../connections/p2p_libp2p/connection.yaml | 2 +- packages/hashes.csv | 2 +- scripts/acn/k8s_deploy_acn_node.py | 6 ++--- tests/test_aea_builder.py | 2 +- .../test_simple_data_request/test_handlers.py | 2 +- .../test_generator/test_validate.py | 10 ++++---- 8 files changed, 29 insertions(+), 22 deletions(-) diff --git a/aea/aea_builder.py b/aea/aea_builder.py index 18362a51ba..7f6d1422cd 100644 --- a/aea/aea_builder.py +++ b/aea/aea_builder.py @@ -22,11 +22,11 @@ import logging.config import os import pprint +import subprocess # nosec import sys from collections import defaultdict from copy import copy, deepcopy from pathlib import Path -from subprocess import check_call # nosec from typing import Any, Collection, Dict, List, Optional, Set, Tuple, Type, Union, cast import jsonschema @@ -962,15 +962,22 @@ def _run_build_entrypoint( command = [sys.executable, build_entrypoint, target_directory] command_str = " ".join(command) if logger: - logger.info(f"Running command '{command_str}'") - try: - check_call( - command, cwd=source_directory, timeout=cls.BUILD_TIMEOUT - ) # nosec - except Exception as e: + logger.info(f"Running command '{command_str}'...") + res = subprocess.run( # nosec + command, + cwd=source_directory, + check=False, + timeout=cls.BUILD_TIMEOUT, + capture_output=True, + ) + if res.returncode == 0: + if logger: + logger.info(f"Command '{command_str}' succeded!") + else: + e = res.stderr.decode("utf-8") raise AEAException( - f"An error occurred while running command '{command_str}': {str(e)}" - ) from e + f"An error occurred while running command '{command_str}':\n{e}" + ) def _build_identity_from_wallet(self, wallet: Wallet) -> Identity: """ diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index 503ce81374..31be8754d1 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -602,7 +602,7 @@ def _check_node_built(self) -> None: ) enforce( os.path.exists(libp2p_node_module_path), - f"Module {LIBP2P_NODE_MODULE_NAME} does not present in {self.configuration.build_directory}, please call `aea build` command", + f"Module {LIBP2P_NODE_MODULE_NAME} is not present in {self.configuration.build_directory}, please call the `aea build` command first!", ) shutil.copy(libp2p_node_module_path, self.libp2p_workdir) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index a26cf1e495..08b45dc1e5 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -16,7 +16,7 @@ fingerprint: aea/pipe_unix.go: QmSzbAexrSUSA9ZE6VT6MmcWF5MhEcmceQjAbnfRoK7Ruv aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK check_dependencies.py: QmYDCgddhBGunicdN5z1397LC53sKv7oFjTuqwz5V8WLT6 - connection.py: QmP53cuL1Z9h51Ji34kPQi8S41qGdC2mYe5JdCT3pSJYuG + connection.py: Qmf3epj1Fdo51qGtKUjpesHDiwR1YZE39kZ6VZbUf4dQ7u dht/dhtclient/dhtclient.go: QmeHnMoBdff59vSqRRsYUGGJPUMi5Euk8pKiKa5jTzHHQD dht/dhtclient/dhtclient_test.go: QmPfnHSHXtbaW5VYuq1QsKQWey64pUEvLEaKKkT9eAcmws dht/dhtclient/options.go: QmPorj38wNrxGrzsbFe5wwLmiHzxbTJ2VsgvSd8tLDYS8s diff --git a/packages/hashes.csv b/packages/hashes.csv index 02ba043792..37912abbc7 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,7 +34,7 @@ fetchai/connections/http_server,QmQ8hNfEts6th9zfnocYHzyYw4dXkDNDj8zoKtov1wB49n fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 -fetchai/connections/p2p_libp2p,QmWmjTisTaNBjdriV3D6oxaVBqw4NVaNWTtrpsKX5A2bnF +fetchai/connections/p2p_libp2p,QmWvL3H1QZQebKgeNf48gdNcSXiLhhcChhJkgRTWj63jq7 fetchai/connections/p2p_libp2p_client,QmQndwAPhGHTpe1J8c6Q6jabV2KfkND7jsYiEF9Ysmyby3 fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T diff --git a/scripts/acn/k8s_deploy_acn_node.py b/scripts/acn/k8s_deploy_acn_node.py index e9b823b741..9413da78da 100644 --- a/scripts/acn/k8s_deploy_acn_node.py +++ b/scripts/acn/k8s_deploy_acn_node.py @@ -269,9 +269,9 @@ def __init__( config[cls.NODE_LAST_ENTRY_PEER_HOST] = '"{}"'.format(peer_host) config[cls.NODE_LAST_ENTRY_PEER_PORT] = '"{}"'.format(peer_port) - config[cls.NODE_URI] = "127.0.0.1:9000".format(acn_port) - config[cls.NODE_URI_DELEGATE] = "127.0.0.1:11000".format(acn_delegate_port) - config[cls.NODE_URI_MONITORING] = "127.0.0.1:8080".format(acn_monitoring_port) + config[cls.NODE_URI] = "127.0.0.1:9000" + config[cls.NODE_URI_DELEGATE] = "127.0.0.1:11000" + config[cls.NODE_URI_MONITORING] = "127.0.0.1:8080" config[cls.NODE_URI_EXTERNAL] = "{}:{}".format(k8s_public_hostname, acn_port) config[cls.NODE_LOG_FILE] = '"{}_{}.log"'.format( cls.Defaults[cls.NODE_LOG_FILE], str(acn_port) diff --git a/tests/test_aea_builder.py b/tests/test_aea_builder.py index c3750c8335..10f161413c 100644 --- a/tests/test_aea_builder.py +++ b/tests/test_aea_builder.py @@ -842,7 +842,7 @@ def test_build_negative_syntax_error(self): ) def test_build_negative_subprocess(self, *_mocks): """Test build, negative due to script error at runtime.""" - match = f"An error occurred while running command '.*script.py .+': some error." + match = "An error occurred while running command '.*script.py .+': some error." with cd(self._get_cwd()), pytest.raises(AEAException, match=match): self.script_path.write_text("") self.builder.call_all_build_entrypoints() diff --git a/tests/test_packages/test_skills/test_simple_data_request/test_handlers.py b/tests/test_packages/test_skills/test_simple_data_request/test_handlers.py index f1e325933c..a9abc4a7aa 100644 --- a/tests/test_packages/test_skills/test_simple_data_request/test_handlers.py +++ b/tests/test_packages/test_skills/test_simple_data_request/test_handlers.py @@ -126,7 +126,7 @@ def test_handle_response(self): ) mock_logger.assert_any_call( - logging.INFO, f"updating shared_state with received data=b'some_body'!", + logging.INFO, "updating shared_state with received data=b'some_body'!", ) assert ( diff --git a/tests/test_protocols/test_generator/test_validate.py b/tests/test_protocols/test_generator/test_validate.py index 9167629b40..d7f6f77c0f 100644 --- a/tests/test_protocols/test_generator/test_validate.py +++ b/tests/test_protocols/test_generator/test_validate.py @@ -1157,7 +1157,7 @@ def test_validate_speech_acts_section(self, mocked_spec): assert invalid_result_6 is False assert ( invalid_msg_6 - == f"Content 'content_name_1' with type 'pt:float' under performative 'perm_2' is already defined under performative 'perm_1' with a different type ('pt:int')." + == "Content 'content_name_1' with type 'pt:float' under performative 'perm_2' is already defined under performative 'perm_1' with a different type ('pt:int')." ) assert invalid_all_per_6 is None assert invalid_all_content_6 is None @@ -1449,7 +1449,7 @@ def test_validate_reply(self): assert invalid_result_5 is False assert ( invalid_msg_5 - == f"Performative 'perm_5' specified in \"reply\" is not defined in the protocol's speech-acts." + == "Performative 'perm_5' specified in \"reply\" is not defined in the protocol's speech-acts." ) assert invalid_terminal_performatives_from_reply_5 is None @@ -1529,7 +1529,7 @@ def test_validate_termination(self): assert invalid_result_5 is False assert ( invalid_msg_5 - == f'The terminal performative \'perm_3\' specified in "termination" is assigned replies in "reply".' + == 'The terminal performative \'perm_3\' specified in "termination" is assigned replies in "reply".' ) invalid_termination_6 = ["perm_4"] @@ -1542,7 +1542,7 @@ def test_validate_termination(self): assert invalid_result_6 is False assert ( invalid_msg_6 - == f"The performative 'perm_3' has no replies but is not listed as a terminal performative in \"termination\"." + == "The performative 'perm_3' has no replies but is not listed as a terminal performative in \"termination\"." ) def test_validate_roles(self): @@ -1796,7 +1796,7 @@ def test_validate_dialogue_section(self, mocked_spec): assert invalid_result_6 is False assert ( invalid_msg_6 - == f"Missing required field 'termination' in the dialogue section of the protocol specification." + == "Missing required field 'termination' in the dialogue section of the protocol specification." ) invalid_value = 521 From 543222110d24b2b69402dc802cc172d4d0015b1d Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 17 Dec 2020 14:27:20 +0000 Subject: [PATCH 014/204] add build commands in docs --- docs/aries-cloud-agent-demo.md | 2 ++ docs/car-park-skills.md | 4 ++++ docs/erc1155-skills.md | 4 ++++ docs/generic-skills-step-by-step.md | 2 ++ docs/generic-skills.md | 4 ++++ docs/ml-skills.md | 4 ++++ docs/oracle-demo.md | 1 + docs/orm-integration.md | 4 ++++ docs/prometheus.md | 1 + docs/skill-guide.md | 1 + docs/tac-skills-contract.md | 8 +++++++- docs/tac-skills.md | 10 +++++++++- docs/thermometer-skills.md | 4 ++++ docs/weather-skills.md | 4 ++++ 14 files changed, 51 insertions(+), 2 deletions(-) diff --git a/docs/aries-cloud-agent-demo.md b/docs/aries-cloud-agent-demo.md index 19e472a8b3..a49141fae4 100644 --- a/docs/aries-cloud-agent-demo.md +++ b/docs/aries-cloud-agent-demo.md @@ -250,6 +250,7 @@ Now install all the dependencies: ``` bash aea install +aea build ``` Finally run **Alice_AEA**: @@ -338,6 +339,7 @@ Now install all the dependencies: ``` bash aea install +aea build ``` Finally run **Faber_AEA**: diff --git a/docs/car-park-skills.md b/docs/car-park-skills.md index 1c5116918f..437e8131ef 100644 --- a/docs/car-park-skills.md +++ b/docs/car-park-skills.md @@ -58,6 +58,7 @@ First, fetch the car detector AEA: aea fetch fetchai/car_detector:0.19.0 cd car_detector aea install +aea build ```
Alternatively, create from scratch. @@ -72,6 +73,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/carpark_detection:0.17.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` @@ -92,6 +94,7 @@ Then, fetch the car data client AEA: aea fetch fetchai/car_data_buyer:0.20.0 cd car_data_buyer aea install +aea build ```
Alternatively, create from scratch. @@ -106,6 +109,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/carpark_client:0.18.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` diff --git a/docs/erc1155-skills.md b/docs/erc1155-skills.md index 9fd1865d4b..cca131aafd 100644 --- a/docs/erc1155-skills.md +++ b/docs/erc1155-skills.md @@ -29,6 +29,7 @@ Fetch the AEA that will deploy the contract. aea fetch fetchai/erc1155_deployer:0.20.0 cd erc1155_deployer aea install +aea build ```
Alternatively, create from scratch. @@ -44,6 +45,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/erc1155_deploy:0.19.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` @@ -94,6 +96,7 @@ In another terminal, fetch the AEA that will get some tokens from the deployer. aea fetch fetchai/erc1155_client:0.20.0 cd erc1155_client aea install +aea build ```
Alternatively, create from scratch. @@ -109,6 +112,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/erc1155_client:0.18.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` diff --git a/docs/generic-skills-step-by-step.md b/docs/generic-skills-step-by-step.md index 2babe8fc18..4a08bce460 100644 --- a/docs/generic-skills-step-by-step.md +++ b/docs/generic-skills-step-by-step.md @@ -3203,6 +3203,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add protocol fetchai/fipa:0.11.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea run ``` @@ -3220,6 +3221,7 @@ aea add connection fetchai/ledger:0.11.0 aea add protocol fetchai/fipa:0.11.0 aea add protocol fetchai/signing:0.8.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` diff --git a/docs/generic-skills.md b/docs/generic-skills.md index 7378ad4b3e..b0e25f9f72 100644 --- a/docs/generic-skills.md +++ b/docs/generic-skills.md @@ -62,6 +62,7 @@ First, fetch the seller AEA: aea fetch fetchai/generic_seller:0.16.0 --alias my_seller_aea cd my_seller_aea aea install +aea build ```
Alternatively, create from scratch. @@ -76,6 +77,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/generic_seller:0.18.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` @@ -96,6 +98,7 @@ Then, fetch the buyer AEA: aea fetch fetchai/generic_buyer:0.17.0 --alias my_buyer_aea cd my_buyer_aea aea install +aea build ```
Alternatively, create from scratch. @@ -110,6 +113,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/generic_buyer:0.18.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` diff --git a/docs/ml-skills.md b/docs/ml-skills.md index ce3d5726d3..f142e9fa9e 100644 --- a/docs/ml-skills.md +++ b/docs/ml-skills.md @@ -65,6 +65,7 @@ First, fetch the data provider AEA: aea fetch fetchai/ml_data_provider:0.19.0 cd ml_data_provider aea install +aea build ```
Alternatively, create from scratch. @@ -80,6 +81,7 @@ aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/ml_data_provider:0.17.0 aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea install +aea build ``` In `ml_data_provider/aea-config.yaml` add @@ -99,6 +101,7 @@ Then, fetch the model trainer AEA: aea fetch fetchai/ml_model_trainer:0.20.0 cd ml_model_trainer aea install +aea build ```
Alternatively, create from scratch. @@ -114,6 +117,7 @@ aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/ml_train:0.18.0 aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea install +aea build ``` In `ml_model_trainer/aea-config.yaml` add diff --git a/docs/oracle-demo.md b/docs/oracle-demo.md index 2178cf00d0..f5ee2bd059 100644 --- a/docs/oracle-demo.md +++ b/docs/oracle-demo.md @@ -22,6 +22,7 @@ Fetch the AEA that will deploy and update the oracle contract. aea fetch fetchai/coin_price_oracle:0.2.0 cd coin_price_oracle aea install +aea build ```
Alternatively, create from scratch. diff --git a/docs/orm-integration.md b/docs/orm-integration.md index 7db0e0f5aa..dbd8e92227 100644 --- a/docs/orm-integration.md +++ b/docs/orm-integration.md @@ -62,6 +62,7 @@ First, fetch the seller AEA, which will provide data: aea fetch fetchai/thermometer_aea:0.17.0 --alias my_thermometer_aea cd my_thermometer_aea aea install +aea build ```
Alternatively, create from scratch. @@ -76,6 +77,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/thermometer:0.17.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` @@ -97,6 +99,7 @@ In another terminal, fetch the AEA that will query the seller AEA. aea fetch fetchai/thermometer_client:0.18.0 --alias my_thermometer_client cd my_thermometer_client aea install +aea build ```
Alternatively, create from scratch. @@ -111,6 +114,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/thermometer_client:0.17.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` diff --git a/docs/prometheus.md b/docs/prometheus.md index 567b0a0ecc..cdfac52b15 100644 --- a/docs/prometheus.md +++ b/docs/prometheus.md @@ -5,6 +5,7 @@ To see this working in an agent, fetch and run the `coin_price_feed` agent and c aea fetch fetchai/coin_price_feed:0.2.0 cd coin_price_feed aea install +aea build aea run ``` You can then instruct a prometheus server running on the same computing cluster as a deployed agent to scrape these metrics for remote monitoring and visualisation with the Prometheus/Grafana toolset. diff --git a/docs/skill-guide.md b/docs/skill-guide.md index 566e3984b8..955fc1e912 100644 --- a/docs/skill-guide.md +++ b/docs/skill-guide.md @@ -415,6 +415,7 @@ We also need to add the soef and p2p connections and install the AEA's dependenc aea add connection fetchai/soef:0.14.0 aea add connection fetchai/p2p_libp2p:0.13.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` diff --git a/docs/tac-skills-contract.md b/docs/tac-skills-contract.md index 5bea5dcca3..95cd24be01 100644 --- a/docs/tac-skills-contract.md +++ b/docs/tac-skills-contract.md @@ -104,6 +104,7 @@ In the root directory, fetch the controller AEA: aea fetch fetchai/tac_controller_contract:0.18.0 cd tac_controller_contract aea install +aea build ```
Alternatively, create from scratch. @@ -118,6 +119,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_control_contract:0.15.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger ethereum ``` @@ -139,6 +141,8 @@ aea fetch fetchai/tac_participant:0.18.0 --alias tac_participant_one cd tac_participant_one aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using_contract 'True' --type bool aea config set vendor.fetchai.skills.tac_negotiation.models.strategy.args.is_contract_tx 'True' --type bool +aea install +aea build cd .. aea fetch fetchai/tac_participant:0.18.0 --alias tac_participant_two cd tac_participant_two @@ -146,7 +150,7 @@ aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using aea config set vendor.fetchai.skills.tac_negotiation.models.strategy.args.is_contract_tx 'True' --type bool aea generate-key ethereum aea add-key ethereum ethereum_private_key.txt -aea install +aea build ```
Alternatively, create from scratch. @@ -169,6 +173,7 @@ aea add skill fetchai/tac_negotiation:0.16.0 aea generate-key ethereum aea add-key ethereum ethereum_private_key.txt aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger ethereum aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using_contract 'True' --type bool @@ -186,6 +191,7 @@ aea add skill fetchai/tac_negotiation:0.16.0 aea generate-key ethereum aea add-key ethereum ethereum_private_key.txt aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger ethereum aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using_contract 'True' --type bool diff --git a/docs/tac-skills.md b/docs/tac-skills.md index 56551b524b..8d385fba6a 100644 --- a/docs/tac-skills.md +++ b/docs/tac-skills.md @@ -103,6 +103,7 @@ In the root directory, fetch the controller AEA: aea fetch fetchai/tac_controller:0.16.0 cd tac_controller aea install +aea build ```
Alternatively, create from scratch. @@ -117,6 +118,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_control:0.13.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger fetchai ``` @@ -135,9 +137,13 @@ default_routing: In a separate terminal, in the root directory, fetch at least two participants: ``` bash aea fetch fetchai/tac_participant:0.18.0 --alias tac_participant_one +cd tac_participant_one +aea install +aea build +cd .. aea fetch fetchai/tac_participant:0.18.0 --alias tac_participant_two cd tac_participant_two -aea install +aea build ```
Alternatively, create from scratch. @@ -158,6 +164,7 @@ aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_participation:0.14.0 aea add skill fetchai/tac_negotiation:0.16.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger fetchai ``` @@ -178,6 +185,7 @@ aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_participation:0.14.0 aea add skill fetchai/tac_negotiation:0.16.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger fetchai ``` diff --git a/docs/thermometer-skills.md b/docs/thermometer-skills.md index 57c180de39..420439dcde 100644 --- a/docs/thermometer-skills.md +++ b/docs/thermometer-skills.md @@ -65,6 +65,7 @@ First, fetch the thermometer AEA: aea fetch fetchai/thermometer_aea:0.17.0 --alias my_thermometer_aea cd my_thermometer_aea aea install +aea build ```
Alternatively, create from scratch. @@ -79,6 +80,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/thermometer:0.17.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` @@ -99,6 +101,7 @@ Then, fetch the thermometer client AEA: aea fetch fetchai/thermometer_client:0.18.0 --alias my_thermometer_client cd my_thermometer_client aea install +aea build ```
Alternatively, create from scratch. @@ -113,6 +116,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/thermometer_client:0.17.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` diff --git a/docs/weather-skills.md b/docs/weather-skills.md index a30dde6212..07aae67f6b 100644 --- a/docs/weather-skills.md +++ b/docs/weather-skills.md @@ -64,6 +64,7 @@ First, fetch the AEA that will provide weather measurements: aea fetch fetchai/weather_station:0.19.0 --alias my_weather_station cd my_weather_station aea install +aea build ```
Alternatively, create from scratch. @@ -78,6 +79,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/weather_station:0.17.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` @@ -99,6 +101,7 @@ In another terminal, fetch the AEA that will query the weather station: aea fetch fetchai/weather_client:0.20.0 --alias my_weather_client cd my_weather_client aea install +aea build ```
Alternatively, create from scratch. @@ -113,6 +116,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/weather_client:0.17.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` From 1bb6e6d52cfc586aa4bab54b93552017cf45fbf7 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Thu, 17 Dec 2020 12:53:33 +0000 Subject: [PATCH 015/204] Add Por support for p2p_libp2p_client --- aea/helpers/acn/__init__.py | 20 + aea/helpers/acn/agent_record.py | 115 +++ aea/helpers/acn/uri.py | 63 ++ .../connections/p2p_libp2p/connection.py | 133 +--- .../connections/p2p_libp2p/connection.yaml | 8 +- .../p2p_libp2p/dht/dhtnode/message.pb.go | 82 +-- .../p2p_libp2p/dht/dhtnode/message.proto | 4 +- .../p2p_libp2p/dht/dhtnode/utils.go | 6 +- .../p2p_libp2p_client/acn_message.proto | 61 ++ .../p2p_libp2p_client/acn_message_pb2.py | 668 ++++++++++++++++++ .../p2p_libp2p_client/connection.py | 104 +-- .../p2p_libp2p_client/connection.yaml | 4 +- packages/hashes.csv | 4 +- tests/conftest.py | 35 +- 14 files changed, 1082 insertions(+), 225 deletions(-) create mode 100644 aea/helpers/acn/__init__.py create mode 100644 aea/helpers/acn/agent_record.py create mode 100644 aea/helpers/acn/uri.py create mode 100644 packages/fetchai/connections/p2p_libp2p_client/acn_message.proto create mode 100644 packages/fetchai/connections/p2p_libp2p_client/acn_message_pb2.py diff --git a/aea/helpers/acn/__init__.py b/aea/helpers/acn/__init__.py new file mode 100644 index 0000000000..18773f4f42 --- /dev/null +++ b/aea/helpers/acn/__init__.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains types and helpers used by libp2p connections.""" diff --git a/aea/helpers/acn/agent_record.py b/aea/helpers/acn/agent_record.py new file mode 100644 index 0000000000..5d92df8608 --- /dev/null +++ b/aea/helpers/acn/agent_record.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains types and helpers for acn Proof-of-Representation.""" + +import base64 +from hashlib import sha256 + +from ecdsa import SECP256k1, VerifyingKey +from ecdsa import util as ecdsa_util + +from aea.crypto.fetchai import FetchAIHelper + + +def verify_message(msg: bytes, signature: str, public_key: str) -> bool: + """ + Verifies signature generate by Crypto.sign_message() + + :param msg: original bytes + :param signature: signature previousely generated by sign_message() + :param public_key: verifying key + :return: True if signature is valid + """ + key = VerifyingKey.from_string( + bytes.fromhex(public_key), curve=SECP256k1, hashfunc=sha256 + ) + signature_bytes = base64.b64decode(signature) + return key.verify( + signature_bytes, msg, hashfunc=sha256, sigdecode=ecdsa_util.sigdecode_string + ) + + +class AgentRecord: + """Agent Proof-of-Representation to peer""" + + def __init__( + self, + address: str, + public_key: str, + peer_public_key: str, + signature: str, + service_id: str, + ): + """Initialize the AgentRecord""" + self._service_id = service_id + self._address = address + self._public_key = public_key + self._peer_public_key = peer_public_key + self._signature = signature + + @property + def address(self) -> str: + """Get agent address""" + return self._address + + @property + def public_key(self) -> str: + """Get agent public key""" + return self._public_key + + @property + def peer_public_key(self) -> str: + """Get agent's representative peer public key""" + return self._peer_public_key + + @property + def signature(self) -> str: + """Get record signature""" + return self._signature + + @property + def service_id(self) -> str: + """Get record service id""" + return self._service_id + + def __str__(self): + """Get string representation.""" + return f"(address={self.address}, pubkey={self.public_key}, peer_pubkey={self.peer_public_key}, signature={self.signature})" + + def is_valid_for(self, address: str, peer_public_key: str) -> bool: + """ + Check if the agent record is valid for `address` and `peer_public_key` + + :param address: the expected agent address concerned by the record + :param peer_public_key: the expected representative peer public key + :return: True if record is valid + """ + + if self._address != address: + return False + if self._peer_public_key != peer_public_key: + return False + if self._address != FetchAIHelper.get_address_from_public_key(self._public_key): + return False + if not verify_message( + self._peer_public_key.encode("utf-8"), self._signature, self._public_key + ): + return False + return True diff --git a/aea/helpers/acn/uri.py b/aea/helpers/acn/uri.py new file mode 100644 index 0000000000..c445e5ba60 --- /dev/null +++ b/aea/helpers/acn/uri.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains types and helpers for libp2p connections Uris.""" + +from random import randint +from typing import Optional + + +class Uri: + """Holds a node address in format "host:port".""" + + def __init__( + self, + uri: Optional[str] = None, + host: Optional[str] = None, + port: Optional[int] = None, + ): + """Initialise Uri.""" + if uri is not None: + split = uri.split(":", 1) + self._host = split[0] + self._port = int(split[1]) + elif host is not None and port is not None: + self._host = host + self._port = port + else: + self._host = "127.0.0.1" + self._port = randint(5000, 10000) # nosec + + def __str__(self): + """Get string representation.""" + return "{}:{}".format(self._host, self._port) + + def __repr__(self): # pragma: no cover + """Get object representation.""" + return self.__str__() + + @property + def host(self) -> str: + """Get host.""" + return self._host + + @property + def port(self) -> int: + """Get port.""" + return self._port diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index f7123d817b..fcdb4aab02 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -18,7 +18,6 @@ # ------------------------------------------------------------------------------ """This module contains the p2p libp2p connection.""" import asyncio -import base64 import logging import os import platform @@ -26,11 +25,8 @@ import subprocess # nosec import tempfile from asyncio import AbstractEventLoop, CancelledError -from ecdsa import SECP256k1, util as ecdsa_util, VerifyingKey -from hashlib import sha256 from ipaddress import ip_address from pathlib import Path -from random import randint from socket import gethostbyname from typing import IO, List, Optional, Sequence, cast @@ -38,9 +34,10 @@ from aea.configurations.constants import DEFAULT_LEDGER from aea.connections.base import Connection, ConnectionStates from aea.crypto.base import Crypto -from aea.crypto.fetchai import FetchAIHelper from aea.crypto.registries import make_crypto from aea.exceptions import enforce +from aea.helpers.acn.agent_record import AgentRecord +from aea.helpers.acn.uri import Uri from aea.helpers.multiaddr.base import MultiAddr from aea.helpers.pipe import IPCChannel, make_ipc_channel from aea.mail.base import Envelope @@ -133,128 +130,6 @@ def _golang_module_run( return proc -class Uri: - """Holds a node address in format "host:port".""" - - def __init__( - self, - uri: Optional[str] = None, - host: Optional[str] = None, - port: Optional[int] = None, - ): - """Initialise Uri.""" - if uri is not None: - split = uri.split(":", 1) - self._host = split[0] - self._port = int(split[1]) - elif host is not None and port is not None: - self._host = host - self._port = port - else: - self._host = "127.0.0.1" - self._port = randint(5000, 10000) # nosec - - def __str__(self): - """Get string representation.""" - return "{}:{}".format(self._host, self._port) - - def __repr__(self): # pragma: no cover - """Get object representation.""" - return self.__str__() - - @property - def host(self) -> str: - """Get host.""" - return self._host - - @property - def port(self) -> int: - """Get port.""" - return self._port - - -def verify_message(msg: bytes, signature: str, public_key: str) -> bool: - """ - Verifies signature generate by Crypto.sign_message() - - :param msg: original bytes - :param signature: signature previousely generated by sign_message() - :param public_key: verifying key - :return: True if signature is valid - """ - key = VerifyingKey.from_string( - bytes.fromhex(public_key), curve=SECP256k1, hashfunc=sha256 - ) - signature_bytes = base64.b64decode(signature) - return key.verify( - signature_bytes, msg, hashfunc=sha256, sigdecode=ecdsa_util.sigdecode_string - ) - - -class AgentRecord: - """Agent Proof-of-Representation to peer""" - - def __init__( - self, address: str, public_key: str, peer_public_key: str, signature: str, - ): - """Initialize the AgentRecord""" - self._service_id = POR_DEFAULT_SERVICE_ID - self._address = address - self._public_key = public_key - self._peer_public_key = peer_public_key - self._signature = signature - - @property - def address(self) -> str: - """Get agent address""" - return self._address - - @property - def public_key(self) -> str: - """Get agent public key""" - return self._public_key - - @property - def peer_public_key(self) -> str: - """Get agent's representative peer public key""" - return self._peer_public_key - - @property - def signature(self) -> str: - """Get record signature""" - return self._signature - - @property - def service_id(self) -> str: - """Get record service id""" - return self._service_id - - def __str__(self): - """Get string representation.""" - return f"(address={self.address}, pubkey={self.public_key}, peer_pubkey={self.peer_public_key}, signature={self.signature})" - - def is_valid_for(self, address: str, peer_public_key: str) -> bool: - """ - Check if the agent record is valid for `address` and `peer_public_key` - - :param address: the expected agent address concerned by the record - :param peer_public_key: the expected representative peer public key - :return: True if record is valid - """ - - if self._address != address: - return False - if self._peer_public_key != peer_public_key: - return False - if self._address != FetchAIHelper.get_address_from_public_key(self._public_key): - return False - if not verify_message( - self._peer_public_key.encode("utf-8"), self._signature, self._public_key - ): - return False - return True - - class Libp2pNode: """Libp2p p2p node as a subprocess with named pipes interface.""" @@ -558,7 +433,9 @@ def stop(self) -> None: def _get_por_from_configuration(key: Crypto) -> AgentRecord: signature = key.sign_message(key.public_key.encode("utf-8")) - record = AgentRecord(key.address, key.public_key, key.public_key, signature) + record = AgentRecord( + key.address, key.public_key, key.public_key, signature, POR_DEFAULT_SERVICE_ID + ) return record diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index 9d23bed4a5..ce5668c88e 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -16,15 +16,15 @@ fingerprint: aea/pipe_unix.go: QmSzbAexrSUSA9ZE6VT6MmcWF5MhEcmceQjAbnfRoK7Ruv aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK check_dependencies.py: QmYDCgddhBGunicdN5z1397LC53sKv7oFjTuqwz5V8WLT6 - connection.py: QmRky1tHapy2XQH5dQP81eFhCKvjHq6QzkcEGZHjAhGaaE + connection.py: QmZfCPFmQpYF9MEs48dkLYEDDArd7Jx1AySNeqBCsnXNFt dht/dhtclient/dhtclient.go: QmZCgMowWw4ueCSsyYY12om3RMQA3Ydp98ti65Eqa5ccuz dht/dhtclient/dhtclient_test.go: QmbTvSmNU1uqjLQo9LdD7zWmDcLuzEacDeg7AG4VXhmriV dht/dhtclient/options.go: QmWs7fQqHRavjwe1PjihHFi85HeJu9qSEVyMdcRviZzaT2 dht/dhtnode/dhtnode.go: QmbyhgbCSAbQ1QsDw7FM7Nt5sZcvhbupA1jv5faxutbV7N - dht/dhtnode/message.pb.go: QmPnnc1KLiqm82H99YbWQbPWHoYbhHDEZvEaMAYWJsRdB2 - dht/dhtnode/message.proto: QmRvtWkeS9i1szhYWe3StnK1t4zhQjEGJhwZuLPrGbBsfX + dht/dhtnode/message.pb.go: QmbMVfoPm74uxHRdAtzhke7ftn8YUQQ7kBs3e7QAjfaEc2 + dht/dhtnode/message.proto: QmfFz18nn89kn7ReBwQri7cijbEh6D9TTd3j2euCkKVvDk dht/dhtnode/streams.go: Qmc2JcyiU4wHsgDj6aUunMAp4c5yMzo2ixeqRZHSW5PVwo - dht/dhtnode/utils.go: Qmd5f5JCA41kutBE4953JA4XzRuVofYJJ8g4dTsJETtXcd + dht/dhtnode/utils.go: QmRhtiX1aUy7APRteJyFtpsJjoz8VshYDY1UcJkD8zJX1o dht/dhtpeer/benchmarks_test.go: QmQcy8WN5DY8Nfp5nkDgRaS6NjQbMzc6DSMbdY8QDPx9ao dht/dhtpeer/dhtpeer.go: QmRkKAj1uJhNQYwAdPH5eKPRykHyhutwwXJ2yZLbn4c7pb dht/dhtpeer/dhtpeer_test.go: QmQvoVABqb3oDsH1uqF22d5Tjb2a8Hyf7U7q8jaYQfrjfa diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go index cd3cbfd4c9..308c5d8b13 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go @@ -35,9 +35,9 @@ const ( Status_ERROR_GENERIC Status_ErrCode = 3 Status_ERROR_SERIALIZATION Status_ErrCode = 4 // register (1x) - Status_ERROR_INVALID_AGENT_ADDRESS Status_ErrCode = 10 - Status_ERROR_INVALID_PUBLIC_KEY Status_ErrCode = 11 - Status_ERROR_INVALID_PROOF Status_ErrCode = 12 + Status_ERROR_WRONG_AGENT_ADDRESS Status_ErrCode = 10 + Status_ERROR_WRONG_PUBLIC_KEY Status_ErrCode = 11 + Status_ERROR_INVALID_PROOF Status_ErrCode = 12 // lookup & delivery (2x) Status_ERROR_UNKNOWN_AGENT_ADDRESS Status_ErrCode = 20 Status_ERROR_AGENT_NOT_READY Status_ErrCode = 21 @@ -51,8 +51,8 @@ var ( 2: "ERROR_UNEXPECTED_PAYLOAD", 3: "ERROR_GENERIC", 4: "ERROR_SERIALIZATION", - 10: "ERROR_INVALID_AGENT_ADDRESS", - 11: "ERROR_INVALID_PUBLIC_KEY", + 10: "ERROR_WRONG_AGENT_ADDRESS", + 11: "ERROR_WRONG_PUBLIC_KEY", 12: "ERROR_INVALID_PROOF", 20: "ERROR_UNKNOWN_AGENT_ADDRESS", 21: "ERROR_AGENT_NOT_READY", @@ -63,8 +63,8 @@ var ( "ERROR_UNEXPECTED_PAYLOAD": 2, "ERROR_GENERIC": 3, "ERROR_SERIALIZATION": 4, - "ERROR_INVALID_AGENT_ADDRESS": 10, - "ERROR_INVALID_PUBLIC_KEY": 11, + "ERROR_WRONG_AGENT_ADDRESS": 10, + "ERROR_WRONG_PUBLIC_KEY": 11, "ERROR_INVALID_PROOF": 12, "ERROR_UNKNOWN_AGENT_ADDRESS": 20, "ERROR_AGENT_NOT_READY": 21, @@ -592,11 +592,11 @@ var file_dht_dhtnode_message_proto_rawDesc = []byte{ 0x65, 0x6c, 0x12, 0x2c, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, - 0x22, 0xdf, 0x02, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x0a, 0x04, 0x63, + 0x22, 0xdb, 0x02, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x45, 0x72, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x73, 0x67, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x22, 0x93, 0x02, 0x0a, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x22, 0x8f, 0x02, 0x0a, 0x07, 0x45, 0x72, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x53, 0x55, 0x50, 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, @@ -604,38 +604,38 @@ var file_dht_dhtnode_message_proto_rawDesc = []byte{ 0x45, 0x58, 0x50, 0x45, 0x43, 0x54, 0x45, 0x44, 0x5f, 0x50, 0x41, 0x59, 0x4c, 0x4f, 0x41, 0x44, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x53, - 0x45, 0x52, 0x49, 0x41, 0x4c, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x04, 0x12, 0x1f, - 0x0a, 0x1b, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, - 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x0a, 0x12, - 0x1c, 0x0a, 0x18, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, - 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x4b, 0x45, 0x59, 0x10, 0x0b, 0x12, 0x17, 0x0a, - 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x50, - 0x52, 0x4f, 0x4f, 0x46, 0x10, 0x0c, 0x12, 0x1f, 0x0a, 0x1b, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, - 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, - 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x14, 0x12, 0x19, 0x0a, 0x15, 0x45, 0x52, 0x52, 0x4f, 0x52, - 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x4e, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x59, - 0x10, 0x15, 0x22, 0xcd, 0x02, 0x0a, 0x0a, 0x41, 0x63, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x06, 0x73, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x64, 0x68, - 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x48, 0x00, 0x52, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, - 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, - 0x64, 0x65, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x48, 0x00, 0x52, 0x08, 0x72, - 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x3f, 0x0a, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, - 0x70, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x16, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, - 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x0f, 0x6c, 0x6f, 0x6f, 0x6b, - 0x75, 0x70, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, - 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x6c, 0x6f, - 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x0c, - 0x61, 0x65, 0x61, 0x5f, 0x65, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x65, 0x61, - 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x65, 0x61, 0x45, - 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, - 0x61, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x45, 0x52, 0x49, 0x41, 0x4c, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x04, 0x12, 0x1d, + 0x0a, 0x19, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x57, 0x52, 0x4f, 0x4e, 0x47, 0x5f, 0x41, 0x47, + 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x0a, 0x12, 0x1a, 0x0a, + 0x16, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x57, 0x52, 0x4f, 0x4e, 0x47, 0x5f, 0x50, 0x55, 0x42, + 0x4c, 0x49, 0x43, 0x5f, 0x4b, 0x45, 0x59, 0x10, 0x0b, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x52, 0x52, + 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x4f, 0x46, + 0x10, 0x0c, 0x12, 0x1f, 0x0a, 0x1b, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, + 0x4f, 0x57, 0x4e, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, + 0x53, 0x10, 0x14, 0x12, 0x19, 0x0a, 0x15, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x41, 0x47, 0x45, + 0x4e, 0x54, 0x5f, 0x4e, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x15, 0x22, 0xcd, + 0x02, 0x0a, 0x0a, 0x41, 0x63, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, + 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, + 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x48, 0x00, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x52, + 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x48, 0x00, 0x52, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, + 0x74, 0x65, 0x72, 0x12, 0x3f, 0x0a, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, 0x68, + 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x0f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x72, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, + 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x0c, 0x61, 0x65, 0x61, 0x5f, + 0x65, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, + 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x65, 0x61, 0x45, 0x6e, 0x76, 0x65, + 0x6c, 0x6f, 0x70, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x65, 0x61, 0x45, 0x6e, 0x76, 0x65, 0x6c, + 0x6f, 0x70, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto index 60e26f9e39..13f02ded29 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto @@ -37,8 +37,8 @@ message Status { ERROR_GENERIC = 3; ERROR_SERIALIZATION = 4; // register (1x) - ERROR_INVALID_AGENT_ADDRESS = 10; - ERROR_INVALID_PUBLIC_KEY = 11; + ERROR_WRONG_AGENT_ADDRESS = 10; + ERROR_WRONG_PUBLIC_KEY = 11; ERROR_INVALID_PROOF = 12; // lookup & delivery (2x) ERROR_UNKNOWN_AGENT_ADDRESS = 20; diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/utils.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/utils.go index 44ce2680dc..046bc054a5 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/utils.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/utils.go @@ -34,14 +34,14 @@ func IsValidProofOfRepresentation(record *AgentRecord, agentAddress string, repr // check agent address matches if record.Address != agentAddress { err := errors.New("Wrong agent address, expected " + agentAddress) - response := &Status{Code: Status_ERROR_INVALID_AGENT_ADDRESS, Msgs: []string{err.Error()}} + response := &Status{Code: Status_ERROR_WRONG_AGENT_ADDRESS, Msgs: []string{err.Error()}} return response, err } // check public key matches if record.PeerPublicKey != representativePeerPubKey { err := errors.New("Wrong peer public key, expected " + representativePeerPubKey) - response := &Status{Code: Status_ERROR_INVALID_PUBLIC_KEY, Msgs: []string{err.Error()}} + response := &Status{Code: Status_ERROR_WRONG_PUBLIC_KEY, Msgs: []string{err.Error()}} return response, err } @@ -51,7 +51,7 @@ func IsValidProofOfRepresentation(record *AgentRecord, agentAddress string, repr if err == nil { err = errors.New("Agent address and public key don't match") } - response := &Status{Code: Status_ERROR_INVALID_AGENT_ADDRESS} + response := &Status{Code: Status_ERROR_WRONG_AGENT_ADDRESS} return response, err } diff --git a/packages/fetchai/connections/p2p_libp2p_client/acn_message.proto b/packages/fetchai/connections/p2p_libp2p_client/acn_message.proto new file mode 100644 index 0000000000..13f02ded29 --- /dev/null +++ b/packages/fetchai/connections/p2p_libp2p_client/acn_message.proto @@ -0,0 +1,61 @@ +syntax = "proto3"; + +package dhtnode; + +message AgentRecord { + string service_id = 1; + string address = 2; + string public_key = 3; + string peer_public_key = 4; + string signature = 5; +} + +message Register { + AgentRecord record = 1; +} + +message LookupRequest { + string agent_address = 1; +} + +message LookupResponse { + AgentRecord agent_record = 1; +} + +message AeaEnvelope { + // TOFIX(LR) import aea.Envelop type + bytes envel = 1; + AgentRecord record = 2; +} + +message Status { + enum ErrCode { + // common (0x) + SUCCESS = 0; + ERROR_UNSUPPORTED_VERSION = 1; + ERROR_UNEXPECTED_PAYLOAD = 2; + ERROR_GENERIC = 3; + ERROR_SERIALIZATION = 4; + // register (1x) + ERROR_WRONG_AGENT_ADDRESS = 10; + ERROR_WRONG_PUBLIC_KEY = 11; + ERROR_INVALID_PROOF = 12; + // lookup & delivery (2x) + ERROR_UNKNOWN_AGENT_ADDRESS = 20; + ERROR_AGENT_NOT_READY = 21; + } + + ErrCode code = 1; + repeated string msgs = 2; +} + +message AcnMessage { + string version = 1; + oneof payload { + Status status = 2; + Register register = 3; + LookupRequest lookup_request = 4; + LookupResponse lookup_response = 5; + AeaEnvelope aea_envelope = 6; + } +} \ No newline at end of file diff --git a/packages/fetchai/connections/p2p_libp2p_client/acn_message_pb2.py b/packages/fetchai/connections/p2p_libp2p_client/acn_message_pb2.py new file mode 100644 index 0000000000..1e3a7e8608 --- /dev/null +++ b/packages/fetchai/connections/p2p_libp2p_client/acn_message_pb2.py @@ -0,0 +1,668 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: acn_message.proto + +import sys + +_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor.FileDescriptor( + name="acn_message.proto", + package="dhtnode", + syntax="proto3", + serialized_pb=_b( + '\n\x11\x61\x63n_message.proto\x12\x07\x64htnode"r\n\x0b\x41gentRecord\x12\x12\n\nservice_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\t\x12\x12\n\npublic_key\x18\x03 \x01(\t\x12\x17\n\x0fpeer_public_key\x18\x04 \x01(\t\x12\x11\n\tsignature\x18\x05 \x01(\t"0\n\x08Register\x12$\n\x06record\x18\x01 \x01(\x0b\x32\x14.dhtnode.AgentRecord"&\n\rLookupRequest\x12\x15\n\ragent_address\x18\x01 \x01(\t"<\n\x0eLookupResponse\x12*\n\x0c\x61gent_record\x18\x01 \x01(\x0b\x32\x14.dhtnode.AgentRecord"B\n\x0b\x41\x65\x61\x45nvelope\x12\r\n\x05\x65nvel\x18\x01 \x01(\x0c\x12$\n\x06record\x18\x02 \x01(\x0b\x32\x14.dhtnode.AgentRecord"\xcf\x02\n\x06Status\x12%\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x17.dhtnode.Status.ErrCode\x12\x0c\n\x04msgs\x18\x02 \x03(\t"\x8f\x02\n\x07\x45rrCode\x12\x0b\n\x07SUCCESS\x10\x00\x12\x1d\n\x19\x45RROR_UNSUPPORTED_VERSION\x10\x01\x12\x1c\n\x18\x45RROR_UNEXPECTED_PAYLOAD\x10\x02\x12\x11\n\rERROR_GENERIC\x10\x03\x12\x17\n\x13\x45RROR_SERIALIZATION\x10\x04\x12\x1d\n\x19\x45RROR_WRONG_AGENT_ADDRESS\x10\n\x12\x1a\n\x16\x45RROR_WRONG_PUBLIC_KEY\x10\x0b\x12\x17\n\x13\x45RROR_INVALID_PROOF\x10\x0c\x12\x1f\n\x1b\x45RROR_UNKNOWN_AGENT_ADDRESS\x10\x14\x12\x19\n\x15\x45RROR_AGENT_NOT_READY\x10\x15"\x86\x02\n\nAcnMessage\x12\x0f\n\x07version\x18\x01 \x01(\t\x12!\n\x06status\x18\x02 \x01(\x0b\x32\x0f.dhtnode.StatusH\x00\x12%\n\x08register\x18\x03 \x01(\x0b\x32\x11.dhtnode.RegisterH\x00\x12\x30\n\x0elookup_request\x18\x04 \x01(\x0b\x32\x16.dhtnode.LookupRequestH\x00\x12\x32\n\x0flookup_response\x18\x05 \x01(\x0b\x32\x17.dhtnode.LookupResponseH\x00\x12,\n\x0c\x61\x65\x61_envelope\x18\x06 \x01(\x0b\x32\x14.dhtnode.AeaEnvelopeH\x00\x42\t\n\x07payloadb\x06proto3' + ), +) +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + + +_STATUS_ERRCODE = _descriptor.EnumDescriptor( + name="ErrCode", + full_name="dhtnode.Status.ErrCode", + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name="SUCCESS", index=0, number=0, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="ERROR_UNSUPPORTED_VERSION", index=1, number=1, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="ERROR_UNEXPECTED_PAYLOAD", index=2, number=2, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="ERROR_GENERIC", index=3, number=3, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="ERROR_SERIALIZATION", index=4, number=4, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="ERROR_WRONG_AGENT_ADDRESS", + index=5, + number=10, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name="ERROR_WRONG_PUBLIC_KEY", index=6, number=11, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="ERROR_INVALID_PROOF", index=7, number=12, options=None, type=None + ), + _descriptor.EnumValueDescriptor( + name="ERROR_UNKNOWN_AGENT_ADDRESS", + index=8, + number=20, + options=None, + type=None, + ), + _descriptor.EnumValueDescriptor( + name="ERROR_AGENT_NOT_READY", index=9, number=21, options=None, type=None + ), + ], + containing_type=None, + options=None, + serialized_start=431, + serialized_end=702, +) +_sym_db.RegisterEnumDescriptor(_STATUS_ERRCODE) + + +_AGENTRECORD = _descriptor.Descriptor( + name="AgentRecord", + full_name="dhtnode.AgentRecord", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="service_id", + full_name="dhtnode.AgentRecord.service_id", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name="address", + full_name="dhtnode.AgentRecord.address", + index=1, + number=2, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name="public_key", + full_name="dhtnode.AgentRecord.public_key", + index=2, + number=3, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name="peer_public_key", + full_name="dhtnode.AgentRecord.peer_public_key", + index=3, + number=4, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name="signature", + full_name="dhtnode.AgentRecord.signature", + index=4, + number=5, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=30, + serialized_end=144, +) + + +_REGISTER = _descriptor.Descriptor( + name="Register", + full_name="dhtnode.Register", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="record", + full_name="dhtnode.Register.record", + index=0, + number=1, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=146, + serialized_end=194, +) + + +_LOOKUPREQUEST = _descriptor.Descriptor( + name="LookupRequest", + full_name="dhtnode.LookupRequest", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="agent_address", + full_name="dhtnode.LookupRequest.agent_address", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=196, + serialized_end=234, +) + + +_LOOKUPRESPONSE = _descriptor.Descriptor( + name="LookupResponse", + full_name="dhtnode.LookupResponse", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="agent_record", + full_name="dhtnode.LookupResponse.agent_record", + index=0, + number=1, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=236, + serialized_end=296, +) + + +_AEAENVELOPE = _descriptor.Descriptor( + name="AeaEnvelope", + full_name="dhtnode.AeaEnvelope", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="envel", + full_name="dhtnode.AeaEnvelope.envel", + index=0, + number=1, + type=12, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b(""), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name="record", + full_name="dhtnode.AeaEnvelope.record", + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=298, + serialized_end=364, +) + + +_STATUS = _descriptor.Descriptor( + name="Status", + full_name="dhtnode.Status", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="code", + full_name="dhtnode.Status.code", + index=0, + number=1, + type=14, + cpp_type=8, + label=1, + has_default_value=False, + default_value=0, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name="msgs", + full_name="dhtnode.Status.msgs", + index=1, + number=2, + type=9, + cpp_type=9, + label=3, + has_default_value=False, + default_value=[], + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + ], + extensions=[], + nested_types=[], + enum_types=[_STATUS_ERRCODE,], + options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[], + serialized_start=367, + serialized_end=702, +) + + +_ACNMESSAGE = _descriptor.Descriptor( + name="AcnMessage", + full_name="dhtnode.AcnMessage", + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name="version", + full_name="dhtnode.AcnMessage.version", + index=0, + number=1, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name="status", + full_name="dhtnode.AcnMessage.status", + index=1, + number=2, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name="register", + full_name="dhtnode.AcnMessage.register", + index=2, + number=3, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name="lookup_request", + full_name="dhtnode.AcnMessage.lookup_request", + index=3, + number=4, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name="lookup_response", + full_name="dhtnode.AcnMessage.lookup_response", + index=4, + number=5, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name="aea_envelope", + full_name="dhtnode.AcnMessage.aea_envelope", + index=5, + number=6, + type=11, + cpp_type=10, + label=1, + has_default_value=False, + default_value=None, + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + ], + extensions=[], + nested_types=[], + enum_types=[], + options=None, + is_extendable=False, + syntax="proto3", + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name="payload", + full_name="dhtnode.AcnMessage.payload", + index=0, + containing_type=None, + fields=[], + ), + ], + serialized_start=705, + serialized_end=967, +) + +_REGISTER.fields_by_name["record"].message_type = _AGENTRECORD +_LOOKUPRESPONSE.fields_by_name["agent_record"].message_type = _AGENTRECORD +_AEAENVELOPE.fields_by_name["record"].message_type = _AGENTRECORD +_STATUS.fields_by_name["code"].enum_type = _STATUS_ERRCODE +_STATUS_ERRCODE.containing_type = _STATUS +_ACNMESSAGE.fields_by_name["status"].message_type = _STATUS +_ACNMESSAGE.fields_by_name["register"].message_type = _REGISTER +_ACNMESSAGE.fields_by_name["lookup_request"].message_type = _LOOKUPREQUEST +_ACNMESSAGE.fields_by_name["lookup_response"].message_type = _LOOKUPRESPONSE +_ACNMESSAGE.fields_by_name["aea_envelope"].message_type = _AEAENVELOPE +_ACNMESSAGE.oneofs_by_name["payload"].fields.append( + _ACNMESSAGE.fields_by_name["status"] +) +_ACNMESSAGE.fields_by_name["status"].containing_oneof = _ACNMESSAGE.oneofs_by_name[ + "payload" +] +_ACNMESSAGE.oneofs_by_name["payload"].fields.append( + _ACNMESSAGE.fields_by_name["register"] +) +_ACNMESSAGE.fields_by_name["register"].containing_oneof = _ACNMESSAGE.oneofs_by_name[ + "payload" +] +_ACNMESSAGE.oneofs_by_name["payload"].fields.append( + _ACNMESSAGE.fields_by_name["lookup_request"] +) +_ACNMESSAGE.fields_by_name[ + "lookup_request" +].containing_oneof = _ACNMESSAGE.oneofs_by_name["payload"] +_ACNMESSAGE.oneofs_by_name["payload"].fields.append( + _ACNMESSAGE.fields_by_name["lookup_response"] +) +_ACNMESSAGE.fields_by_name[ + "lookup_response" +].containing_oneof = _ACNMESSAGE.oneofs_by_name["payload"] +_ACNMESSAGE.oneofs_by_name["payload"].fields.append( + _ACNMESSAGE.fields_by_name["aea_envelope"] +) +_ACNMESSAGE.fields_by_name[ + "aea_envelope" +].containing_oneof = _ACNMESSAGE.oneofs_by_name["payload"] +DESCRIPTOR.message_types_by_name["AgentRecord"] = _AGENTRECORD +DESCRIPTOR.message_types_by_name["Register"] = _REGISTER +DESCRIPTOR.message_types_by_name["LookupRequest"] = _LOOKUPREQUEST +DESCRIPTOR.message_types_by_name["LookupResponse"] = _LOOKUPRESPONSE +DESCRIPTOR.message_types_by_name["AeaEnvelope"] = _AEAENVELOPE +DESCRIPTOR.message_types_by_name["Status"] = _STATUS +DESCRIPTOR.message_types_by_name["AcnMessage"] = _ACNMESSAGE + +AgentRecord = _reflection.GeneratedProtocolMessageType( + "AgentRecord", + (_message.Message,), + dict( + DESCRIPTOR=_AGENTRECORD, + __module__="acn_message_pb2" + # @@protoc_insertion_point(class_scope:dhtnode.AgentRecord) + ), +) +_sym_db.RegisterMessage(AgentRecord) + +Register = _reflection.GeneratedProtocolMessageType( + "Register", + (_message.Message,), + dict( + DESCRIPTOR=_REGISTER, + __module__="acn_message_pb2" + # @@protoc_insertion_point(class_scope:dhtnode.Register) + ), +) +_sym_db.RegisterMessage(Register) + +LookupRequest = _reflection.GeneratedProtocolMessageType( + "LookupRequest", + (_message.Message,), + dict( + DESCRIPTOR=_LOOKUPREQUEST, + __module__="acn_message_pb2" + # @@protoc_insertion_point(class_scope:dhtnode.LookupRequest) + ), +) +_sym_db.RegisterMessage(LookupRequest) + +LookupResponse = _reflection.GeneratedProtocolMessageType( + "LookupResponse", + (_message.Message,), + dict( + DESCRIPTOR=_LOOKUPRESPONSE, + __module__="acn_message_pb2" + # @@protoc_insertion_point(class_scope:dhtnode.LookupResponse) + ), +) +_sym_db.RegisterMessage(LookupResponse) + +AeaEnvelope = _reflection.GeneratedProtocolMessageType( + "AeaEnvelope", + (_message.Message,), + dict( + DESCRIPTOR=_AEAENVELOPE, + __module__="acn_message_pb2" + # @@protoc_insertion_point(class_scope:dhtnode.AeaEnvelope) + ), +) +_sym_db.RegisterMessage(AeaEnvelope) + +Status = _reflection.GeneratedProtocolMessageType( + "Status", + (_message.Message,), + dict( + DESCRIPTOR=_STATUS, + __module__="acn_message_pb2" + # @@protoc_insertion_point(class_scope:dhtnode.Status) + ), +) +_sym_db.RegisterMessage(Status) + +AcnMessage = _reflection.GeneratedProtocolMessageType( + "AcnMessage", + (_message.Message,), + dict( + DESCRIPTOR=_ACNMESSAGE, + __module__="acn_message_pb2" + # @@protoc_insertion_point(class_scope:dhtnode.AcnMessage) + ), +) +_sym_db.RegisterMessage(AcnMessage) + + +# @@protoc_insertion_point(module_scope) diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.py b/packages/fetchai/connections/p2p_libp2p_client/connection.py index 2a341d4581..74a0aa49a7 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.py +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.py @@ -24,16 +24,22 @@ import random import struct from asyncio import CancelledError -from random import randint from typing import List, Optional, Union, cast from aea.configurations.base import PublicId from aea.configurations.constants import DEFAULT_LEDGER from aea.connections.base import Connection, ConnectionStates +from aea.crypto.base import Crypto from aea.crypto.registries import make_crypto from aea.exceptions import enforce +from aea.helpers.acn.agent_record import AgentRecord +from aea.helpers.acn.uri import Uri from aea.mail.base import Envelope +from .acn_message_pb2 import AcnMessage +from .acn_message_pb2 import AgentRecord as AgentRecordPb +from .acn_message_pb2 import Register, Status + _default_logger = logging.getLogger( "aea.packages.fetchai.connections.p2p_libp2p_client" @@ -43,45 +49,17 @@ SUPPORTED_LEDGER_IDS = ["fetchai", "cosmos", "ethereum"] +POR_DEFAULT_SERVICE_ID = "" -class Uri: - """Holds a node address in format "host:port".""" - - def __init__( - self, - uri: Optional[str] = None, - host: Optional[str] = None, - port: Optional[int] = None, - ): - """Initialise Uri.""" - if uri is not None: - split = uri.split(":", 1) - self._host = split[0] - self._port = int(split[1]) - elif host is not None and port is not None: - self._host = host - self._port = port - else: - self._host = "127.0.0.1" - self._port = randint(5000, 10000) # nosec - - def __str__(self): - """Get string representation.""" - return "{}:{}".format(self._host, self._port) - - def __repr__(self): # pragma: no cover - """Get object representation.""" - return self.__str__() +ACN_CURRENT_VERSION = "0.1.0" - @property - def host(self) -> str: - """Get host.""" - return self._host - @property - def port(self) -> int: - """Get port.""" - return self._port +def _get_pors_from_connection_config(key: Crypto, count: int) -> List[AgentRecord]: + signature = key.sign_message(key.public_key.encode("utf-8")) + record = AgentRecord( + key.address, key.public_key, key.public_key, signature, POR_DEFAULT_SERVICE_ID + ) + return [record for i in range(count)] class P2PLibp2pClientConnection(Connection): @@ -119,6 +97,14 @@ def __init__(self, **kwargs): "Delegate Uri should be provided for each node", ) + nodes_certs = [node["cert"] for node in nodes] + enforce( + len(nodes_certs) == len(nodes), + "Delegate Cert should be provided for each node", + ) + + # verify keys are correct + if ( self.has_crypto_store and self.crypto_store.crypto_objects.get(ledger_id, None) is not None @@ -136,13 +122,19 @@ def __init__(self, **kwargs): # delegate uris self.delegate_uris = [Uri(node_uri) for node_uri in nodes_uris] - # delegates certificates - # TOFIX(LR) will be mandatory - self.delegate_certs = [] + # delegates PoRs + self.delegate_pors = _get_pors_from_connection_config(key, len(nodes)) + for i in range(len(self.delegate_pors)): + record = self.delegate_pors[i] + enforce( + True or record.is_valid_for(self.address, key.public_key), + f"Invalid Proof-of-Representation for node {self.delegate_uris[i]}", + ) # select a delegate index = random.randint(0, len(self.delegate_uris) - 1) # nosec self.node_uri = self.delegate_uris[index] + self.node_por = self.delegate_pors[index] self.logger.debug("Node to use as delegate: {}".format(self.node_uri)) # tcp connection @@ -191,8 +183,36 @@ async def connect(self) -> None: raise e async def _setup_connection(self): - await self._send(bytes(self.address, "utf-8")) - await self._receive() + record = AgentRecordPb() + record.address = self.node_por.address + record.public_key = self.node_por.public_key + record.peer_public_key = self.node_por.peer_public_key + record.signature = self.node_por.signature + record.service_id = self.node_por.service_id # pylint: disable=no-member + + registration = Register() + registration.record.CopyFrom(record) # pylint: disable=no-member + msg = AcnMessage() + msg.version = ACN_CURRENT_VERSION + msg.register.CopyFrom(registration) # pylint: disable=no-member + + buf = msg.SerializeToString() + await self._send(buf) + + buf = await self._receive() + msg = AcnMessage() + msg.ParseFromString(buf) + payload = msg.WhichOneof("payload") + if payload != "status": + raise Exception(f"Wrong response message from peer: {payload}") + response = msg.status # pylint: disable=no-member + + if response.code != Status.SUCCESS: # pylint: disable=no-member + raise Exception( + "Registration to peer failed: {}".format( + Status.ErrCode.Name(response.code) # pylint: disable=no-member + ) + ) async def disconnect(self) -> None: """ diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml index c5b1e3d377..58cd360076 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml @@ -10,7 +10,9 @@ aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmP9mRtKmt8c1s9XcFeycewPbAQ9BDfkA1Zo4fNcjsTFra __init__.py: QmT1FEHkPGMHV5oiVEfQHHr25N2qdZxydSNRJabJvYiTgf - connection.py: QmRSshSZ6EwUosAjTCXWXWmRefungEFkcTAezjcbHBgrc1 + acn_message.proto: QmfFz18nn89kn7ReBwQri7cijbEh6D9TTd3j2euCkKVvDk + acn_message_pb2.py: QmXF5A7PiUiduTWEEkZQZuGRmX7QemyKx8ZDrWnevsi82i + connection.py: QmShEAFNqCKSM3gaYUb5yYHHu6ZBr1hnTB6f11EAnm23EL fingerprint_ignore_patterns: [] connections: [] protocols: [] diff --git a/packages/hashes.csv b/packages/hashes.csv index 95d17daff9..66cd8f0acb 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,8 +34,8 @@ fetchai/connections/http_server,QmQ8hNfEts6th9zfnocYHzyYw4dXkDNDj8zoKtov1wB49n fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 -fetchai/connections/p2p_libp2p,QmWm4rMxh3CeogLdwsqrUUsnQdP2fBp7gDEwb66MUofXon -fetchai/connections/p2p_libp2p_client,QmQndwAPhGHTpe1J8c6Q6jabV2KfkND7jsYiEF9Ysmyby3 +fetchai/connections/p2p_libp2p,QmRmAzuajxVsmE9thLwALj7RAwn4HVKnkfHeopaqbBSCDv +fetchai/connections/p2p_libp2p_client,QmcHPi4VPha5WwydNkVVU4yGSvQvxtoStYZYEmKhw61fXp fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T fetchai/connections/scaffold,QmQcfdXmL248RFWGphcYeJt7jVNTN7fCJAgwZbYpdB6PK4 diff --git a/tests/conftest.py b/tests/conftest.py index 11f5d754c3..9643d48bb2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -53,6 +53,7 @@ from aea.configurations.loader import load_component_configuration from aea.connections.base import Connection from aea.contracts.base import Contract, contract_registry +from aea.crypto.base import Crypto from aea.crypto.cosmos import DEFAULT_ADDRESS as COSMOS_DEFAULT_ADDRESS from aea.crypto.cosmos import _COSMOS from aea.crypto.ethereum import DEFAULT_ADDRESS as ETHEREUM_DEFAULT_ADDRESS @@ -899,7 +900,7 @@ def _make_libp2p_connection( return P2PLibp2pConnection(configuration=configuration, identity=identity) -def _make_libp2p_client_connection( +def _make_libp2p_client_connection_to_restore( node_port: int = 11234, node_host: str = "127.0.0.1", uri: Optional[str] = None, ) -> P2PLibp2pClientConnection: crypto = make_crypto(COSMOS) @@ -911,7 +912,37 @@ def _make_libp2p_client_connection( "uri": str(uri) if uri is not None else "{}:{}".format(node_host, node_port) - } + }, + ], + connection_id=P2PLibp2pClientConnection.connection_id, + ) + return P2PLibp2pClientConnection(configuration=configuration, identity=identity) + + +def _make_libp2p_client_connection( + node_port: int = 11234, + node_host: str = "127.0.0.1", + uri: Optional[str] = None, + cert: Optional[str] = None, + key: Optional[Crypto] = None, +) -> P2PLibp2pClientConnection: + crypto = key + if crypto is None: + crypto = make_crypto(COSMOS) + identity = Identity("", address=crypto.address) + key_file = f"./{crypto.public_key}.txt" + with open(key_file, "wb") as f: + crypto.dump(f) + + configuration = ConnectionConfig( + client_key_file=key_file, + nodes=[ + { + "uri": str(uri) + if uri is not None + else "{}:{}".format(node_host, node_port), + "cert": cert if cert is not None else crypto.public_key, + }, ], connection_id=P2PLibp2pClientConnection.connection_id, ) From 9fdae8c68fefd153afed99a40ee8f3defb952ab5 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 17 Dec 2020 16:12:55 +0000 Subject: [PATCH 016/204] fix builder tests --- aea/aea_builder.py | 30 ++++++++++++++++++++++-------- tests/test_aea_builder.py | 5 +++-- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/aea/aea_builder.py b/aea/aea_builder.py index 7f6d1422cd..feab8cda4e 100644 --- a/aea/aea_builder.py +++ b/aea/aea_builder.py @@ -963,6 +963,24 @@ def _run_build_entrypoint( command_str = " ".join(command) if logger: logger.info(f"Running command '{command_str}'...") + stdout, stderr, code = cls._run_in_subprocess(command, source_directory) + if code == 0: + if logger: + logger.info(f"Command '{command_str}' succeded with output:\n{stdout}") + else: + raise AEAException( + f"An error occurred while running command '{command_str}':\n{stderr}" + ) + + @classmethod + def _run_in_subprocess( + cls, command: List[str], source_directory: str + ) -> Tuple[str, str, int]: + """ + Run in subprocess. + + :return: stdout, stderr, code + """ res = subprocess.run( # nosec command, cwd=source_directory, @@ -970,14 +988,10 @@ def _run_build_entrypoint( timeout=cls.BUILD_TIMEOUT, capture_output=True, ) - if res.returncode == 0: - if logger: - logger.info(f"Command '{command_str}' succeded!") - else: - e = res.stderr.decode("utf-8") - raise AEAException( - f"An error occurred while running command '{command_str}':\n{e}" - ) + code = res.returncode + stdout = res.stdout.decode("utf-8") + stderr = res.stderr.decode("utf-8") + return stdout, stderr, code def _build_identity_from_wallet(self, wallet: Wallet) -> Identity: """ diff --git a/tests/test_aea_builder.py b/tests/test_aea_builder.py index 10f161413c..9a2de55891 100644 --- a/tests/test_aea_builder.py +++ b/tests/test_aea_builder.py @@ -838,11 +838,12 @@ def test_build_negative_syntax_error(self): self.builder.call_all_build_entrypoints() @mock.patch( - "aea.aea_builder.check_call", side_effect=Exception("some error."), + "aea.aea_builder.AEABuilder._run_in_subprocess", + return_value=("", "some error.", 1), ) def test_build_negative_subprocess(self, *_mocks): """Test build, negative due to script error at runtime.""" - match = "An error occurred while running command '.*script.py .+': some error." + match = "An error occurred while running command '.*script.py .+':\nsome error." with cd(self._get_cwd()), pytest.raises(AEAException, match=match): self.script_path.write_text("") self.builder.call_all_build_entrypoints() From 32c8fce32cf5b7ab50504aabfc8adc40d584727a Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 17 Dec 2020 17:16:32 +0000 Subject: [PATCH 017/204] add multiaddress command in docs, update doc tests --- docs/aries-cloud-agent-demo.md | 2 +- docs/car-park-skills.md | 2 +- docs/erc1155-skills.md | 2 +- docs/generic-skills.md | 2 +- docs/ml-skills.md | 2 +- docs/tac-skills.md | 2 +- docs/thermometer-skills.md | 2 +- docs/weather-skills.md | 2 +- .../md_files/bash-aries-cloud-agent-demo.md | 2 ++ .../test_bash_yaml/md_files/bash-car-park-skills.md | 4 ++++ .../test_bash_yaml/md_files/bash-erc1155-skills.md | 4 ++++ .../md_files/bash-generic-skills-step-by-step.md | 2 ++ .../test_bash_yaml/md_files/bash-generic-skills.md | 4 ++++ .../test_bash_yaml/md_files/bash-ml-skills.md | 4 ++++ .../test_bash_yaml/md_files/bash-orm-integration.md | 5 +++++ .../test_bash_yaml/md_files/bash-skill-guide.md | 1 + .../md_files/bash-tac-skills-contract.md | 8 +++++++- .../test_bash_yaml/md_files/bash-tac-skills.md | 10 +++++++++- .../test_bash_yaml/md_files/bash-thermometer-skills.md | 4 ++++ .../test_bash_yaml/md_files/bash-weather-skills.md | 4 ++++ 20 files changed, 58 insertions(+), 10 deletions(-) diff --git a/docs/aries-cloud-agent-demo.md b/docs/aries-cloud-agent-demo.md index a49141fae4..6ad7ce9506 100644 --- a/docs/aries-cloud-agent-demo.md +++ b/docs/aries-cloud-agent-demo.md @@ -259,7 +259,7 @@ Finally run **Alice_AEA**: aea run ``` -Once you see a message of the form `To join its network use multiaddr: ['SOME_ADDRESS']` take note of the address. We will refer to this as **Alice_AEA's p2p address**. +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) We will refer to this as **Alice_AEA's p2p address**. ### Faber_AEA diff --git a/docs/car-park-skills.md b/docs/car-park-skills.md index 437e8131ef..5d28eb1ff4 100644 --- a/docs/car-park-skills.md +++ b/docs/car-park-skills.md @@ -158,7 +158,7 @@ First, run the car data seller AEA: aea run ``` -Once you see a message of the form `To join its network use multiaddr: ['SOME_ADDRESS']` take note of the address. +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) Then, update the configuration of the car data buyer AEA's p2p connection (in `vendor/fetchai/connections/p2p_libp2p/connection.yaml`) replace the following: diff --git a/docs/erc1155-skills.md b/docs/erc1155-skills.md index cca131aafd..00c287db4d 100644 --- a/docs/erc1155-skills.md +++ b/docs/erc1155-skills.md @@ -191,7 +191,7 @@ First, run the deployer AEA. aea run ``` -Once you see a message of the form `To join its network use multiaddr: ['SOME_ADDRESS']` take note of the address. +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. It will perform the following steps: - deploy the smart contract diff --git a/docs/generic-skills.md b/docs/generic-skills.md index b0e25f9f72..724110a86b 100644 --- a/docs/generic-skills.md +++ b/docs/generic-skills.md @@ -230,7 +230,7 @@ First, run the seller AEA: aea run ``` -Once you see a message of the form `To join its network use multiaddr: ['SOME_ADDRESS']` take note of the address. +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) Then, update the configuration of the buyer AEA's p2p connection (in `vendor/fetchai/connections/p2p_libp2p/connection.yaml`) replace the following: diff --git a/docs/ml-skills.md b/docs/ml-skills.md index f142e9fa9e..7936d7d8ec 100644 --- a/docs/ml-skills.md +++ b/docs/ml-skills.md @@ -165,7 +165,7 @@ First, run the data provider AEA: aea run ``` -Once you see a message of the form `To join its network use multiaddr: ['SOME_ADDRESS']` take note of the address. +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) Then, update the configuration of the model trainer AEA's p2p connection (in `vendor/fetchai/connections/p2p_libp2p/connection.yaml`) replace the following: diff --git a/docs/tac-skills.md b/docs/tac-skills.md index 8d385fba6a..e7e38f4a61 100644 --- a/docs/tac-skills.md +++ b/docs/tac-skills.md @@ -226,7 +226,7 @@ Briefly run the controller AEA: aea run ``` -Once you see a message of the form `To join its network use multiaddr: ['SOME_ADDRESS']` take note of the address. +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) Then, update the configuration of the weather client AEA's p2p connection (in `aea-config.yaml`) add the following: diff --git a/docs/thermometer-skills.md b/docs/thermometer-skills.md index 420439dcde..8d03d4a719 100644 --- a/docs/thermometer-skills.md +++ b/docs/thermometer-skills.md @@ -165,7 +165,7 @@ First, run the thermometer AEA: aea run ``` -Once you see a message of the form `To join its network use multiaddr: ['SOME_ADDRESS']` take note of the address. +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) Then, update the configuration of the thermometer client AEA's p2p connection (in `vendor/fetchai/connections/p2p_libp2p/connection.yaml`) replace the following: diff --git a/docs/weather-skills.md b/docs/weather-skills.md index 07aae67f6b..dd58636665 100644 --- a/docs/weather-skills.md +++ b/docs/weather-skills.md @@ -166,7 +166,7 @@ First, run the weather station AEA: aea run ``` -Once you see a message of the form `To join its network use multiaddr: ['SOME_ADDRESS']` take note of the address. +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) Then, update the configuration of the weather client AEA's p2p connection (in `vendor/fetchai/connections/p2p_libp2p/connection.yaml`) replace the following: diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-aries-cloud-agent-demo.md b/tests/test_docs/test_bash_yaml/md_files/bash-aries-cloud-agent-demo.md index c56524506d..a173eb3438 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-aries-cloud-agent-demo.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-aries-cloud-agent-demo.md @@ -49,6 +49,7 @@ config: ``` ``` bash aea install +aea build ``` ``` bash aea run @@ -88,6 +89,7 @@ config: ``` ``` bash aea install +aea build ``` ``` bash aea run diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md index 4971d16f87..b6e261abda 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md @@ -2,6 +2,7 @@ aea fetch fetchai/car_detector:0.19.0 cd car_detector aea install +aea build ``` ``` bash aea create car_detector @@ -11,12 +12,14 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/carpark_detection:0.17.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash aea fetch fetchai/car_data_buyer:0.20.0 cd car_data_buyer aea install +aea build ``` ``` bash aea create car_data_buyer @@ -26,6 +29,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/carpark_client:0.18.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-erc1155-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-erc1155-skills.md index d6a0e3657a..318d41ff2b 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-erc1155-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-erc1155-skills.md @@ -2,6 +2,7 @@ aea fetch fetchai/erc1155_deployer:0.20.0 cd erc1155_deployer aea install +aea build ``` ``` bash aea create erc1155_deployer @@ -11,6 +12,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/erc1155_deploy:0.19.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash @@ -28,6 +30,7 @@ aea add-key fetchai fetchai_private_key.txt --connection aea fetch fetchai/erc1155_client:0.20.0 cd erc1155_client aea install +aea build ``` ``` bash aea create erc1155_client @@ -37,6 +40,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/erc1155_client:0.18.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills-step-by-step.md b/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills-step-by-step.md index 71d4438a62..6d67e9f480 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills-step-by-step.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills-step-by-step.md @@ -53,6 +53,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add protocol fetchai/fipa:0.11.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea run ``` @@ -63,6 +64,7 @@ aea add connection fetchai/ledger:0.11.0 aea add protocol fetchai/fipa:0.11.0 aea add protocol fetchai/signing:0.8.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills.md index 8a5df07dd3..c9f6cd34ef 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills.md @@ -2,6 +2,7 @@ aea fetch fetchai/generic_seller:0.16.0 --alias my_seller_aea cd my_seller_aea aea install +aea build ``` ``` bash aea create my_seller_aea @@ -11,12 +12,14 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/generic_seller:0.18.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash aea fetch fetchai/generic_buyer:0.17.0 --alias my_buyer_aea cd my_buyer_aea aea install +aea build ``` ``` bash aea create my_buyer_aea @@ -26,6 +29,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/generic_buyer:0.18.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md index fd6fb75108..5ac88ca8f5 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md @@ -2,6 +2,7 @@ aea fetch fetchai/ml_data_provider:0.19.0 cd ml_data_provider aea install +aea build ``` ``` bash aea create ml_data_provider @@ -12,11 +13,13 @@ aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/ml_data_provider:0.17.0 aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea install +aea build ``` ``` bash aea fetch fetchai/ml_model_trainer:0.20.0 cd ml_model_trainer aea install +aea build ``` ``` bash aea create ml_model_trainer @@ -27,6 +30,7 @@ aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/ml_train:0.18.0 aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea install +aea build ``` ``` bash aea generate-key fetchai diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-orm-integration.md b/tests/test_docs/test_bash_yaml/md_files/bash-orm-integration.md index 3a21f295c0..fccc093912 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-orm-integration.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-orm-integration.md @@ -2,6 +2,7 @@ aea fetch fetchai/thermometer_aea:0.17.0 --alias my_thermometer_aea cd my_thermometer_aea aea install +aea build ``` ``` bash aea create my_thermometer_aea @@ -11,12 +12,14 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/thermometer:0.17.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash aea fetch fetchai/thermometer_client:0.18.0 --alias my_thermometer_client cd my_thermometer_client aea install +aea build ``` ``` bash aea create my_thermometer_client @@ -26,6 +29,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/thermometer_client:0.17.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash @@ -43,6 +47,7 @@ aea generate-wealth fetchai ``` ``` bash aea install +aea build ``` ``` bash aea eject skill fetchai/thermometer:0.17.0 diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-skill-guide.md b/tests/test_docs/test_bash_yaml/md_files/bash-skill-guide.md index dcbc53bd56..c1bca3c3a2 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-skill-guide.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-skill-guide.md @@ -12,6 +12,7 @@ aea add protocol fetchai/oef_search:0.11.0 aea add connection fetchai/soef:0.14.0 aea add connection fetchai/p2p_libp2p:0.13.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills-contract.md b/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills-contract.md index f289907ff4..f021d06840 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills-contract.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills-contract.md @@ -2,6 +2,7 @@ aea fetch fetchai/tac_controller_contract:0.18.0 cd tac_controller_contract aea install +aea build ``` ``` bash aea create tac_controller_contract @@ -11,6 +12,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_control_contract:0.15.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger ethereum ``` @@ -23,6 +25,8 @@ aea fetch fetchai/tac_participant:0.18.0 --alias tac_participant_one cd tac_participant_one aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using_contract 'True' --type bool aea config set vendor.fetchai.skills.tac_negotiation.models.strategy.args.is_contract_tx 'True' --type bool +aea install +aea build cd .. aea fetch fetchai/tac_participant:0.18.0 --alias tac_participant_two cd tac_participant_two @@ -30,7 +34,7 @@ aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using aea config set vendor.fetchai.skills.tac_negotiation.models.strategy.args.is_contract_tx 'True' --type bool aea generate-key ethereum aea add-key ethereum ethereum_private_key.txt -aea install +aea build ``` ``` bash aea create tac_participant_one @@ -46,6 +50,7 @@ aea add skill fetchai/tac_negotiation:0.16.0 aea generate-key ethereum aea add-key ethereum ethereum_private_key.txt aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger ethereum aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using_contract 'True' --type bool @@ -61,6 +66,7 @@ aea add skill fetchai/tac_negotiation:0.16.0 aea generate-key ethereum aea add-key ethereum ethereum_private_key.txt aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger ethereum aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using_contract 'True' --type bool diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md index cb9d19d6c4..8b5705d136 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md @@ -2,6 +2,7 @@ aea fetch fetchai/tac_controller:0.16.0 cd tac_controller aea install +aea build ``` ``` bash aea create tac_controller @@ -11,14 +12,19 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_control:0.13.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger fetchai ``` ``` bash aea fetch fetchai/tac_participant:0.18.0 --alias tac_participant_one +cd tac_participant_one +aea install +aea build +cd .. aea fetch fetchai/tac_participant:0.18.0 --alias tac_participant_two cd tac_participant_two -aea install +aea build ``` ``` bash aea create tac_participant_one @@ -32,6 +38,7 @@ aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_participation:0.14.0 aea add skill fetchai/tac_negotiation:0.16.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger fetchai ``` @@ -43,6 +50,7 @@ aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_participation:0.14.0 aea add skill fetchai/tac_negotiation:0.16.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger fetchai ``` diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-thermometer-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-thermometer-skills.md index bbcbba978b..a8facf8281 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-thermometer-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-thermometer-skills.md @@ -2,6 +2,7 @@ aea fetch fetchai/thermometer_aea:0.17.0 --alias my_thermometer_aea cd my_thermometer_aea aea install +aea build ``` ``` bash aea create my_thermometer_aea @@ -11,12 +12,14 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/thermometer:0.17.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash aea fetch fetchai/thermometer_client:0.18.0 --alias my_thermometer_client cd my_thermometer_client aea install +aea build ``` ``` bash aea create my_thermometer_client @@ -26,6 +29,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/thermometer_client:0.17.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-weather-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-weather-skills.md index 2f2fb2c202..14059dabab 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-weather-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-weather-skills.md @@ -2,6 +2,7 @@ aea fetch fetchai/weather_station:0.19.0 --alias my_weather_station cd my_weather_station aea install +aea build ``` ``` bash aea create my_weather_station @@ -11,12 +12,14 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/weather_station:0.17.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash aea fetch fetchai/weather_client:0.20.0 --alias my_weather_client cd my_weather_client aea install +aea build ``` ``` bash aea create my_weather_client @@ -26,6 +29,7 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/weather_client:0.17.0 aea install +aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash From 837d80c7e412c59837a7c9d6b2722df39c61e352 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 17 Dec 2020 18:16:31 +0000 Subject: [PATCH 018/204] fix subprocess run incompatibility with 3.6 --- aea/aea_builder.py | 3 ++- tests/test_act_storage.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/aea/aea_builder.py b/aea/aea_builder.py index b69bca3e6a..340fdabb03 100644 --- a/aea/aea_builder.py +++ b/aea/aea_builder.py @@ -980,7 +980,8 @@ def _run_in_subprocess( cwd=source_directory, check=False, timeout=cls.BUILD_TIMEOUT, - capture_output=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) code = res.returncode stdout = res.stdout.decode("utf-8") diff --git a/tests/test_act_storage.py b/tests/test_act_storage.py index 1c573f016e..576be436fa 100644 --- a/tests/test_act_storage.py +++ b/tests/test_act_storage.py @@ -48,7 +48,7 @@ from tests.common.utils import wait_for_condition -class TestBehaviour(TickerBehaviour): +class TBehaviour(TickerBehaviour): """Simple behaviour to count how many acts were called.""" OBJ_ID = "some" @@ -67,7 +67,7 @@ def act(self) -> None: self.counter += 1 -class TestHandler(Handler): +class THandler(Handler): """Simple behaviour to count how many acts were called.""" SUPPORTED_PROTOCOL = DefaultMessage.protocol_id @@ -99,7 +99,7 @@ def test_storage_access_from_behaviour(): builder.add_private_key(DEFAULT_LEDGER) skill_context = SkillContext() - behaviour = TestBehaviour(name="behaviour", skill_context=skill_context) + behaviour = TBehaviour(name="behaviour", skill_context=skill_context) test_skill = Skill( SkillConfig(name="test_skill", author="fetchai"), skill_context=skill_context, @@ -133,7 +133,7 @@ def test_storage_access_from_handler(): builder.add_private_key(DEFAULT_LEDGER) skill_context = SkillContext() - handler = TestHandler(name="behaviour", skill_context=skill_context) + handler = THandler(name="behaviour", skill_context=skill_context) test_skill = Skill( SkillConfig(name="test_skill", author="fetchai"), skill_context=skill_context, From 3a700970a0cc94fc85046e7bb81faed7061a7f95 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Thu, 17 Dec 2020 18:55:13 +0000 Subject: [PATCH 019/204] Address vulture ci comment --- scripts/whitelist.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/whitelist.py b/scripts/whitelist.py index a82f4c269e..7a9e14ed95 100644 --- a/scripts/whitelist.py +++ b/scripts/whitelist.py @@ -237,3 +237,6 @@ _.decoding_error_count _.ErrorHandler # unused class (aea/error_handler/scaffold.py:27) ensure_dir # unused function (aea/helpers/base.py:561) +AgentRecord # unused class (aea/helpers/acn/agent_record.py:49) +is_valid_for # unused method (aea/helpers/acn/agent_record.py:96) +Uri # unused class (aea/helpers/acn/uri.py:26) \ No newline at end of file From d369d3f845a29cb2048c3c47364a4eb1c3d104af Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Thu, 17 Dec 2020 19:01:11 +0000 Subject: [PATCH 020/204] Minor - black also check script --- scripts/whitelist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/whitelist.py b/scripts/whitelist.py index 7a9e14ed95..63bc24b13c 100644 --- a/scripts/whitelist.py +++ b/scripts/whitelist.py @@ -239,4 +239,4 @@ ensure_dir # unused function (aea/helpers/base.py:561) AgentRecord # unused class (aea/helpers/acn/agent_record.py:49) is_valid_for # unused method (aea/helpers/acn/agent_record.py:96) -Uri # unused class (aea/helpers/acn/uri.py:26) \ No newline at end of file +Uri # unused class (aea/helpers/acn/uri.py:26) From 0e0b39a32ca15e2531112df8b596c4b3432c089b Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Fri, 18 Dec 2020 16:23:59 +0300 Subject: [PATCH 021/204] multiagenmanager: same project add error handling --- aea/manager.py | 13 +++++++++---- tests/test_manager.py | 12 ++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/aea/manager.py b/aea/manager.py index 8b4e896180..7c1bac4bb1 100644 --- a/aea/manager.py +++ b/aea/manager.py @@ -27,7 +27,7 @@ from pathlib import Path from shutil import rmtree from threading import Thread -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Set from aea.aea import AEA from aea.aea_builder import AEABuilder @@ -158,6 +158,7 @@ def __init__( self._was_working_dir_created = False self._is_running = False self._projects: Dict[PublicId, Project] = {} + self._projects_set: Set[PublicId] = set() self._keys_dir = os.path.abspath(os.path.join(self.working_dir, "keys")) self._agents: Dict[str, AgentAlias] = {} self._agents_tasks: Dict[str, AgentRunAsyncTask] = {} @@ -310,8 +311,12 @@ def add_project( :param local: whether or not to fetch from local registry. :param restore: bool flag for restoring already fetched agent. """ - if public_id in self._projects: - raise ValueError(f"Project {public_id} was already added!") + if public_id.to_latest() in self._projects_set: + raise ValueError( + f"The project ({public_id.author}/{public_id.name}) was already added!" + ) + self._projects_set.add(public_id.to_latest()) + self._projects[public_id] = Project.load( self.working_dir, public_id, @@ -327,13 +332,13 @@ def remove_project( """Remove agent project.""" if public_id not in self._projects: raise ValueError(f"Project {public_id} is not present!") - if self._projects[public_id].agents: raise ValueError( f"Can not remove projects with aliases exists: {self._projects[public_id].agents}" ) project = self._projects.pop(public_id) + self._projects_set.remove(public_id.to_latest()) if not keep_files: project.remove() diff --git a/tests/test_manager.py b/tests/test_manager.py index e12c13f934..6b747d8c6e 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -362,6 +362,18 @@ def test_list_agents_info_positive(self): ] assert result == expected_result + def test_add_same_project_versions(self): + """Test add the same project twice.""" + self.manager.start_manager() + + self.manager.add_project(self.project_public_id, local=True) + with pytest.raises( + ValueError, match=r"The project \(fetchai/my_first_aea\) was already added!" + ): + self.manager.add_project( + PublicId.from_str("fetchai/my_first_aea:0.15.0"), local=False + ) + class TestMultiAgentManagerThreadedMode(TestMultiAgentManagerAsyncMode): """Tests for MultiAgentManager in threaded mode.""" From 5257fe802d27f13e678eacb2debff3839efe0ec8 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Fri, 18 Dec 2020 16:59:43 +0300 Subject: [PATCH 022/204] fix --- aea/manager.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/aea/manager.py b/aea/manager.py index 7c1bac4bb1..cb76e5e983 100644 --- a/aea/manager.py +++ b/aea/manager.py @@ -158,7 +158,7 @@ def __init__( self._was_working_dir_created = False self._is_running = False self._projects: Dict[PublicId, Project] = {} - self._projects_set: Set[PublicId] = set() + self._versionless_projects_set: Set[PublicId] = set() self._keys_dir = os.path.abspath(os.path.join(self.working_dir, "keys")) self._agents: Dict[str, AgentAlias] = {} self._agents_tasks: Dict[str, AgentRunAsyncTask] = {} @@ -311,11 +311,12 @@ def add_project( :param local: whether or not to fetch from local registry. :param restore: bool flag for restoring already fetched agent. """ - if public_id.to_latest() in self._projects_set: + if public_id.to_any() in self._versionless_projects_set: raise ValueError( f"The project ({public_id.author}/{public_id.name}) was already added!" ) - self._projects_set.add(public_id.to_latest()) + + self._versionless_projects_set.add(public_id.to_any()) self._projects[public_id] = Project.load( self.working_dir, @@ -332,13 +333,14 @@ def remove_project( """Remove agent project.""" if public_id not in self._projects: raise ValueError(f"Project {public_id} is not present!") + if self._projects[public_id].agents: raise ValueError( f"Can not remove projects with aliases exists: {self._projects[public_id].agents}" ) project = self._projects.pop(public_id) - self._projects_set.remove(public_id.to_latest()) + self._versionless_projects_set.remove(public_id.to_any()) if not keep_files: project.remove() @@ -348,7 +350,7 @@ def list_projects(self) -> List[PublicId]: """ List all agents projects added. - :return: lit of public ids of projects + :return: list of public ids of projects """ return list(self._projects.keys()) From 1215222f4602fd0904a46d8ac618ac816a84a885 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 21 Dec 2020 20:07:06 +0100 Subject: [PATCH 023/204] add cert request class --- aea/configurations/base.py | 120 ++++++++++++++++++ .../schemas/connection-config_schema.json | 3 + aea/configurations/schemas/definitions.json | 28 ++++ tests/test_configurations/test_base.py | 62 +++++++++ 4 files changed, 213 insertions(+) diff --git a/aea/configurations/base.py b/aea/configurations/base.py index 9b3210b6eb..37bcf559f0 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -18,11 +18,13 @@ # ------------------------------------------------------------------------------ """Classes to handle AEA configurations.""" +import datetime import functools import pprint import re from abc import ABC, abstractmethod from collections import OrderedDict +from contextlib import suppress from copy import copy, deepcopy from enum import Enum from pathlib import Path @@ -289,6 +291,120 @@ def dependencies_to_json(dependencies: Dependencies) -> Dict[str, Dict]: return result +class CertRequest: + """Certificate request for proof of representation.""" + + def __init__( + self, + public_key: str, + identifier: SimpleIdOrStr, + not_before: str, + not_after: str, + path: str, + ): + """ + Initialize the certificate request. + + :param public_key: the public key, or the key id. + :param identifier: certificate identifier. + :param not_before: specify the lower bound for certificate vailidity. + Must follow the format: 'YYYY-MM-DDTHH:MM:SS' + :param not_before: specify the lower bound for certificate vailidity. + Must follow the format: 'YYYY-MM-DDTHH:MM:SS' + :param path: the path to the certificate. + """ + self._key_identifier: Optional[str] = None + self._public_key: Optional[str] = None + self._identifier = SimpleId(identifier) + self._not_before = self._parse_datetime(not_before) + self._not_after = self._parse_datetime(not_after) + self._path = Path(path) + + self._parse_public_key(public_key) + self._check_validation_boundaries() + + @staticmethod + def _parse_datetime(datetime_str: str) -> datetime.datetime: + """ + Parse datetime string. + + It is expected to follow ISO 8601. + The result is interpreted with UTC timezone. + + :param datetime_str: the input to parse. + :return: a datetime.datetime instance. + """ + result = datetime.datetime.fromisoformat(datetime_str) + enforce(result.microsecond == 0, "Microsecond field not allowed.") + return result.astimezone(tz=datetime.timezone.utc) + + def _check_validation_boundaries(self): + """ + Check the validation boundaries are consistent. + + Namely, that not_before < not_after. + """ + enforce( + self._not_before < self._not_after, + f"Inconsistent certificate validity period: 'not_before' field '{self._not_before.isoformat()}' is not before than 'not_after' field '{self._not_after.isoformat()}'", + ValueError, + ) + + def _parse_public_key(self, public_key_str: str) -> None: + """ + Parse public key from string. + + It first tries to parse it as an identifier, + and in case of failure as a sequence of hexadecimals, starting with "0x". + """ + with suppress(ValueError): + # if this raises ValueError, we don't return + self._key_identifier = SimpleId(public_key_str) + return + + with suppress(ValueError): + # this raises ValueError if the input is not a valid hexadecimal string. + int(public_key_str, 16) + self._public_key = public_key_str + return + + enforce( + False, + f"Public key field '{public_key_str}' is neither a valid identifier nor an address.", + exception_class=ValueError, + ) + + @property + def public_key(self) -> Optional[str]: + """Get the public key.""" + return self._public_key + + @property + def key_identifier(self) -> Optional[str]: + """Get the key identifier.""" + return self._key_identifier + + @property + def identifier(self) -> str: + """Get the identifier.""" + return self._identifier + + @property + def not_before(self) -> datetime.datetime: + """Get the not_before field.""" + return self._not_before + + @property + def not_after(self) -> datetime.datetime: + """Get the not_after field.""" + return self._not_after + + @property + def path(self) -> Path: + """Get the path""" + return self._path + + VersionInfoClass = semver.VersionInfo PackageVersionLike = Union[str, semver.VersionInfo] @@ -1227,6 +1343,7 @@ def __init__( description: str = "", connection_id: Optional[PublicId] = None, is_abstract: bool = False, + cert_requests: Optional[List[CertRequest]] = None, **config, ): """Initialize a connection configuration object.""" @@ -1275,6 +1392,7 @@ def __init__( self.description = description self.config = config if len(config) > 0 else {} self.is_abstract = is_abstract + self.cert_requests = cert_requests or [] @property def package_dependencies(self) -> Set[ComponentId]: @@ -1338,6 +1456,7 @@ def from_json(cls, obj: Dict): dependencies = dependencies_from_json(obj.get("dependencies", {})) protocols = {PublicId.from_str(id_) for id_ in obj.get(PROTOCOLS, set())} connections = {PublicId.from_str(id_) for id_ in obj.get(CONNECTIONS, set())} + cert_requests = obj.get("cert_requests", []) return ConnectionConfig( name=cast(str, obj.get("name")), author=cast(str, obj.get("author")), @@ -1358,6 +1477,7 @@ def from_json(cls, obj: Dict): dependencies=cast(Dependencies, dependencies), description=cast(str, obj.get("description", "")), is_abstract=obj.get("is_abstract", False), + cert_requests=cert_requests, **cast(dict, obj.get("config", {})), ) diff --git a/aea/configurations/schemas/connection-config_schema.json b/aea/configurations/schemas/connection-config_schema.json index ee94c0eaed..cd76a04787 100644 --- a/aea/configurations/schemas/connection-config_schema.json +++ b/aea/configurations/schemas/connection-config_schema.json @@ -87,6 +87,9 @@ }, "is_abstract": { "$ref": "skill-config_schema.json#/properties/is_abstract" + }, + "cert_requests": { + "$ref": "definitions.json#/definitions/cert_requests" } } } diff --git a/aea/configurations/schemas/definitions.json b/aea/configurations/schemas/definitions.json index b71039b5c5..26cb190a2b 100644 --- a/aea/configurations/schemas/definitions.json +++ b/aea/configurations/schemas/definitions.json @@ -150,6 +150,34 @@ }, "keep_terminal_state_dialogues": { "type": "boolean" + }, + "cert_requests": { + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "definitions.json#/definitions/cert_request" + } + }, + "cert_request": { + "type": "object", + "additionalProperties": false, + "properties": { + "public_key": { + "type": "string" + }, + "identifier": { + "$ref": "definitions.json#/definitions/resource_name" + }, + "not_before": { + "type": "string" + }, + "not_after": { + "type": "string" + }, + "path": { + "type": "string" + } + } } } } diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index 29d45eb061..9d715023fe 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -18,6 +18,7 @@ # ------------------------------------------------------------------------------ """This module contains the tests for the aea.configurations.base module.""" +import datetime import re from copy import copy from pathlib import Path @@ -30,6 +31,7 @@ from aea.configurations.base import ( AgentConfig, CRUDCollection, + CertRequest, ComponentId, ComponentType, ConnectionConfig, @@ -982,3 +984,63 @@ def test_check_public_id_consistency_negative(): with pytest.raises(ValueError, match=f"Directory {random_dir_name} is not valid."): component_configuration = ProtocolConfig("name", "author") component_configuration.check_public_id_consistency(Path(random_dir_name)) + + +class BaseTestCertRequestError: + """Test errors when instantiating a CertRequest object.""" + + PUBLIC_KEY = "a_public_key" + IDENTIFIER = "an_identifier" + NOT_BEFORE = datetime.datetime.now().isoformat() + NOT_AFTER = datetime.datetime.now().isoformat() + PATH = "some/path" + ERROR_MESSAGE_PATTERN = "" + + def test_error(self): + """Test error during instantiation..""" + with pytest.raises(ValueError, match=self.ERROR_MESSAGE_PATTERN): + CertRequest( + self.PUBLIC_KEY, + self.IDENTIFIER, + self.NOT_BEFORE, + self.NOT_AFTER, + self.PATH, + ) + + +class TestCertRequestBadPublicKey(BaseTestCertRequestError): + """Test instantiation of CertRequest class with bad public key.""" + + PUBLIC_KEY = "0a_bad_identifier" + ERROR_MESSAGE_PATTERN = "Public key field '0a_bad_identifier' is neither a valid identifier nor an address." + + +class TestCertRequestBadIdentifier(BaseTestCertRequestError): + """Test instantiation of CertRequest class with bad identifier.""" + + IDENTIFIER = "0bad_identifier" + ERROR_MESSAGE_PATTERN = ( + "Value 0bad_identifier does not match the regular expression.*" + ) + + +class TestCertRequestBadNotBefore(BaseTestCertRequestError): + """Test instantiation of CertRequest class with bad not_before date.""" + + NOT_BEFORE = "bad-formatted-date" + ERROR_MESSAGE_PATTERN = "Invalid isoformat string: 'bad-formatted-date'" + + +class TestCertRequestBadNotAfter(BaseTestCertRequestError): + """Test instantiation of CertRequest class with bad not_after date.""" + + NOT_AFTER = "bad-formatted-date" + ERROR_MESSAGE_PATTERN = "Invalid isoformat string: 'bad-formatted-date'" + + +class TestCertRequestInconsistentDates(BaseTestCertRequestError): + """Test instantiation of CertRequest class when not_before >= not_after""" + + NOT_BEFORE = "1954-06-07T00:00:00+00:00" + NOT_AFTER = "1900-01-01T00:00:01+00:00" + ERROR_MESSAGE_PATTERN = r"Inconsistent certificate validity period: 'not_before' field '1954-06-07T00:00:00\+00:00' is not before than 'not_after' field '1900-01-01T00:00:01\+00:00'" From aab45444caafdbca79876f6eaf3764b81016926d Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 21 Dec 2020 22:27:34 +0100 Subject: [PATCH 024/204] integrate with connection config --- aea/configurations/base.py | 51 ++++++++++++++++++++++++-- tests/test_configurations/test_base.py | 46 ++++++++++++++++++++++- 2 files changed, 92 insertions(+), 5 deletions(-) diff --git a/aea/configurations/base.py b/aea/configurations/base.py index 37bcf559f0..676475058c 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -27,6 +27,7 @@ from contextlib import suppress from copy import copy, deepcopy from enum import Enum +from operator import attrgetter from pathlib import Path from typing import ( Any, @@ -315,7 +316,9 @@ def __init__( """ self._key_identifier: Optional[str] = None self._public_key: Optional[str] = None - self._identifier = SimpleId(identifier) + self._identifier = str(SimpleId(identifier)) + self._not_before_string = not_before + self._not_after_string = not_after self._not_before = self._parse_datetime(not_before) self._not_after = self._parse_datetime(not_after) self._path = Path(path) @@ -359,7 +362,7 @@ def _parse_public_key(self, public_key_str: str) -> None: """ with suppress(ValueError): # if this raises ValueError, we don't return - self._key_identifier = SimpleId(public_key_str) + self._key_identifier = str(SimpleId(public_key_str)) return with suppress(ValueError): @@ -404,6 +407,38 @@ def path(self) -> Path: """Get the path""" return self._path + @property + def json(self) -> Dict: + """Compute the JSON representation.""" + result = dict( + identifier=self.identifier, + not_before=self._not_before_string, + not_after=self._not_after_string, + path=self.path, + ) + if self.public_key is not None: + result["public_key"] = self.public_key + elif self.key_identifier is not None: + result["public_key"] = self.key_identifier + return result + + @classmethod + def from_json(cls, obj: Dict) -> "CertRequest": + """Compute the JSON representation.""" + return cls(**obj) + + def __eq__(self, other): + """Check equality.""" + return ( + isinstance(other, CertRequest) + and self.identifier == other.identifier + and self.public_key == other.public_key + and self.key_identifier == other.key_identifier + and self.not_after == other.not_after + and self.not_before == other.not_before + and self.path == other.path + ) + VersionInfoClass = semver.VersionInfo PackageVersionLike = Union[str, semver.VersionInfo] @@ -1438,6 +1473,9 @@ def json(self) -> Dict: "is_abstract": self.is_abstract, } ) + + if self.cert_requests is not None: + result["cert_requests"] = list(map(attrgetter("json"), self.cert_requests)) if self.build_entrypoint: result["build_entrypoint"] = self.build_entrypoint if self.build_directory: @@ -1456,7 +1494,14 @@ def from_json(cls, obj: Dict): dependencies = dependencies_from_json(obj.get("dependencies", {})) protocols = {PublicId.from_str(id_) for id_ in obj.get(PROTOCOLS, set())} connections = {PublicId.from_str(id_) for id_ in obj.get(CONNECTIONS, set())} - cert_requests = obj.get("cert_requests", []) + cert_requests = ( + [ + CertRequest.from_json(cert_request_json) + for cert_request_json in obj["cert_requests"] + ] + if "cert_requests" in obj + else None + ) return ConnectionConfig( name=cast(str, obj.get("name")), author=cast(str, obj.get("author")), diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index 9d715023fe..7860c02bcf 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -991,8 +991,8 @@ class BaseTestCertRequestError: PUBLIC_KEY = "a_public_key" IDENTIFIER = "an_identifier" - NOT_BEFORE = datetime.datetime.now().isoformat() - NOT_AFTER = datetime.datetime.now().isoformat() + NOT_BEFORE = "2020-01-01T00:00:00+00:00" + NOT_AFTER = "2020-01-02T00:00:00+00:00" PATH = "some/path" ERROR_MESSAGE_PATTERN = "" @@ -1044,3 +1044,45 @@ class TestCertRequestInconsistentDates(BaseTestCertRequestError): NOT_BEFORE = "1954-06-07T00:00:00+00:00" NOT_AFTER = "1900-01-01T00:00:01+00:00" ERROR_MESSAGE_PATTERN = r"Inconsistent certificate validity period: 'not_before' field '1954-06-07T00:00:00\+00:00' is not before than 'not_after' field '1900-01-01T00:00:01\+00:00'" + + +class TestCertRequestInstantiation: + """Test (successful) instantiation of CertRequest class.""" + + @classmethod + def setup_class(cls): + """Set up class.""" + cls.expected_public_key = "public_key" + cls.expected_identifier = "identifier" + cls.not_before = "2020-01-01T00:00:00+00:00" + cls.not_after = "2020-01-02T00:00:00+00:00" + cls.expected_path = "some/path" + cls.cert_request = CertRequest( + cls.expected_public_key, + cls.expected_identifier, + cls.not_before, + cls.not_after, + cls.expected_path, + ) + + def test_instantiation(self): + """Test instantiation.""" + assert self.cert_request.public_key is None + assert self.cert_request.key_identifier == self.expected_public_key + assert self.cert_request.identifier == self.expected_identifier + + expected_not_before = datetime.datetime( + 2020, 1, 1, 0, 0, 0, 0, datetime.timezone.utc + ) + assert self.cert_request.not_before == expected_not_before + + expected_not_after = datetime.datetime( + 2020, 1, 2, 0, 0, 0, 0, datetime.timezone.utc + ) + assert self.cert_request.not_after == expected_not_after + + assert self.cert_request.path == Path(self.expected_path) + + def test_from_to_json(self): + """Test from-to json methods.""" + assert self.cert_request == self.cert_request.from_json(self.cert_request.json) From bcec3121bd722eea1d7ea0da746d440329b1d58f Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 21 Dec 2020 22:54:56 +0100 Subject: [PATCH 025/204] add tests on hexadecimal public key --- tests/test_configurations/test_base.py | 28 ++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index 7860c02bcf..485d8a3196 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -1046,13 +1046,17 @@ class TestCertRequestInconsistentDates(BaseTestCertRequestError): ERROR_MESSAGE_PATTERN = r"Inconsistent certificate validity period: 'not_before' field '1954-06-07T00:00:00\+00:00' is not before than 'not_after' field '1900-01-01T00:00:01\+00:00'" -class TestCertRequestInstantiation: +class BaseTestCertRequestInstantiation: """Test (successful) instantiation of CertRequest class.""" + PUBLIC_KEY = "" + EXPECTED_PUBLIC_KEY = "" + EXPECTED_KEY_IDENTIFIER = "" + @classmethod def setup_class(cls): """Set up class.""" - cls.expected_public_key = "public_key" + cls.expected_public_key = cls.PUBLIC_KEY cls.expected_identifier = "identifier" cls.not_before = "2020-01-01T00:00:00+00:00" cls.not_after = "2020-01-02T00:00:00+00:00" @@ -1067,8 +1071,8 @@ def setup_class(cls): def test_instantiation(self): """Test instantiation.""" - assert self.cert_request.public_key is None - assert self.cert_request.key_identifier == self.expected_public_key + assert self.cert_request.public_key == self.EXPECTED_PUBLIC_KEY + assert self.cert_request.key_identifier == self.EXPECTED_KEY_IDENTIFIER assert self.cert_request.identifier == self.expected_identifier expected_not_before = datetime.datetime( @@ -1086,3 +1090,19 @@ def test_instantiation(self): def test_from_to_json(self): """Test from-to json methods.""" assert self.cert_request == self.cert_request.from_json(self.cert_request.json) + + +class TestCertRequestInstantiationWithKeyIdentifier(BaseTestCertRequestInstantiation): + """Test (successful) instantiation of CertRequest class.""" + + PUBLIC_KEY = "public_key" + EXPECTED_PUBLIC_KEY = None + EXPECTED_KEY_IDENTIFIER = PUBLIC_KEY + + +class TestCertRequestInstantiationWithKeyHex(BaseTestCertRequestInstantiation): + """Test (successful) instantiation of CertRequest class.""" + + PUBLIC_KEY = "0xABCDE12345" + EXPECTED_PUBLIC_KEY = "0xABCDE12345" + EXPECTED_KEY_IDENTIFIER = None From d4cac28259912265d3c8bd9c5c5664518abac0d8 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 21 Dec 2020 23:03:08 +0100 Subject: [PATCH 026/204] fix static type checks --- tests/test_configurations/test_base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index 485d8a3196..ec5727f191 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -22,6 +22,7 @@ import re from copy import copy from pathlib import Path +from typing import Optional from unittest import TestCase, mock import pytest @@ -1049,9 +1050,9 @@ class TestCertRequestInconsistentDates(BaseTestCertRequestError): class BaseTestCertRequestInstantiation: """Test (successful) instantiation of CertRequest class.""" - PUBLIC_KEY = "" - EXPECTED_PUBLIC_KEY = "" - EXPECTED_KEY_IDENTIFIER = "" + PUBLIC_KEY: Optional[str] = "" + EXPECTED_PUBLIC_KEY: Optional[str] = "" + EXPECTED_KEY_IDENTIFIER: Optional[str] = "" @classmethod def setup_class(cls): From a84644bbbb7872d1d335b35f6fe731588ac8b5fc Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 21 Dec 2020 23:49:58 +0100 Subject: [PATCH 027/204] add test with connection yaml --- aea/configurations/base.py | 26 ++++++++++++--------- aea/connections/scaffold/connection.yaml | 1 + packages/hashes.csv | 2 +- tests/data/dummy_connection/connection.yaml | 11 +++++++++ tests/data/hashes.csv | 2 +- tests/test_configurations/test_base.py | 4 ++-- 6 files changed, 31 insertions(+), 15 deletions(-) diff --git a/aea/configurations/base.py b/aea/configurations/base.py index 676475058c..3c4a18f813 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -299,8 +299,8 @@ def __init__( self, public_key: str, identifier: SimpleIdOrStr, - not_before: str, - not_after: str, + not_before: Union[str, datetime.datetime], + not_after: Union[str, datetime.datetime], path: str, ): """ @@ -309,9 +309,9 @@ def __init__( :param public_key: the public key, or the key id. :param identifier: certificate identifier. :param not_before: specify the lower bound for certificate vailidity. - Must follow the format: 'YYYY-MM-DDTHH:MM:SS' + If it is a string, it must follow the format: 'YYYY-MM-DDTHH:MM:SS' :param not_before: specify the lower bound for certificate vailidity. - Must follow the format: 'YYYY-MM-DDTHH:MM:SS' + if it is a string, it must follow the format: 'YYYY-MM-DDTHH:MM:SS' :param path: the path to the certificate. """ self._key_identifier: Optional[str] = None @@ -327,19 +327,22 @@ def __init__( self._check_validation_boundaries() @staticmethod - def _parse_datetime(datetime_str: str) -> datetime.datetime: + def _parse_datetime(obj: Union[str, datetime.datetime]) -> datetime.datetime: """ Parse datetime string. It is expected to follow ISO 8601. - The result is interpreted with UTC timezone. - :param datetime_str: the input to parse. + :param obj: the input to parse. :return: a datetime.datetime instance. """ - result = datetime.datetime.fromisoformat(datetime_str) + result = ( + obj + if isinstance(obj, datetime.datetime) + else datetime.datetime.fromisoformat(obj) + ) enforce(result.microsecond == 0, "Microsecond field not allowed.") - return result.astimezone(tz=datetime.timezone.utc) + return result def _check_validation_boundaries(self): """ @@ -414,7 +417,7 @@ def json(self) -> Dict: identifier=self.identifier, not_before=self._not_before_string, not_after=self._not_after_string, - path=self.path, + path=str(self.path), ) if self.public_key is not None: result["public_key"] = self.public_key @@ -1427,7 +1430,7 @@ def __init__( self.description = description self.config = config if len(config) > 0 else {} self.is_abstract = is_abstract - self.cert_requests = cert_requests or [] + self.cert_requests = cert_requests @property def package_dependencies(self) -> Set[ComponentId]: @@ -1496,6 +1499,7 @@ def from_json(cls, obj: Dict): connections = {PublicId.from_str(id_) for id_ in obj.get(CONNECTIONS, set())} cert_requests = ( [ + # notice: yaml.load resolves datetimes strings to datetime.datetime objects CertRequest.from_json(cert_request_json) for cert_request_json in obj["cert_requests"] ] diff --git a/aea/connections/scaffold/connection.yaml b/aea/connections/scaffold/connection.yaml index acaa94642a..5fb2316cb1 100644 --- a/aea/connections/scaffold/connection.yaml +++ b/aea/connections/scaffold/connection.yaml @@ -20,3 +20,4 @@ excluded_protocols: [] restricted_to_protocols: [] dependencies: {} is_abstract: false +cert_requests: [] diff --git a/packages/hashes.csv b/packages/hashes.csv index f20cd2319f..0cc67909ae 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -38,7 +38,7 @@ fetchai/connections/p2p_libp2p,QmSDodHZsSwTvfCvnSHV9D82Uo1fcHJVqy9yBfqkHzewAr fetchai/connections/p2p_libp2p_client,QmQndwAPhGHTpe1J8c6Q6jabV2KfkND7jsYiEF9Ysmyby3 fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T -fetchai/connections/scaffold,QmQcfdXmL248RFWGphcYeJt7jVNTN7fCJAgwZbYpdB6PK4 +fetchai/connections/scaffold,QmaCupER4vedakPPLzn2Rw9YHPmqSPaZh5xnqi6xVxQHvn fetchai/connections/soef,QmcsCsU9cYKozwR2Ag2V38xaCLJwTVfWNSmHmQTLqZki9j fetchai/connections/stub,QmUviVXuCuBu1c6PPioNg3htJ9aatpATDNDJ8mapVriQM8 fetchai/connections/tcp,QmcAEg3XkYFZ6oKqYSWpJodiakxDAVYadwcczpqeUjo7op diff --git a/tests/data/dummy_connection/connection.yaml b/tests/data/dummy_connection/connection.yaml index 36de53172e..5a007b5b5a 100644 --- a/tests/data/dummy_connection/connection.yaml +++ b/tests/data/dummy_connection/connection.yaml @@ -27,3 +27,14 @@ dependencies: dep4: version: <=1.11.12b2 is_abstract: false +cert_requests: +- identifier: cert_id + not_after: '2020-01-02T00:00:00' + not_before: '2020-01-01T00:00:00' + path: some/path_1 + public_key: key_id +- identifier: cert_id + not_after: '2020-01-02T00:00:00' + not_before: '2020-01-01T00:00:00' + path: some/path_2 + public_key: '0xABCDEF123456' diff --git a/tests/data/hashes.csv b/tests/data/hashes.csv index 52bf7dcf05..39a639de56 100644 --- a/tests/data/hashes.csv +++ b/tests/data/hashes.csv @@ -1,6 +1,6 @@ dummy_author/agents/dummy_aea,QmPWxJVM9xnkRFtB41Wr1Xi2NGKpeRgDXUzbAzA8rferGh dummy_author/skills/dummy_skill,QmSmzcjGNhiz2KpSE6wSi19AeesCVjzupJm1NpXYJVnJuZ -fetchai/connections/dummy_connection,QmYCTgUKPqyCYGBjkBKXDjDAGmgVtn6Yg3DJ6nwk9KRs7n +fetchai/connections/dummy_connection,QmbHGef6NEiuZUJL3reGRieF415KVDnhzMZZmBmU593Bwz fetchai/contracts/dummy_contract,QmXhMKRF5upco1efaC9oNcesStEMkPWKhBUv6CK9Vngk2S fetchai/protocols/t_protocol,QmNhzDCznorQpySrvKt4F4sQFXjzF1ZecEd5inixiNavDH fetchai/protocols/t_protocol_no_ct,QmWwMwHxvuPZJaMh44Bd1d5aBzYiogep8TJswTKkRwLfBu diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index ec5727f191..832d5a0bff 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -1104,6 +1104,6 @@ class TestCertRequestInstantiationWithKeyIdentifier(BaseTestCertRequestInstantia class TestCertRequestInstantiationWithKeyHex(BaseTestCertRequestInstantiation): """Test (successful) instantiation of CertRequest class.""" - PUBLIC_KEY = "0xABCDE12345" - EXPECTED_PUBLIC_KEY = "0xABCDE12345" + PUBLIC_KEY = "0xABCDEF12345" + EXPECTED_PUBLIC_KEY = "0xABCDEF12345" EXPECTED_KEY_IDENTIFIER = None From cebe7152adbba19fdc34f3569a9c2283ed9723e0 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Tue, 22 Dec 2020 13:19:25 +0300 Subject: [PATCH 028/204] configurations refactoring: validation extracted from loader, data types extracted from base.py. create or update from json. --- aea/cli/config.py | 5 +- aea/configurations/base.py | 969 ++++------------------- aea/configurations/data_types.py | 827 +++++++++++++++++++ aea/configurations/loader.py | 180 +---- aea/configurations/validation.py | 179 +++++ aea/helpers/base.py | 30 +- tests/test_configurations/test_base.py | 11 +- tests/test_configurations/test_loader.py | 4 +- tests/test_configurations/test_schema.py | 2 +- 9 files changed, 1199 insertions(+), 1008 deletions(-) create mode 100644 aea/configurations/data_types.py create mode 100644 aea/configurations/validation.py diff --git a/aea/cli/config.py b/aea/cli/config.py index 749ff6a628..9310ded548 100644 --- a/aea/cli/config.py +++ b/aea/cli/config.py @@ -40,6 +40,7 @@ AgentConfig, ComponentId, DEFAULT_AEA_CONFIG_FILE, + PACKAGE_TYPE_TO_CONFIG_CLASS, PackageType, SkillConfig, ) @@ -220,7 +221,7 @@ def _check_set_field_name(self) -> None: top_level_key = self.json_path[0] if self.component_id: - config_class = self.component_id.package_type.configuration_class() + config_class = PACKAGE_TYPE_TO_CONFIG_CLASS[self.component_id.package_type] else: config_class = type(self.agent_config) @@ -351,7 +352,7 @@ def _dump_agent_configuration( configuration_obj = self.agent_config_loader.configuration_class.from_json( agent_configuration_object ) - self.agent_config_loader.validate(configuration_obj.json) + configuration_obj.validate_config_data(configuration_obj.json) with open(self.agent_config_file_path, "w") as file_pointer: self.agent_config_loader.dump(configuration_obj, file_pointer) except Exception as e: # pragma: nocover diff --git a/aea/configurations/base.py b/aea/configurations/base.py index 9b3210b6eb..adfa05712d 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -16,27 +16,21 @@ # limitations under the License. # # ------------------------------------------------------------------------------ - """Classes to handle AEA configurations.""" -import functools import pprint -import re -from abc import ABC, abstractmethod +from abc import ABC from collections import OrderedDict from copy import copy, deepcopy -from enum import Enum from pathlib import Path from typing import ( Any, Collection, Dict, FrozenSet, - Generic, List, Optional, Sequence, Set, - Tuple, Type, TypeVar, Union, @@ -44,220 +38,49 @@ ) import packaging -import semver from packaging.specifiers import SpecifierSet from packaging.version import Version -from urllib3.util import Url, parse_url from aea.__version__ import __version__ as __aea_version__ from aea.configurations.constants import ( - AGENT, - CONNECTION, CONNECTIONS, - CONTRACT, CONTRACTS, DEFAULT_AEA_CONFIG_FILE, DEFAULT_CONNECTION_CONFIG_FILE, DEFAULT_CONTRACT_CONFIG_FILE, DEFAULT_FINGERPRINT_IGNORE_PATTERNS, - DEFAULT_GIT_REF, DEFAULT_LICENSE, DEFAULT_PROTOCOL_CONFIG_FILE, DEFAULT_REGISTRY_NAME, DEFAULT_SKILL_CONFIG_FILE, DEFAULT_VERSION, PACKAGE_PUBLIC_ID_VAR_NAME, - PROTOCOL, PROTOCOLS, - SKILL, SKILLS, ) -from aea.exceptions import enforce -from aea.helpers.base import ( - RegexConstrainedString, - STRING_LENGTH_LIMIT, - SimpleId, - SimpleIdOrStr, - load_module, - recursive_update, +from aea.configurations.data_types import ( + CRUDCollection, + ComponentId, + ComponentType, + Dependencies, + Dependency, + JSONSerializable, + PackageId, + PackageType, + PackageVersion, + PublicId, ) +from aea.configurations.validation import ConfigValidator +from aea.exceptions import enforce +from aea.helpers.base import SimpleId, SimpleIdOrStr, load_module, recursive_update from aea.helpers.ipfs.base import IPFSHashOnly -T = TypeVar("T") - - -class PyPIPackageName(RegexConstrainedString): - """A PyPI Package name.""" - - REGEX = re.compile(r"^([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9._-]*[A-Za-z0-9])$") - - -class GitRef(RegexConstrainedString): - """ - A Git reference. - - It can be a branch name, a commit hash or a tag. - """ - - REGEX = re.compile(r"^[A-Za-z0-9/.\-_]+$") - - -class Dependency: - """ - This class represents a PyPI dependency. - - It contains the following information: - - version: a version specifier(s) (e.g. '==0.1.0'). - - index: the PyPI index where to download the package from (default: https://pypi.org) - - git: the URL to the Git repository (e.g. https://github.com/fetchai/agents-aea.git) - - ref: either the branch name, the tag, the commit number or a Git reference (default: 'master'.) - - If the 'git' field is set, the 'version' field will be ignored. - These fields will be forwarded to the 'pip' command. - """ - - def __init__( - self, - name: Union[PyPIPackageName, str], - version: Union[str, SpecifierSet] = "", - index: Optional[Union[str, Url]] = None, - git: Optional[Union[str, Url]] = None, - ref: Optional[Union[GitRef, str]] = None, - ): - """ - Initialize a PyPI dependency. - - :param name: the package name. - :param version: the specifier set object - :param index: the URL to the PyPI server. - :param git: the URL to a git repository. - :param ref: the Git reference (branch/commit/tag). - """ - self._name: PyPIPackageName = PyPIPackageName(name) - self._version: SpecifierSet = self._parse_version(version) - self._index: Optional[Url] = self._parse_url( - index - ) if index is not None else None - self._git: Optional[Url] = self._parse_url(git) if git is not None else None - self._ref: Optional[GitRef] = GitRef(ref) if ref is not None else None +# for tests +_ = [PackageId, PackageVersion] - @property - def name(self) -> str: - """Get the name.""" - return str(self._name) - - @property - def version(self) -> str: - """Get the version.""" - return str(self._version) - @property - def index(self) -> Optional[str]: - """Get the index.""" - return str(self._index) if self._index else None - - @property - def git(self) -> Optional[str]: - """Get the git.""" - return str(self._git) if self._git else None - - @property - def ref(self) -> Optional[str]: - """Get the ref.""" - return str(self._ref) if self._ref else None - - @staticmethod - def _parse_version(version: Union[str, SpecifierSet]) -> SpecifierSet: - """ - Parse a version specifier set. - - :param version: the version, a string or a SpecifierSet instance. - :return: the SpecifierSet instance. - """ - return version if isinstance(version, SpecifierSet) else SpecifierSet(version) - - @staticmethod - def _parse_url(url: Union[str, Url]) -> Url: - """ - Parse an URL. - - :param url: the URL, in either string or an urllib3.Url instance. - :return: the urllib3.Url instance. - """ - return url if isinstance(url, Url) else parse_url(url) - - @classmethod - def from_json(cls, obj: Dict[str, Dict[str, str]]) -> "Dependency": - """Parse a dependency object from a dictionary.""" - if len(obj) != 1: - raise ValueError(f"Only one key allowed, found {set(obj.keys())}") - name, attributes = list(obj.items())[0] - allowed_keys = {"version", "index", "git", "ref"} - not_allowed_keys = set(attributes.keys()).difference(allowed_keys) - if len(not_allowed_keys) > 0: - raise ValueError(f"Not allowed keys: {not_allowed_keys}") - - version = attributes.get("version", "") - index = attributes.get("index", None) - git = attributes.get("git", None) - ref = attributes.get("ref", None) - - return Dependency(name=name, version=version, index=index, git=git, ref=ref) - - def to_json(self) -> Dict[str, Dict[str, str]]: - """Transform the object to JSON.""" - result = {} - if self.version != "": - result["version"] = self.version - if self.index is not None: - result["index"] = self.index - if self.git is not None: - result["git"] = cast(str, self.git) - if self.ref is not None: - result["ref"] = cast(str, self.ref) - return {self.name: result} - - def get_pip_install_args(self) -> List[str]: - """Get 'pip install' arguments.""" - name = self.name - index = self.index - git_url = self.git - revision = self.ref if self.ref is not None else DEFAULT_GIT_REF - version_constraint = str(self.version) - command: List[str] = [] - if index is not None: - command += ["-i", index] - if git_url is not None: - command += ["git+" + git_url + "@" + revision + "#egg=" + name] - else: - command += [name + version_constraint] - return command - - def __str__(self) -> str: - """Get the string representation.""" - return f"{self.__class__.__name__}(name='{self.name}', version='{self.version}', index='{self.index}', git='{self.git}', ref='{self.ref}')" - - def __eq__(self, other): - """Compare with another object.""" - return ( - isinstance(other, Dependency) - and self._name == other._name - and self._version == other._version - and self._index == other._index - and self._git == other._git - and self._ref == other._ref - ) - - -Dependencies = Dict[str, Dependency] -""" -A dictionary from package name to dependency data structure (see above). -The package name must satisfy the constraints on Python packages names. - -The main advantage of having a dictionary is that we implicitly filter out dependency duplicates. -We cannot have two items with the same package name since the keys of a YAML object form a set. -""" +T = TypeVar("T") def dependencies_from_json(obj: Dict[str, Dict]) -> Dependencies: @@ -289,101 +112,6 @@ def dependencies_to_json(dependencies: Dependencies) -> Dict[str, Dict]: return result -VersionInfoClass = semver.VersionInfo -PackageVersionLike = Union[str, semver.VersionInfo] - - -@functools.total_ordering -class PackageVersion: - """A package version.""" - - _version: PackageVersionLike - - def __init__(self, version_like: PackageVersionLike): - """ - Initialize a package version. - - :param version_like: a string, os a semver.VersionInfo object. - """ - if isinstance(version_like, str) and version_like == "latest": - self._version = version_like - elif isinstance(version_like, str) and version_like == "any": - self._version = version_like - elif isinstance(version_like, str): - self._version = VersionInfoClass.parse(version_like) - elif isinstance(version_like, VersionInfoClass): - self._version = version_like - else: - raise ValueError("Version type not valid.") - - @property - def is_latest(self) -> bool: - """Check whether the version is 'latest'.""" - return isinstance(self._version, str) and self._version == "latest" - - def __str__(self) -> str: - """Get the string representation.""" - return str(self._version) - - def __eq__(self, other) -> bool: - """Check equality.""" - return isinstance(other, PackageVersion) and self._version == other._version - - def __lt__(self, other): - """Compare with another object.""" - enforce( - isinstance(other, PackageVersion), - f"Cannot compare {type(self)} with type {type(other)}.", - ) - other = cast(PackageVersion, other) - if self.is_latest or other.is_latest: - return self.is_latest < other.is_latest - return str(self) < str(other) - - -class PackageType(Enum): - """Package types.""" - - AGENT = AGENT - PROTOCOL = PROTOCOL - CONNECTION = CONNECTION - CONTRACT = CONTRACT - SKILL = SKILL - - def to_plural(self) -> str: - """ - Get the plural name. - - >>> PackageType.AGENT.to_plural() - 'agents' - >>> PackageType.PROTOCOL.to_plural() - 'protocols' - >>> PackageType.CONNECTION.to_plural() - 'connections' - >>> PackageType.SKILL.to_plural() - 'skills' - >>> PackageType.CONTRACT.to_plural() - 'contracts' - - """ - return self.value + "s" - - def configuration_class(self) -> Type["PackageConfiguration"]: - """Get the configuration class.""" - d: Dict[PackageType, Type["PackageConfiguration"]] = { - PackageType.AGENT: AgentConfig, - PackageType.PROTOCOL: ProtocolConfig, - PackageType.CONNECTION: ConnectionConfig, - PackageType.CONTRACT: ContractConfig, - PackageType.SKILL: SkillConfig, - } - return d[self] - - def __str__(self): - """Convert to string.""" - return str(self.value) - - def _get_default_configuration_file_name_from_type( item_type: Union[str, PackageType] ) -> str: @@ -404,65 +132,10 @@ def _get_default_configuration_file_name_from_type( ) -class ComponentType(Enum): - """Enum of component types supported.""" - - PROTOCOL = PROTOCOL - CONNECTION = CONNECTION - SKILL = SKILL - CONTRACT = CONTRACT - - def to_configuration_type(self) -> PackageType: - """Get package type for component type.""" - return PackageType(self.value) - - @staticmethod - def plurals() -> Collection[str]: - """ - Get the collection of type names, plural. - - >>> ComponentType.plurals() - ['protocols', 'connections', 'skills', 'contracts'] - """ - return list(map(lambda x: x.to_plural(), ComponentType)) - - def to_plural(self) -> str: - """ - Get the plural version of the component type. - - >>> ComponentType.PROTOCOL.to_plural() - 'protocols' - >>> ComponentType.CONNECTION.to_plural() - 'connections' - >>> ComponentType.SKILL.to_plural() - 'skills' - >>> ComponentType.CONTRACT.to_plural() - 'contracts' - """ - return self.value + "s" - - def __str__(self) -> str: - """Get the string representation.""" - return str(self.value) - - class ProtocolSpecificationParseError(Exception): """Exception for parsing a protocol specification file.""" -class JSONSerializable(ABC): - """Interface for JSON-serializable objects.""" - - @property - @abstractmethod - def json(self) -> Dict: - """Compute the JSON representation.""" - - @classmethod - def from_json(cls, obj: Dict): - """Build from a JSON object.""" - - class Configuration(JSONSerializable, ABC): """Configuration class.""" @@ -511,463 +184,6 @@ def ordered_json(self) -> OrderedDict: return result -class CRUDCollection(Generic[T]): - """Interface of a CRUD collection.""" - - def __init__(self): - """Instantiate a CRUD collection.""" - self._items_by_id = {} # type: Dict[str, T] - - def create(self, item_id: str, item: T) -> None: - """ - Add an item. - - :param item_id: the item id. - :param item: the item to be added. - :return: None - :raises ValueError: if the item with the same id is already in the collection. - """ - if item_id in self._items_by_id: - raise ValueError("Item with name {} already present!".format(item_id)) - self._items_by_id[item_id] = item - - def read(self, item_id: str) -> Optional[T]: - """ - Get an item by its name. - - :param item_id: the item id. - :return: the associated item, or None if the item id is not present. - """ - return self._items_by_id.get(item_id, None) - - def update(self, item_id: str, item: T) -> None: - """ - Update an existing item. - - :param item_id: the item id. - :param item: the item to be added. - :return: None - """ - self._items_by_id[item_id] = item - - def delete(self, item_id: str) -> None: - """Delete an item.""" - if item_id in self._items_by_id.keys(): - del self._items_by_id[item_id] - - def read_all(self) -> List[Tuple[str, T]]: - """Read all the items.""" - return [ # pylint: disable=unnecessary-comprehension - (k, v) for k, v in self._items_by_id.items() - ] - - def keys(self) -> Set[str]: - """Get the set of keys.""" - return set(self._items_by_id.keys()) - - -class PublicId(JSONSerializable): - """This class implement a public identifier. - - A public identifier is composed of three elements: - - author - - name - - version - - The concatenation of those three elements gives the public identifier: - - author/name:version - - >>> public_id = PublicId("author", "my_package", "0.1.0") - >>> assert public_id.author == "author" - >>> assert public_id.name == "my_package" - >>> assert public_id.version == "0.1.0" - >>> another_public_id = PublicId("author", "my_package", "0.1.0") - >>> assert hash(public_id) == hash(another_public_id) - >>> assert public_id == another_public_id - >>> latest_public_id = PublicId("author", "my_package", "latest") - >>> latest_public_id - - >>> latest_public_id.package_version.is_latest - True - """ - - AUTHOR_REGEX = fr"[a-zA-Z_][a-zA-Z0-9_]{{0,{STRING_LENGTH_LIMIT - 1}}}" - PACKAGE_NAME_REGEX = fr"[a-zA-Z_][a-zA-Z0-9_]{{0,{STRING_LENGTH_LIMIT - 1}}}" - VERSION_NUMBER_PART_REGEX = r"(0|[1-9]\d*)" - VERSION_REGEX = fr"(any|latest|({VERSION_NUMBER_PART_REGEX})\.({VERSION_NUMBER_PART_REGEX})\.({VERSION_NUMBER_PART_REGEX})(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)" - PUBLIC_ID_REGEX = fr"^({AUTHOR_REGEX})/({PACKAGE_NAME_REGEX})(:({VERSION_REGEX}))?$" - PUBLIC_ID_URI_REGEX = ( - fr"^({AUTHOR_REGEX})/({PACKAGE_NAME_REGEX})/({VERSION_REGEX})$" - ) - - ANY_VERSION = "any" - LATEST_VERSION = "latest" - - def __init__( - self, - author: SimpleIdOrStr, - name: SimpleIdOrStr, - version: Optional[PackageVersionLike] = None, - ): - """Initialize the public identifier.""" - self._author = SimpleId(author) - self._name = SimpleId(name) - self._package_version = ( - PackageVersion(version) - if version is not None - else PackageVersion(self.LATEST_VERSION) - ) - - @property - def author(self) -> str: - """Get the author.""" - return str(self._author) - - @property - def name(self) -> str: - """Get the name.""" - return str(self._name) - - @property - def version(self) -> str: - """Get the version string.""" - return str(self._package_version) - - @property - def package_version(self) -> PackageVersion: - """Get the package version object.""" - return self._package_version - - def to_any(self) -> "PublicId": - """Return the same public id, but with any version.""" - return PublicId(self.author, self.name, self.ANY_VERSION) - - def same_prefix(self, other: "PublicId") -> bool: - """Check if the other public id has the same author and name of this.""" - return self.name == other.name and self.author == other.author - - def to_latest(self) -> "PublicId": - """Return the same public id, but with latest version.""" - return PublicId(self.author, self.name, self.LATEST_VERSION) - - @classmethod - def is_valid_str(cls, public_id_string: str) -> bool: - """ - Check if a string is a public id. - - :param public_id_string: the public id in string format. - :return: bool indicating validity - """ - match = re.match(cls.PUBLIC_ID_REGEX, public_id_string) - return match is not None - - @classmethod - def from_str(cls, public_id_string: str) -> "PublicId": - """ - Initialize the public id from the string. - - >>> str(PublicId.from_str("author/package_name:0.1.0")) - 'author/package_name:0.1.0' - - A bad formatted input raises value error: - >>> PublicId.from_str("bad/formatted:input") - Traceback (most recent call last): - ... - ValueError: Input 'bad/formatted:input' is not well formatted. - - :param public_id_string: the public id in string format. - :return: the public id object. - :raises ValueError: if the string in input is not well formatted. - """ - match = re.match(cls.PUBLIC_ID_REGEX, public_id_string) - if match is None: - raise ValueError( - "Input '{}' is not well formatted.".format(public_id_string) - ) - username = match.group(1) - package_name = match.group(2) - version = match.group(3)[1:] if ":" in public_id_string else None - return PublicId(username, package_name, version) - - @classmethod - def from_uri_path(cls, public_id_uri_path: str) -> "PublicId": - """ - Initialize the public id from the string. - - >>> str(PublicId.from_uri_path("author/package_name/0.1.0")) - 'author/package_name:0.1.0' - - A bad formatted input raises value error: - >>> PublicId.from_uri_path("bad/formatted:input") - Traceback (most recent call last): - ... - ValueError: Input 'bad/formatted:input' is not well formatted. - - :param public_id_uri_path: the public id in uri path string format. - :return: the public id object. - :raises ValueError: if the string in input is not well formatted. - """ - if not re.match(cls.PUBLIC_ID_URI_REGEX, public_id_uri_path): - raise ValueError( - "Input '{}' is not well formatted.".format(public_id_uri_path) - ) - username, package_name, version = re.findall( - cls.PUBLIC_ID_URI_REGEX, public_id_uri_path - )[0][:3] - return PublicId(username, package_name, version) - - @property - def to_uri_path(self) -> str: - """ - Turn the public id into a uri path string. - - :return: uri path string - """ - return "{author}/{name}/{version}".format( - author=self.author, name=self.name, version=self.version - ) - - @property - def json(self) -> Dict: - """Compute the JSON representation.""" - return {"author": self.author, "name": self.name, "version": self.version} - - @classmethod - def from_json(cls, obj: Dict): - """Build from a JSON object.""" - return PublicId(obj["author"], obj["name"], obj["version"],) - - def __hash__(self): - """Get the hash.""" - return hash((self.author, self.name, self.version)) - - def __str__(self): - """Get the string representation.""" - return "{author}/{name}:{version}".format( - author=self.author, name=self.name, version=self.version - ) - - def __repr__(self): - """Get the representation.""" - return f"<{self}>" - - def __eq__(self, other): - """Compare with another object.""" - return ( - isinstance(other, PublicId) - and self.author == other.author - and self.name == other.name - and self.version == other.version - ) - - def __lt__(self, other): - """ - Compare two public ids. - - >>> public_id_1 = PublicId("author_1", "name_1", "0.1.0") - >>> public_id_2 = PublicId("author_1", "name_1", "0.1.1") - >>> public_id_3 = PublicId("author_1", "name_2", "0.1.0") - >>> public_id_1 > public_id_2 - False - >>> public_id_1 < public_id_2 - True - - >>> public_id_1 < public_id_3 - Traceback (most recent call last): - ... - ValueError: The public IDs author_1/name_1:0.1.0 and author_1/name_2:0.1.0 cannot be compared. Their author or name attributes are different. - - """ - if ( - isinstance(other, PublicId) - and self.author == other.author - and self.name == other.name - ): - return self.package_version < other.package_version - raise ValueError( - "The public IDs {} and {} cannot be compared. Their author or name attributes are different.".format( - self, other - ) - ) - - -class PackageId: - """A package identifier.""" - - PACKAGE_TYPE_REGEX = r"({}|{}|{}|{}|{})".format( - PackageType.AGENT, - PackageType.PROTOCOL, - PackageType.SKILL, - PackageType.CONNECTION, - PackageType.CONTRACT, - ) - PACKAGE_ID_URI_REGEX = r"{}/{}".format( - PACKAGE_TYPE_REGEX, PublicId.PUBLIC_ID_URI_REGEX[1:-1] - ) - - def __init__(self, package_type: Union[PackageType, str], public_id: PublicId): - """ - Initialize the package id. - - :param package_type: the package type. - :param public_id: the public id. - """ - self._package_type = PackageType(package_type) - self._public_id = public_id - - @property - def package_type(self) -> PackageType: - """Get the package type.""" - return self._package_type - - @property - def public_id(self) -> PublicId: - """Get the public id.""" - return self._public_id - - @property - def author(self) -> str: - """Get the author of the package.""" - return self.public_id.author - - @property - def name(self) -> str: - """Get the name of the package.""" - return self.public_id.name - - @property - def version(self) -> str: - """Get the version of the package.""" - return self.public_id.version - - @property - def package_prefix(self) -> Tuple[PackageType, str, str]: - """Get the package identifier without the version.""" - return self.package_type, self.author, self.name - - @classmethod - def from_uri_path(cls, package_id_uri_path: str) -> "PackageId": - """ - Initialize the public id from the string. - - >>> str(PackageId.from_uri_path("skill/author/package_name/0.1.0")) - '(skill, author/package_name:0.1.0)' - - A bad formatted input raises value error: - >>> PackageId.from_uri_path("very/bad/formatted:input") - Traceback (most recent call last): - ... - ValueError: Input 'very/bad/formatted:input' is not well formatted. - - :param public_id_uri_path: the public id in uri path string format. - :return: the public id object. - :raises ValueError: if the string in input is not well formatted. - """ - if not re.match(cls.PACKAGE_ID_URI_REGEX, package_id_uri_path): - raise ValueError( - "Input '{}' is not well formatted.".format(package_id_uri_path) - ) - package_type_str, username, package_name, version = re.findall( - cls.PACKAGE_ID_URI_REGEX, package_id_uri_path - )[0][:4] - package_type = PackageType(package_type_str) - public_id = PublicId(username, package_name, version) - return PackageId(package_type, public_id) - - @property - def to_uri_path(self) -> str: - """ - Turn the package id into a uri path string. - - :return: uri path string - """ - return f"{str(self.package_type)}/{self.author}/{self.name}/{self.version}" - - def __hash__(self): - """Get the hash.""" - return hash((self.package_type, self.public_id)) - - def __str__(self): - """Get the string representation.""" - return "({package_type}, {public_id})".format( - package_type=self.package_type.value, public_id=self.public_id, - ) - - def __repr__(self): - """Get the object representation in string.""" - return f"PackageId{self.__str__()}" - - def __eq__(self, other): - """Compare with another object.""" - return ( - isinstance(other, PackageId) - and self.package_type == other.package_type - and self.public_id == other.public_id - ) - - def __lt__(self, other): - """Compare two public ids.""" - return str(self) < str(other) - - -class ComponentId(PackageId): - """ - Class to represent a component identifier. - - A component id is a package id, but excludes the case when the package is an agent. - >>> pacakge_id = PackageId(PackageType.PROTOCOL, PublicId("author", "name", "0.1.0")) - >>> component_id = ComponentId(ComponentType.PROTOCOL, PublicId("author", "name", "0.1.0")) - >>> pacakge_id == component_id - True - - >>> component_id2 = ComponentId(ComponentType.PROTOCOL, PublicId("author", "name", "0.1.1")) - >>> pacakge_id == component_id2 - False - """ - - def __init__(self, component_type: Union[ComponentType, str], public_id: PublicId): - """ - Initialize the component id. - - :param component_type: the component type. - :param public_id: the public id. - """ - component_type = ComponentType(component_type) - super().__init__(component_type.to_configuration_type(), public_id) - - @property - def component_type(self) -> ComponentType: - """Get the component type.""" - return ComponentType(self.package_type.value) - - @property - def component_prefix(self) -> Tuple[ComponentType, str, str]: - """Get the component identifier without the version.""" - package_prefix = super().package_prefix - package_type, author, name = package_prefix - return ComponentType(package_type.value), author, name - - def same_prefix(self, other: "ComponentId") -> bool: - """Check if the other component id has the same type, author and name of this.""" - return ( - self.component_type == other.component_type - and self.public_id.same_prefix(other.public_id) - ) - - @property - def prefix_import_path(self) -> str: - """Get the prefix import path for this component.""" - return "packages.{}.{}.{}".format( - self.public_id.author, self.component_type.to_plural(), self.public_id.name - ) - - @property - def json(self) -> Dict: - """Get the JSON representation.""" - return dict(**self.public_id.json, type=str(self.component_type)) - - class PackageConfiguration(Configuration, ABC): """ This class represent a package configuration. @@ -983,6 +199,7 @@ class PackageConfiguration(Configuration, ABC): default_configuration_filename: str package_type: PackageType FIELDS_ALLOWED_TO_UPDATE: FrozenSet[str] = frozenset() + schema: str def __init__( self, @@ -1093,6 +310,23 @@ def update(self, data: Dict) -> None: :return: None """ + @classmethod + def validate_config_data(cls, json_data: Dict) -> None: + """Perform config validation.""" + ConfigValidator(cls.schema).validate(json_data) + + @classmethod + def from_json(cls, obj: Dict): + """Initialize from a JSON object.""" + return cls._create_or_update_from_json(obj=obj, instance=None) + + @classmethod + def _create_or_update_from_json( + cls, obj: Dict, instance=None + ) -> "PackageConfiguration": + """Create new config object or updates existing one from json data.""" + raise NotImplementedError + class ComponentConfiguration(PackageConfiguration, ABC): """Class to represent an agent component configuration.""" @@ -1204,6 +438,7 @@ class ConnectionConfig(ComponentConfiguration): default_configuration_filename = DEFAULT_CONNECTION_CONFIG_FILE package_type = PackageType.CONNECTION + schema = "connection-config_schema.json" FIELDS_ALLOWED_TO_UPDATE: FrozenSet[str] = frozenset(["config", "is_abstract"]) @@ -1327,8 +562,10 @@ def json(self) -> Dict: return result @classmethod - def from_json(cls, obj: Dict): - """Initialize from a JSON object.""" + def _create_or_update_from_json( + cls, obj: Dict, instance: Optional["ConnectionConfig"] = None + ) -> "ConnectionConfig": + """Create new config object or updates existing one from json data.""" restricted_to_protocols = obj.get("restricted_to_protocols", set()) restricted_to_protocols = { PublicId.from_str(id_) for id_ in restricted_to_protocols @@ -1338,7 +575,7 @@ def from_json(cls, obj: Dict): dependencies = dependencies_from_json(obj.get("dependencies", {})) protocols = {PublicId.from_str(id_) for id_ in obj.get(PROTOCOLS, set())} connections = {PublicId.from_str(id_) for id_ in obj.get(CONNECTIONS, set())} - return ConnectionConfig( + params = dict( name=cast(str, obj.get("name")), author=cast(str, obj.get("author")), version=cast(str, obj.get("version")), @@ -1360,6 +597,11 @@ def from_json(cls, obj: Dict): is_abstract=obj.get("is_abstract", False), **cast(dict, obj.get("config", {})), ) + if instance is None: + instance = cls(**params) # type: ignore + else: + instance.__init__(**params) # type: ignore + return instance def update(self, data: Dict) -> None: """ @@ -1380,7 +622,7 @@ class ProtocolConfig(ComponentConfiguration): default_configuration_filename = DEFAULT_PROTOCOL_CONFIG_FILE package_type = PackageType.PROTOCOL - + schema = "protocol-config_schema.json" FIELDS_ALLOWED_TO_UPDATE: FrozenSet[str] = frozenset() def __init__( @@ -1437,10 +679,12 @@ def json(self) -> Dict: return result @classmethod - def from_json(cls, obj: Dict): + def _create_or_update_from_json( + cls, obj: Dict, instance: "ProtocolConfig" = Optional[None] + ) -> "ProtocolConfig": """Initialize from a JSON object.""" dependencies = dependencies_from_json(obj.get("dependencies", {})) - return ProtocolConfig( + params = dict( name=cast(str, obj.get("name")), author=cast(str, obj.get("author")), version=cast(str, obj.get("version")), @@ -1455,6 +699,11 @@ def from_json(cls, obj: Dict): dependencies=dependencies, description=cast(str, obj.get("description", "")), ) + if instance is None: + instance = cls(**params) # type: ignore + else: + instance.__init__(**params) # type: ignore + return instance class SkillComponentConfiguration: @@ -1477,10 +726,24 @@ def json(self) -> Dict: return {"class_name": self.class_name, "args": self.args} @classmethod - def from_json(cls, obj: Dict): + def from_json(cls, obj: Dict) -> "SkillComponentConfiguration": + """Initialize from a JSON object.""" + return cls._create_or_update_from_json(obj, instance=None) + + @classmethod + def _create_or_update_from_json( + cls, obj: Dict, instance: Optional["SkillComponentConfiguration"] = None + ) -> "SkillComponentConfiguration": """Initialize from a JSON object.""" class_name = cast(str, obj.get("class_name")) - return SkillComponentConfiguration(class_name=class_name, **obj.get("args", {})) + params = dict(class_name=class_name, **obj.get("args", {})) + + if instance is None: + instance = cls(**params) # type: ignore + else: + instance.__init__(**params) # type: ignore + + return instance class SkillConfig(ComponentConfiguration): @@ -1488,6 +751,7 @@ class SkillConfig(ComponentConfiguration): default_configuration_filename = DEFAULT_SKILL_CONFIG_FILE package_type = PackageType.SKILL + schema = "skill-config_schema.json" FIELDS_ALLOWED_TO_UPDATE: FrozenSet[str] = frozenset( ["behaviours", "handlers", "models", "is_abstract"] @@ -1603,7 +867,9 @@ def json(self) -> Dict: return result @classmethod - def from_json(cls, obj: Dict): + def _create_or_update_from_json( + cls, obj: Dict, instance: "SkillConfig" = Optional[None] + ) -> "SkillConfig": """Initialize from a JSON object.""" name = cast(str, obj.get("name")) author = cast(str, obj.get("author")) @@ -1621,7 +887,7 @@ def from_json(cls, obj: Dict): skills = {PublicId.from_str(id_) for id_ in obj.get(SKILLS, set())} dependencies = dependencies_from_json(obj.get("dependencies", {})) description = cast(str, obj.get("description", "")) - skill_config = SkillConfig( + params = dict( name=name, author=author, version=version, @@ -1640,19 +906,24 @@ def from_json(cls, obj: Dict): build_directory=cast(Optional[str], obj.get("build_directory")), ) + if instance is None: + instance = cls(**params) # type: ignore + else: + instance.__init__(**params) # type: ignore + for behaviour_id, behaviour_data in obj.get("behaviours", {}).items(): behaviour_config = SkillComponentConfiguration.from_json(behaviour_data) - skill_config.behaviours.create(behaviour_id, behaviour_config) + instance.behaviours.create(behaviour_id, behaviour_config) for handler_id, handler_data in obj.get("handlers", {}).items(): handler_config = SkillComponentConfiguration.from_json(handler_data) - skill_config.handlers.create(handler_id, handler_config) + instance.handlers.create(handler_id, handler_config) for model_id, model_data in obj.get("models", {}).items(): model_config = SkillComponentConfiguration.from_json(model_data) - skill_config.models.create(model_id, model_config) + instance.models.create(model_id, model_config) - return skill_config + return instance def update(self, data: Dict) -> None: """ @@ -1707,6 +978,7 @@ class AgentConfig(PackageConfiguration): default_configuration_filename = DEFAULT_AEA_CONFIG_FILE package_type = PackageType.AGENT + schema = "aea-config_schema.json" FIELDS_ALLOWED_TO_UPDATE: FrozenSet[str] = frozenset( [ @@ -1844,11 +1116,7 @@ def component_configurations(self, d: Dict[ComponentId, Dict]) -> None: in package_type_to_set[component_id.package_type], f"Component {component_id} not declared in the agent configuration.", ) - from aea.configurations.loader import ( # pylint: disable=import-outside-toplevel,cyclic-import - ConfigLoader, - ) - - ConfigLoader.validate_component_configuration( + ConfigValidator.validate_component_configuration( component_id, component_configuration ) self._component_configurations = d @@ -1964,9 +1232,9 @@ def json(self) -> Dict: return config @classmethod - def from_json(cls, obj: Dict): - """Initialize from a JSON object.""" - agent_config = AgentConfig( + def _create_or_update_from_json(cls, obj: Dict, instance=None) -> "AgentConfig": + """Create new config object or updates existing one from json data.""" + params = dict( agent_name=cast(str, obj.get("agent_name")), author=cast(str, obj.get("author")), version=cast(str, obj.get("version")), @@ -1998,6 +1266,12 @@ def from_json(cls, obj: Dict): storage_uri=cast(str, obj.get("storage_uri")), component_configurations=None, ) + if instance is None: + # create + agent_config = AgentConfig(**params) # type: ignore + else: + agent_config = instance + agent_config.__init__(**params) # type: ignore #  parse private keys for crypto_id, path in obj.get("private_key_paths", {}).items(): @@ -2045,13 +1319,16 @@ def update(self, data: Dict) -> None: data = copy(data) # update component parts new_component_configurations: Dict = data.pop("component_configurations", {}) - result: Dict[ComponentId, Dict] = copy(self.component_configurations) + updated_component_configuration: Dict[ComponentId, Dict] = copy( + self.component_configurations + ) for component_id, obj in new_component_configurations.items(): - if component_id not in result: - result[component_id] = obj + if component_id not in updated_component_configuration: + updated_component_configuration[component_id] = obj else: - recursive_update(result[component_id], obj) - self.component_configurations = result + recursive_update(updated_component_configuration[component_id], obj) + + self.component_configurations = updated_component_configuration # update other fields for item_id, value in data.get("private_key_paths", {}).items(): @@ -2148,9 +1425,11 @@ def json(self) -> Dict: return result @classmethod - def from_json(cls, obj: Dict): + def _create_or_update_from_json( # type: ignore + cls, obj: Dict, instance: Optional["ProtocolSpecification"] = None + ) -> "ProtocolSpecification": """Initialize from a JSON object.""" - protocol_specification = ProtocolSpecification( + params = dict( name=cast(str, obj.get("name")), author=cast(str, obj.get("author")), version=cast(str, obj.get("version")), @@ -2158,6 +1437,13 @@ def from_json(cls, obj: Dict): aea_version=cast(str, obj.get("aea_version", "")), description=cast(str, obj.get("description", "")), ) + + if instance is None: + protocol_specification = cls(**params) # type: ignore + else: + protocol_specification = instance + protocol_specification.__init__(**params) # type: ignore + for speech_act, speech_act_content in obj.get("speech_acts", {}).items(): speech_act_content_config = SpeechActContentConfig.from_json( speech_act_content @@ -2173,6 +1459,7 @@ class ContractConfig(ComponentConfiguration): default_configuration_filename = DEFAULT_CONTRACT_CONFIG_FILE package_type = PackageType.CONTRACT + schema = "contract-config_schema.json" FIELDS_ALLOWED_TO_UPDATE: FrozenSet[str] = frozenset([]) @@ -2238,12 +1525,14 @@ def json(self) -> Dict: return result @classmethod - def from_json(cls, obj: Dict): + def _create_or_update_from_json( + cls, obj: Dict, instance: "ContractConfig" = Optional[None] + ) -> "ContractConfig": """Initialize from a JSON object.""" dependencies = cast( Dependencies, dependencies_from_json(obj.get("dependencies", {})) ) - return ContractConfig( + params = dict( name=cast(str, obj.get("name")), author=cast(str, obj.get("author")), version=cast(str, obj.get("version")), @@ -2262,6 +1551,11 @@ def from_json(cls, obj: Dict): ), class_name=obj.get("class_name", ""), ) + if instance is None: + instance = cls(**params) # type: ignore + else: + instance.__init__(**params) # type: ignore + return instance """The following functions are called from aea.cli.utils.""" @@ -2392,3 +1686,12 @@ def _get_public_id_from_file( module = load_module(component_configuration.prefix_import_path, path_to_file) package_public_id = getattr(module, PACKAGE_PUBLIC_ID_VAR_NAME, None) return package_public_id + + +PACKAGE_TYPE_TO_CONFIG_CLASS: Dict[PackageType, Type[PackageConfiguration]] = { + PackageType.AGENT: AgentConfig, + PackageType.PROTOCOL: ProtocolConfig, + PackageType.CONNECTION: ConnectionConfig, + PackageType.SKILL: SkillConfig, + PackageType.CONTRACT: ContractConfig, +} diff --git a/aea/configurations/data_types.py b/aea/configurations/data_types.py new file mode 100644 index 0000000000..56a548f7c6 --- /dev/null +++ b/aea/configurations/data_types.py @@ -0,0 +1,827 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""Base config data types.""" +import functools +import re +from abc import ABC, abstractmethod +from enum import Enum +from typing import ( + Collection, + Dict, + Generic, + List, + Optional, + Set, + Tuple, + TypeVar, + Union, + cast, +) + +import semver +from packaging.specifiers import SpecifierSet +from urllib3.util import Url, parse_url + +from aea.configurations.constants import ( + AGENT, + CONNECTION, + CONTRACT, + DEFAULT_GIT_REF, + PROTOCOL, + SKILL, +) +from aea.exceptions import enforce +from aea.helpers.base import ( + RegexConstrainedString, + STRING_LENGTH_LIMIT, + SimpleId, + SimpleIdOrStr, +) + + +T = TypeVar("T") + + +VersionInfoClass = semver.VersionInfo +PackageVersionLike = Union[str, semver.VersionInfo] + + +class JSONSerializable(ABC): + """Interface for JSON-serializable objects.""" + + @property + @abstractmethod + def json(self) -> Dict: + """Compute the JSON representation.""" + + @classmethod + def from_json(cls, obj: Dict): + """Build from a JSON object.""" + + +@functools.total_ordering +class PackageVersion: + """A package version.""" + + _version: PackageVersionLike + + def __init__(self, version_like: PackageVersionLike): + """ + Initialize a package version. + + :param version_like: a string, os a semver.VersionInfo object. + """ + if isinstance(version_like, str) and version_like == "latest": + self._version = version_like + elif isinstance(version_like, str) and version_like == "any": + self._version = version_like + elif isinstance(version_like, str): + self._version = VersionInfoClass.parse(version_like) + elif isinstance(version_like, VersionInfoClass): + self._version = version_like + else: + raise ValueError("Version type not valid.") + + @property + def is_latest(self) -> bool: + """Check whether the version is 'latest'.""" + return isinstance(self._version, str) and self._version == "latest" + + def __str__(self) -> str: + """Get the string representation.""" + return str(self._version) + + def __eq__(self, other) -> bool: + """Check equality.""" + return isinstance(other, PackageVersion) and self._version == other._version + + def __lt__(self, other): + """Compare with another object.""" + enforce( + isinstance(other, PackageVersion), + f"Cannot compare {type(self)} with type {type(other)}.", + ) + other = cast(PackageVersion, other) + if self.is_latest or other.is_latest: + return self.is_latest < other.is_latest + return str(self) < str(other) + + +class PackageType(Enum): + """Package types.""" + + AGENT = AGENT + PROTOCOL = PROTOCOL + CONNECTION = CONNECTION + CONTRACT = CONTRACT + SKILL = SKILL + + def to_plural(self) -> str: + """ + Get the plural name. + + >>> PackageType.AGENT.to_plural() + 'agents' + >>> PackageType.PROTOCOL.to_plural() + 'protocols' + >>> PackageType.CONNECTION.to_plural() + 'connections' + >>> PackageType.SKILL.to_plural() + 'skills' + >>> PackageType.CONTRACT.to_plural() + 'contracts' + + """ + return self.value + "s" + + def __str__(self): + """Convert to string.""" + return str(self.value) + + +class ComponentType(Enum): + """Enum of component types supported.""" + + PROTOCOL = PROTOCOL + CONNECTION = CONNECTION + SKILL = SKILL + CONTRACT = CONTRACT + + def to_configuration_type(self) -> PackageType: + """Get package type for component type.""" + return PackageType(self.value) + + @staticmethod + def plurals() -> Collection[str]: + """ + Get the collection of type names, plural. + + >>> ComponentType.plurals() + ['protocols', 'connections', 'skills', 'contracts'] + """ + return list(map(lambda x: x.to_plural(), ComponentType)) + + def to_plural(self) -> str: + """ + Get the plural version of the component type. + + >>> ComponentType.PROTOCOL.to_plural() + 'protocols' + >>> ComponentType.CONNECTION.to_plural() + 'connections' + >>> ComponentType.SKILL.to_plural() + 'skills' + >>> ComponentType.CONTRACT.to_plural() + 'contracts' + """ + return self.value + "s" + + def __str__(self) -> str: + """Get the string representation.""" + return str(self.value) + + +class PublicId(JSONSerializable): + """This class implement a public identifier. + + A public identifier is composed of three elements: + - author + - name + - version + + The concatenation of those three elements gives the public identifier: + + author/name:version + + >>> public_id = PublicId("author", "my_package", "0.1.0") + >>> assert public_id.author == "author" + >>> assert public_id.name == "my_package" + >>> assert public_id.version == "0.1.0" + >>> another_public_id = PublicId("author", "my_package", "0.1.0") + >>> assert hash(public_id) == hash(another_public_id) + >>> assert public_id == another_public_id + >>> latest_public_id = PublicId("author", "my_package", "latest") + >>> latest_public_id + + >>> latest_public_id.package_version.is_latest + True + """ + + AUTHOR_REGEX = fr"[a-zA-Z_][a-zA-Z0-9_]{{0,{STRING_LENGTH_LIMIT - 1}}}" + PACKAGE_NAME_REGEX = fr"[a-zA-Z_][a-zA-Z0-9_]{{0,{STRING_LENGTH_LIMIT - 1}}}" + VERSION_NUMBER_PART_REGEX = r"(0|[1-9]\d*)" + VERSION_REGEX = fr"(any|latest|({VERSION_NUMBER_PART_REGEX})\.({VERSION_NUMBER_PART_REGEX})\.({VERSION_NUMBER_PART_REGEX})(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)" + PUBLIC_ID_REGEX = fr"^({AUTHOR_REGEX})/({PACKAGE_NAME_REGEX})(:({VERSION_REGEX}))?$" + PUBLIC_ID_URI_REGEX = ( + fr"^({AUTHOR_REGEX})/({PACKAGE_NAME_REGEX})/({VERSION_REGEX})$" + ) + + ANY_VERSION = "any" + LATEST_VERSION = "latest" + + def __init__( + self, + author: SimpleIdOrStr, + name: SimpleIdOrStr, + version: Optional[PackageVersionLike] = None, + ): + """Initialize the public identifier.""" + self._author = SimpleId(author) + self._name = SimpleId(name) + self._package_version = ( + PackageVersion(version) + if version is not None + else PackageVersion(self.LATEST_VERSION) + ) + + @property + def author(self) -> str: + """Get the author.""" + return str(self._author) + + @property + def name(self) -> str: + """Get the name.""" + return str(self._name) + + @property + def version(self) -> str: + """Get the version string.""" + return str(self._package_version) + + @property + def package_version(self) -> PackageVersion: + """Get the package version object.""" + return self._package_version + + def to_any(self) -> "PublicId": + """Return the same public id, but with any version.""" + return PublicId(self.author, self.name, self.ANY_VERSION) + + def same_prefix(self, other: "PublicId") -> bool: + """Check if the other public id has the same author and name of this.""" + return self.name == other.name and self.author == other.author + + def to_latest(self) -> "PublicId": + """Return the same public id, but with latest version.""" + return PublicId(self.author, self.name, self.LATEST_VERSION) + + @classmethod + def is_valid_str(cls, public_id_string: str) -> bool: + """ + Check if a string is a public id. + + :param public_id_string: the public id in string format. + :return: bool indicating validity + """ + match = re.match(cls.PUBLIC_ID_REGEX, public_id_string) + return match is not None + + @classmethod + def from_str(cls, public_id_string: str) -> "PublicId": + """ + Initialize the public id from the string. + + >>> str(PublicId.from_str("author/package_name:0.1.0")) + 'author/package_name:0.1.0' + + A bad formatted input raises value error: + >>> PublicId.from_str("bad/formatted:input") + Traceback (most recent call last): + ... + ValueError: Input 'bad/formatted:input' is not well formatted. + + :param public_id_string: the public id in string format. + :return: the public id object. + :raises ValueError: if the string in input is not well formatted. + """ + match = re.match(cls.PUBLIC_ID_REGEX, public_id_string) + if match is None: + raise ValueError( + "Input '{}' is not well formatted.".format(public_id_string) + ) + username = match.group(1) + package_name = match.group(2) + version = match.group(3)[1:] if ":" in public_id_string else None + return PublicId(username, package_name, version) + + @classmethod + def from_uri_path(cls, public_id_uri_path: str) -> "PublicId": + """ + Initialize the public id from the string. + + >>> str(PublicId.from_uri_path("author/package_name/0.1.0")) + 'author/package_name:0.1.0' + + A bad formatted input raises value error: + >>> PublicId.from_uri_path("bad/formatted:input") + Traceback (most recent call last): + ... + ValueError: Input 'bad/formatted:input' is not well formatted. + + :param public_id_uri_path: the public id in uri path string format. + :return: the public id object. + :raises ValueError: if the string in input is not well formatted. + """ + if not re.match(cls.PUBLIC_ID_URI_REGEX, public_id_uri_path): + raise ValueError( + "Input '{}' is not well formatted.".format(public_id_uri_path) + ) + username, package_name, version = re.findall( + cls.PUBLIC_ID_URI_REGEX, public_id_uri_path + )[0][:3] + return PublicId(username, package_name, version) + + @property + def to_uri_path(self) -> str: + """ + Turn the public id into a uri path string. + + :return: uri path string + """ + return "{author}/{name}/{version}".format( + author=self.author, name=self.name, version=self.version + ) + + @property + def json(self) -> Dict: + """Compute the JSON representation.""" + return {"author": self.author, "name": self.name, "version": self.version} + + @classmethod + def from_json(cls, obj: Dict): + """Build from a JSON object.""" + return PublicId(obj["author"], obj["name"], obj["version"],) + + def __hash__(self): + """Get the hash.""" + return hash((self.author, self.name, self.version)) + + def __str__(self): + """Get the string representation.""" + return "{author}/{name}:{version}".format( + author=self.author, name=self.name, version=self.version + ) + + def __repr__(self): + """Get the representation.""" + return f"<{self}>" + + def __eq__(self, other): + """Compare with another object.""" + return ( + isinstance(other, PublicId) + and self.author == other.author + and self.name == other.name + and self.version == other.version + ) + + def __lt__(self, other): + """ + Compare two public ids. + + >>> public_id_1 = PublicId("author_1", "name_1", "0.1.0") + >>> public_id_2 = PublicId("author_1", "name_1", "0.1.1") + >>> public_id_3 = PublicId("author_1", "name_2", "0.1.0") + >>> public_id_1 > public_id_2 + False + >>> public_id_1 < public_id_2 + True + + >>> public_id_1 < public_id_3 + Traceback (most recent call last): + ... + ValueError: The public IDs author_1/name_1:0.1.0 and author_1/name_2:0.1.0 cannot be compared. Their author or name attributes are different. + + """ + if ( + isinstance(other, PublicId) + and self.author == other.author + and self.name == other.name + ): + return self.package_version < other.package_version + raise ValueError( + "The public IDs {} and {} cannot be compared. Their author or name attributes are different.".format( + self, other + ) + ) + + +class PackageId: + """A package identifier.""" + + PACKAGE_TYPE_REGEX = r"({}|{}|{}|{}|{})".format( + PackageType.AGENT, + PackageType.PROTOCOL, + PackageType.SKILL, + PackageType.CONNECTION, + PackageType.CONTRACT, + ) + PACKAGE_ID_URI_REGEX = r"{}/{}".format( + PACKAGE_TYPE_REGEX, PublicId.PUBLIC_ID_URI_REGEX[1:-1] + ) + + def __init__(self, package_type: Union[PackageType, str], public_id: PublicId): + """ + Initialize the package id. + + :param package_type: the package type. + :param public_id: the public id. + """ + self._package_type = PackageType(package_type) + self._public_id = public_id + + @property + def package_type(self) -> PackageType: + """Get the package type.""" + return self._package_type + + @property + def public_id(self) -> PublicId: + """Get the public id.""" + return self._public_id + + @property + def author(self) -> str: + """Get the author of the package.""" + return self.public_id.author + + @property + def name(self) -> str: + """Get the name of the package.""" + return self.public_id.name + + @property + def version(self) -> str: + """Get the version of the package.""" + return self.public_id.version + + @property + def package_prefix(self) -> Tuple[PackageType, str, str]: + """Get the package identifier without the version.""" + return self.package_type, self.author, self.name + + @classmethod + def from_uri_path(cls, package_id_uri_path: str) -> "PackageId": + """ + Initialize the public id from the string. + + >>> str(PackageId.from_uri_path("skill/author/package_name/0.1.0")) + '(skill, author/package_name:0.1.0)' + + A bad formatted input raises value error: + >>> PackageId.from_uri_path("very/bad/formatted:input") + Traceback (most recent call last): + ... + ValueError: Input 'very/bad/formatted:input' is not well formatted. + + :param public_id_uri_path: the public id in uri path string format. + :return: the public id object. + :raises ValueError: if the string in input is not well formatted. + """ + if not re.match(cls.PACKAGE_ID_URI_REGEX, package_id_uri_path): + raise ValueError( + "Input '{}' is not well formatted.".format(package_id_uri_path) + ) + package_type_str, username, package_name, version = re.findall( + cls.PACKAGE_ID_URI_REGEX, package_id_uri_path + )[0][:4] + package_type = PackageType(package_type_str) + public_id = PublicId(username, package_name, version) + return PackageId(package_type, public_id) + + @property + def to_uri_path(self) -> str: + """ + Turn the package id into a uri path string. + + :return: uri path string + """ + return f"{str(self.package_type)}/{self.author}/{self.name}/{self.version}" + + def __hash__(self): + """Get the hash.""" + return hash((self.package_type, self.public_id)) + + def __str__(self): + """Get the string representation.""" + return "({package_type}, {public_id})".format( + package_type=self.package_type.value, public_id=self.public_id, + ) + + def __repr__(self): + """Get the object representation in string.""" + return f"PackageId{self.__str__()}" + + def __eq__(self, other): + """Compare with another object.""" + return ( + isinstance(other, PackageId) + and self.package_type == other.package_type + and self.public_id == other.public_id + ) + + def __lt__(self, other): + """Compare two public ids.""" + return str(self) < str(other) + + +class ComponentId(PackageId): + """ + Class to represent a component identifier. + + A component id is a package id, but excludes the case when the package is an agent. + >>> pacakge_id = PackageId(PackageType.PROTOCOL, PublicId("author", "name", "0.1.0")) + >>> component_id = ComponentId(ComponentType.PROTOCOL, PublicId("author", "name", "0.1.0")) + >>> pacakge_id == component_id + True + + >>> component_id2 = ComponentId(ComponentType.PROTOCOL, PublicId("author", "name", "0.1.1")) + >>> pacakge_id == component_id2 + False + """ + + def __init__(self, component_type: Union[ComponentType, str], public_id: PublicId): + """ + Initialize the component id. + + :param component_type: the component type. + :param public_id: the public id. + """ + component_type = ComponentType(component_type) + super().__init__(component_type.to_configuration_type(), public_id) + + @property + def component_type(self) -> ComponentType: + """Get the component type.""" + return ComponentType(self.package_type.value) + + @property + def component_prefix(self) -> Tuple[ComponentType, str, str]: + """Get the component identifier without the version.""" + package_prefix = super().package_prefix + package_type, author, name = package_prefix + return ComponentType(package_type.value), author, name + + def same_prefix(self, other: "ComponentId") -> bool: + """Check if the other component id has the same type, author and name of this.""" + return ( + self.component_type == other.component_type + and self.public_id.same_prefix(other.public_id) + ) + + @property + def prefix_import_path(self) -> str: + """Get the prefix import path for this component.""" + return "packages.{}.{}.{}".format( + self.public_id.author, self.component_type.to_plural(), self.public_id.name + ) + + @property + def json(self) -> Dict: + """Get the JSON representation.""" + return dict(**self.public_id.json, type=str(self.component_type)) + + +class PyPIPackageName(RegexConstrainedString): + """A PyPI Package name.""" + + REGEX = re.compile(r"^([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9._-]*[A-Za-z0-9])$") + + +class GitRef(RegexConstrainedString): + """ + A Git reference. + + It can be a branch name, a commit hash or a tag. + """ + + REGEX = re.compile(r"^[A-Za-z0-9/.\-_]+$") + + +class Dependency: + """ + This class represents a PyPI dependency. + + It contains the following information: + - version: a version specifier(s) (e.g. '==0.1.0'). + - index: the PyPI index where to download the package from (default: https://pypi.org) + - git: the URL to the Git repository (e.g. https://github.com/fetchai/agents-aea.git) + - ref: either the branch name, the tag, the commit number or a Git reference (default: 'master'.) + + If the 'git' field is set, the 'version' field will be ignored. + These fields will be forwarded to the 'pip' command. + """ + + def __init__( + self, + name: Union[PyPIPackageName, str], + version: Union[str, SpecifierSet] = "", + index: Optional[Union[str, Url]] = None, + git: Optional[Union[str, Url]] = None, + ref: Optional[Union[GitRef, str]] = None, + ): + """ + Initialize a PyPI dependency. + + :param name: the package name. + :param version: the specifier set object + :param index: the URL to the PyPI server. + :param git: the URL to a git repository. + :param ref: the Git reference (branch/commit/tag). + """ + self._name: PyPIPackageName = PyPIPackageName(name) + self._version: SpecifierSet = self._parse_version(version) + self._index: Optional[Url] = self._parse_url( + index + ) if index is not None else None + self._git: Optional[Url] = self._parse_url(git) if git is not None else None + self._ref: Optional[GitRef] = GitRef(ref) if ref is not None else None + + @property + def name(self) -> str: + """Get the name.""" + return str(self._name) + + @property + def version(self) -> str: + """Get the version.""" + return str(self._version) + + @property + def index(self) -> Optional[str]: + """Get the index.""" + return str(self._index) if self._index else None + + @property + def git(self) -> Optional[str]: + """Get the git.""" + return str(self._git) if self._git else None + + @property + def ref(self) -> Optional[str]: + """Get the ref.""" + return str(self._ref) if self._ref else None + + @staticmethod + def _parse_version(version: Union[str, SpecifierSet]) -> SpecifierSet: + """ + Parse a version specifier set. + + :param version: the version, a string or a SpecifierSet instance. + :return: the SpecifierSet instance. + """ + return version if isinstance(version, SpecifierSet) else SpecifierSet(version) + + @staticmethod + def _parse_url(url: Union[str, Url]) -> Url: + """ + Parse an URL. + + :param url: the URL, in either string or an urllib3.Url instance. + :return: the urllib3.Url instance. + """ + return url if isinstance(url, Url) else parse_url(url) + + @classmethod + def from_json(cls, obj: Dict[str, Dict[str, str]]) -> "Dependency": + """Parse a dependency object from a dictionary.""" + if len(obj) != 1: + raise ValueError(f"Only one key allowed, found {set(obj.keys())}") + name, attributes = list(obj.items())[0] + allowed_keys = {"version", "index", "git", "ref"} + not_allowed_keys = set(attributes.keys()).difference(allowed_keys) + if len(not_allowed_keys) > 0: + raise ValueError(f"Not allowed keys: {not_allowed_keys}") + + version = attributes.get("version", "") + index = attributes.get("index", None) + git = attributes.get("git", None) + ref = attributes.get("ref", None) + + return Dependency(name=name, version=version, index=index, git=git, ref=ref) + + def to_json(self) -> Dict[str, Dict[str, str]]: + """Transform the object to JSON.""" + result = {} + if self.version != "": + result["version"] = self.version + if self.index is not None: + result["index"] = self.index + if self.git is not None: + result["git"] = cast(str, self.git) + if self.ref is not None: + result["ref"] = cast(str, self.ref) + return {self.name: result} + + def get_pip_install_args(self) -> List[str]: + """Get 'pip install' arguments.""" + name = self.name + index = self.index + git_url = self.git + revision = self.ref if self.ref is not None else DEFAULT_GIT_REF + version_constraint = str(self.version) + command: List[str] = [] + if index is not None: + command += ["-i", index] + if git_url is not None: + command += ["git+" + git_url + "@" + revision + "#egg=" + name] + else: + command += [name + version_constraint] + return command + + def __str__(self) -> str: + """Get the string representation.""" + return f"{self.__class__.__name__}(name='{self.name}', version='{self.version}', index='{self.index}', git='{self.git}', ref='{self.ref}')" + + def __eq__(self, other): + """Compare with another object.""" + return ( + isinstance(other, Dependency) + and self._name == other._name + and self._version == other._version + and self._index == other._index + and self._git == other._git + and self._ref == other._ref + ) + + +Dependencies = Dict[str, Dependency] +""" +A dictionary from package name to dependency data structure (see above). +The package name must satisfy the constraints on Python packages names. + +The main advantage of having a dictionary is that we implicitly filter out dependency duplicates. +We cannot have two items with the same package name since the keys of a YAML object form a set. +""" + + +class CRUDCollection(Generic[T]): + """Interface of a CRUD collection.""" + + def __init__(self): + """Instantiate a CRUD collection.""" + self._items_by_id = {} # type: Dict[str, T] + + def create(self, item_id: str, item: T) -> None: + """ + Add an item. + + :param item_id: the item id. + :param item: the item to be added. + :return: None + :raises ValueError: if the item with the same id is already in the collection. + """ + if item_id in self._items_by_id: + raise ValueError("Item with name {} already present!".format(item_id)) + self._items_by_id[item_id] = item + + def read(self, item_id: str) -> Optional[T]: + """ + Get an item by its name. + + :param item_id: the item id. + :return: the associated item, or None if the item id is not present. + """ + return self._items_by_id.get(item_id, None) + + def update(self, item_id: str, item: T) -> None: + """ + Update an existing item. + + :param item_id: the item id. + :param item: the item to be added. + :return: None + """ + self._items_by_id[item_id] = item + + def delete(self, item_id: str) -> None: + """Delete an item.""" + if item_id in self._items_by_id.keys(): + del self._items_by_id[item_id] + + def read_all(self) -> List[Tuple[str, T]]: + """Read all the items.""" + return [ # pylint: disable=unnecessary-comprehension + (k, v) for k, v in self._items_by_id.items() + ] + + def keys(self) -> Set[str]: + """Get the set of keys.""" + return set(self._items_by_id.keys()) diff --git a/aea/configurations/loader.py b/aea/configurations/loader.py index f5ea5b433a..e71886fff7 100644 --- a/aea/configurations/loader.py +++ b/aea/configurations/loader.py @@ -16,19 +16,11 @@ # limitations under the License. # # ------------------------------------------------------------------------------ - """Implementation of the parser for configuration file.""" - -import inspect -import json -import os -from copy import deepcopy from pathlib import Path from typing import Dict, Generic, List, TextIO, Type, TypeVar, Union, cast -import jsonschema import yaml -from jsonschema import Draft4Validator from aea.configurations.base import ( AgentConfig, @@ -37,23 +29,22 @@ ComponentType, ConnectionConfig, ContractConfig, + PACKAGE_TYPE_TO_CONFIG_CLASS, + PackageConfiguration, PackageType, ProtocolConfig, ProtocolSpecification, - PublicId, SkillConfig, ) +from aea.configurations.validation import ConfigValidator, make_jsonschema_base_uri from aea.helpers.yaml_utils import yaml_dump, yaml_dump_all, yaml_load, yaml_load_all -_CUR_DIR = os.path.dirname(inspect.getfile(inspect.currentframe())) # type: ignore -_SCHEMAS_DIR = os.path.join(_CUR_DIR, "schemas") - -_PREFIX_BASE_CONFIGURABLE_PARTS = "base" -_SCHEMAS_CONFIGURABLE_PARTS_DIRNAME = "configurable_parts" -_POSTFIX_CUSTOM_CONFIG = "-custom_config.json" _STARTING_INDEX_CUSTOM_CONFIGS = 1 + +_ = make_jsonschema_base_uri # for tests compatibility + T = TypeVar( "T", AgentConfig, @@ -65,53 +56,19 @@ ) -def make_jsonschema_base_uri(base_uri_path: Path) -> str: - """ - Make the JSONSchema base URI, cross-platform. - - :param base_uri_path: the path to the base directory. - :return: the string in URI form. - """ - if os.name == "nt": # pragma: nocover # cause platform depended - root_path = "file:///{}/".format("/".join(base_uri_path.absolute().parts)) - else: # pragma: nocover # cause platform depended - root_path = "file://{}/".format(base_uri_path.absolute()) - return root_path - - -def _get_path_to_custom_config_schema_from_type(component_type: ComponentType) -> str: - """ - Get the path to the custom config schema - - :param component_type: a component type. - :return: the path to the JSON schema file. - """ - path_prefix: Path = Path(_SCHEMAS_DIR) / _SCHEMAS_CONFIGURABLE_PARTS_DIRNAME - if component_type in {ComponentType.SKILL, ComponentType.CONNECTION}: - filename_prefix = component_type.value - else: - filename_prefix = _PREFIX_BASE_CONFIGURABLE_PARTS - full_path = path_prefix / (filename_prefix + _POSTFIX_CUSTOM_CONFIG) - return str(full_path) - - class BaseConfigLoader: """Base class for configuration loader classes.""" def __init__(self, schema_filename: str): """ - Initialize the base configuration loader. + Initialize the parser for configuration files. - :param schema_filename: the path to the schema. + :param schema_filename: the path to the JSON-schema file in 'aea/configurations/schemas'. """ - base_uri = Path(_SCHEMAS_DIR) - self._schema = json.load((base_uri / schema_filename).open()) - root_path = make_jsonschema_base_uri(base_uri) - self._resolver = jsonschema.RefResolver(root_path, self._schema) - self._validator = Draft4Validator(self._schema, resolver=self._resolver) + self._validator = ConfigValidator(schema_filename) @property - def validator(self) -> Draft4Validator: + def validator(self) -> ConfigValidator: """Get the json schema validator.""" return self._validator @@ -131,7 +88,7 @@ def required_fields(self) -> List[str]: :return: list of required fields. """ - return self._schema["required"] + return self.validator.required_fields class ConfigLoader(Generic[T], BaseConfigLoader): @@ -181,7 +138,7 @@ def load_protocol_specification(self, file_pointer: TextIO) -> T: "Incorrect number of Yaml documents in the protocol specification." ) - self.validator.validate(instance=configuration_file_json) + self.validate(configuration_file_json) protocol_specification = self.configuration_class.from_json( configuration_file_json @@ -190,35 +147,6 @@ def load_protocol_specification(self, file_pointer: TextIO) -> T: protocol_specification.dialogue_config = dialogue_configuration return protocol_specification - def validate(self, json_data: Dict) -> None: - """ - Validate a JSON object against the right JSON schema. - - :param json_data: the JSON data. - :return: None. - """ - if self.configuration_class.package_type == PackageType.AGENT: - json_data_copy = deepcopy(json_data) - - # validate component_configurations - component_configurations = json_data_copy.pop( - "component_configurations", {} - ) - for idx, component_configuration_json in enumerate( - component_configurations - ): - component_id = self._split_component_id_and_config( - idx, component_configuration_json - ) - self.validate_component_configuration( - component_id, component_configuration_json - ) - - # validate agent config - self._validator.validate(instance=json_data_copy) - else: - self._validator.validate(instance=json_data) - def load(self, file_pointer: TextIO) -> T: """ Load a configuration file. @@ -333,7 +261,7 @@ def _dump_agent_config( def _dump_component_config(self, configuration: T, file_pointer: TextIO) -> None: """Dump component configuration.""" result = configuration.ordered_json - self.validator.validate(instance=result) + self.validate(result) yaml_dump(result, file_pointer) def _process_component_section( @@ -351,88 +279,18 @@ def _process_component_section( :param component_configuration_json: the JSON object. :return: the processed component configuration. """ - component_id = self._split_component_id_and_config( + component_id = self.validator.split_component_id_and_config( component_index, component_configuration_json ) - self.validate_component_configuration( + self.validator.validate_component_configuration( component_id, component_configuration_json ) return component_id - @staticmethod - def _split_component_id_and_config( - component_index: int, component_configuration_json: Dict - ) -> ComponentId: - """ - Split component id and configuration. - - :param component_index: the position of the component configuration in the agent config file.. - :param component_configuration_json: the JSON object to process. - :return: the component id and the configuration object. - :raises ValueError: if the component id cannot be extracted. - """ - # author, name, version, type are mandatory fields - missing_fields = {"public_id", "type"}.difference( - component_configuration_json.keys() - ) - if len(missing_fields) > 0: - raise ValueError( - f"There are missing fields in component id {component_index + 1}: {missing_fields}." - ) - public_id_str = component_configuration_json.pop("public_id") - component_type = ComponentType(component_configuration_json.pop("type")) - component_public_id = PublicId.from_str(public_id_str) - component_id = ComponentId(component_type, component_public_id) - return component_id - - @staticmethod - def validate_component_configuration( - component_id: ComponentId, configuration: Dict - ) -> None: - """ - Validate the component configuration of an agent configuration file. - - This check is to detect inconsistencies in the specified fields. - - :param component_id: the component id. - :param configuration: the configuration dictionary. - :return: None - :raises ValueError: if the configuration is not valid. - """ - schema_file = _get_path_to_custom_config_schema_from_type( - component_id.component_type - ) - try: - BaseConfigLoader(schema_file).validate( - dict( - type=str(component_id.component_type), - public_id=str(component_id.public_id), - **configuration, - ) - ) - except jsonschema.ValidationError as e: - raise ValueError( - f"Configuration of component {component_id} is not valid. {e}" - ) from e - class ConfigLoaders: """Configuration Loader class to load any package type.""" - _from_configuration_type_to_loaders = { - PackageType.AGENT: ConfigLoader("aea-config_schema.json", AgentConfig), - PackageType.PROTOCOL: ConfigLoader( - "protocol-config_schema.json", ProtocolConfig - ), - PackageType.CONNECTION: ConfigLoader( - "connection-config_schema.json", ConnectionConfig - ), - PackageType.SKILL: ConfigLoader("skill-config_schema.json", SkillConfig), - PackageType.CONTRACT: ConfigLoader( - "contract-config_schema.json", ContractConfig - ), - } # type: Dict[PackageType, ConfigLoader] - @classmethod def from_package_type( cls, configuration_type: Union[PackageType, str] @@ -442,8 +300,10 @@ def from_package_type( :param configuration_type: the configuration type """ - configuration_type = PackageType(configuration_type) - return cls._from_configuration_type_to_loaders[configuration_type] + config_class: Type[PackageConfiguration] = PACKAGE_TYPE_TO_CONFIG_CLASS[ + PackageType(configuration_type) + ] + return ConfigLoader(config_class.schema, config_class) def load_component_configuration( @@ -469,7 +329,7 @@ def load_component_configuration( def _load_configuration_object( component_type: ComponentType, directory: Path -) -> "ComponentConfiguration": +) -> ComponentConfiguration: """ Load the configuration object, without consistency checks. diff --git a/aea/configurations/validation.py b/aea/configurations/validation.py new file mode 100644 index 0000000000..b30c8513f7 --- /dev/null +++ b/aea/configurations/validation.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""Implementation of the configuration validation.""" +import inspect +import json +import os +from copy import deepcopy +from pathlib import Path +from typing import Dict, List + +import jsonschema +from jsonschema import Draft4Validator + +from aea.configurations.constants import AGENT +from aea.configurations.data_types import ComponentId, ComponentType, PublicId + + +_CUR_DIR = os.path.dirname(inspect.getfile(inspect.currentframe())) # type: ignore +_SCHEMAS_DIR = os.path.join(_CUR_DIR, "schemas") + +_PREFIX_BASE_CONFIGURABLE_PARTS = "base" +_SCHEMAS_CONFIGURABLE_PARTS_DIRNAME = "configurable_parts" +_POSTFIX_CUSTOM_CONFIG = "-custom_config.json" + + +def make_jsonschema_base_uri(base_uri_path: Path) -> str: + """ + Make the JSONSchema base URI, cross-platform. + + :param base_uri_path: the path to the base directory. + :return: the string in URI form. + """ + if os.name == "nt": # pragma: nocover # cause platform depended + root_path = "file:///{}/".format("/".join(base_uri_path.absolute().parts)) + else: # pragma: nocover # cause platform depended + root_path = "file://{}/".format(base_uri_path.absolute()) + return root_path + + +def _get_path_to_custom_config_schema_from_type(component_type: ComponentType) -> str: + """ + Get the path to the custom config schema + + :param component_type: a component type. + :return: the path to the JSON schema file. + """ + path_prefix: Path = Path(_SCHEMAS_DIR) / _SCHEMAS_CONFIGURABLE_PARTS_DIRNAME + if component_type in {ComponentType.SKILL, ComponentType.CONNECTION}: + filename_prefix = component_type.value + else: + filename_prefix = _PREFIX_BASE_CONFIGURABLE_PARTS + full_path = path_prefix / (filename_prefix + _POSTFIX_CUSTOM_CONFIG) + return str(full_path) + + +class ConfigValidator: + """Configuration validator implementation.""" + + def __init__(self, schema_filename: str): + """ + Initialize the parser for configuration files. + + :param schema_filename: the path to the JSON-schema file in 'aea/configurations/schemas'. + """ + base_uri = Path(_SCHEMAS_DIR) + self._schema = json.load((base_uri / schema_filename).open()) + root_path = make_jsonschema_base_uri(base_uri) + self._resolver = jsonschema.RefResolver(root_path, self._schema) + self._validator = Draft4Validator(self._schema, resolver=self._resolver) + + @staticmethod + def split_component_id_and_config( + component_index: int, component_configuration_json: Dict + ) -> ComponentId: + """ + Split component id and configuration. + + :param component_index: the position of the component configuration in the agent config file.. + :param component_configuration_json: the JSON object to process. + :return: the component id and the configuration object. + :raises ValueError: if the component id cannot be extracted. + """ + # author, name, version, type are mandatory fields + missing_fields = {"public_id", "type"}.difference( + component_configuration_json.keys() + ) + if len(missing_fields) > 0: + raise ValueError( + f"There are missing fields in component id {component_index + 1}: {missing_fields}." + ) + public_id_str = component_configuration_json.pop("public_id") + component_type = ComponentType(component_configuration_json.pop("type")) + component_public_id = PublicId.from_str(public_id_str) + component_id = ComponentId(component_type, component_public_id) + return component_id + + @classmethod + def validate_component_configuration( + cls, component_id: ComponentId, configuration: Dict + ) -> None: + """ + Validate the component configuration of an agent configuration file. + + This check is to detect inconsistencies in the specified fields. + + :param component_id: the component id. + :param configuration: the configuration dictionary. + :return: None + :raises ValueError: if the configuration is not valid. + """ + schema_file = _get_path_to_custom_config_schema_from_type( + component_id.component_type + ) + try: + cls(schema_file).validate( + dict( + type=str(component_id.component_type), + public_id=str(component_id.public_id), + **configuration, + ) + ) + except jsonschema.ValidationError as e: + raise ValueError( + f"Configuration of component {component_id} is not valid. {e}" + ) from e + + def validate(self, json_data: Dict) -> None: + """ + Validate a JSON object against the right JSON schema. + + :param json_data: the JSON data. + :return: None. + """ + if json_data.get("type", AGENT) == AGENT: + json_data_copy = deepcopy(json_data) + + # validate component_configurations + component_configurations = json_data_copy.pop( + "component_configurations", {} + ) + for idx, component_configuration_json in enumerate( + component_configurations + ): + component_id = self.split_component_id_and_config( + idx, component_configuration_json + ) + self.validate_component_configuration( + component_id, component_configuration_json + ) + + # validate agent config + self._validator.validate(instance=json_data_copy) + else: + self._validator.validate(instance=json_data) + + @property + def required_fields(self) -> List[str]: + """ + Get the required fields. + + :return: list of required fields. + """ + return self._schema["required"] diff --git a/aea/helpers/base.py b/aea/helpers/base.py index 481673ab56..a44188d997 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -30,11 +30,24 @@ import time import types from collections import OrderedDict, UserString, defaultdict, deque +from collections.abc import Mapping from copy import copy from functools import wraps from pathlib import Path from threading import RLock -from typing import Any, Callable, Deque, Dict, List, Set, TypeVar, Union +from typing import ( + Any, + Callable, + Deque, + Dict, + Iterable, + List, + Optional, + Set, + Tuple, + TypeVar, + Union, +) from dotenv import load_dotenv @@ -564,3 +577,18 @@ def ensure_dir(dir_path: str) -> None: os.makedirs(dir_path) else: enforce(os.path.isdir(dir_path), f"{dir_path} is not a directory!") + + +def dict_to_path_value( + data: Mapping, path: Optional[List] = None +) -> Iterable[Tuple[List[str], Any]]: + """Convert dict to sequence of terminal path build of keys and value.""" + path = path or [] + for key, value in data.items(): + path.append(key) + if not isinstance(value, Mapping): + # terminal value + yield path, value + else: + for p, v in dict_to_path_value(value, path): + yield p, v diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index 29d45eb061..052d129a32 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -411,7 +411,7 @@ def test__get_default_configuration_file_name_from_type_positive(self): class PublicIdTestCase(TestCase): """Test case for PublicId class.""" - @mock.patch("aea.configurations.base.re.match", return_value=None) + @mock.patch("aea.configurations.data_types.re.match", return_value=None) def test_public_id_from_str_not_matching(self, *mocks): """Test case for from_str method regex not matching.""" with self.assertRaises(ValueError): @@ -872,15 +872,6 @@ def test_package_version_lt(): assert v1 < v2 < v3 -def test_configuration_class(): - """Test the attribute 'configuration class' of PackageType.""" - assert PackageType.PROTOCOL.configuration_class() == ProtocolConfig - assert PackageType.CONNECTION.configuration_class() == ConnectionConfig - assert PackageType.CONTRACT.configuration_class() == ContractConfig - assert PackageType.SKILL.configuration_class() == SkillConfig - assert PackageType.AGENT.configuration_class() == AgentConfig - - class TestDependencyGetPipInstallArgs: """Test 'get_pip_install_args' of 'Dependency' class.""" diff --git a/tests/test_configurations/test_loader.py b/tests/test_configurations/test_loader.py index d1d59fd06b..d798ce7cf7 100644 --- a/tests/test_configurations/test_loader.py +++ b/tests/test_configurations/test_loader.py @@ -17,6 +17,7 @@ # # ------------------------------------------------------------------------------ + """This module contains the tests for the aea.configurations.loader module.""" import os from pathlib import Path @@ -28,7 +29,8 @@ import aea from aea.configurations.base import PackageType, ProtocolSpecification -from aea.configurations.loader import ConfigLoader, make_jsonschema_base_uri +from aea.configurations.loader import ConfigLoader +from aea.configurations.validation import make_jsonschema_base_uri from aea.protocols.generator.common import load_protocol_specification from tests.conftest import protocol_specification_files diff --git a/tests/test_configurations/test_schema.py b/tests/test_configurations/test_schema.py index 08c1fcefd7..0c77774b4c 100644 --- a/tests/test_configurations/test_schema.py +++ b/tests/test_configurations/test_schema.py @@ -29,7 +29,7 @@ import yaml from jsonschema import Draft4Validator # type: ignore -from aea.configurations.loader import make_jsonschema_base_uri +from aea.configurations.validation import make_jsonschema_base_uri from tests.conftest import ( AGENT_CONFIGURATION_SCHEMA, From 021b1b64f9f864bad8f5ae2d8f2328f5bc730e60 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 22 Dec 2020 14:08:12 +0100 Subject: [PATCH 029/204] add ledger_id field --- aea/configurations/base.py | 20 +++++++++++++++----- aea/configurations/schemas/definitions.json | 11 +++++++++++ aea/helpers/base.py | 12 ++++++++++++ tests/data/dummy_connection/connection.yaml | 6 ++++-- tests/data/hashes.csv | 2 +- tests/test_configurations/test_base.py | 14 ++++++++++++++ 6 files changed, 57 insertions(+), 8 deletions(-) diff --git a/aea/configurations/base.py b/aea/configurations/base.py index 3c4a18f813..e3a31fac84 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -82,6 +82,7 @@ SimpleId, SimpleIdOrStr, load_module, + parse_datetime_from_str, recursive_update, ) from aea.helpers.ipfs.base import IPFSHashOnly @@ -299,6 +300,7 @@ def __init__( self, public_key: str, identifier: SimpleIdOrStr, + ledger_id: SimpleIdOrStr, not_before: Union[str, datetime.datetime], not_after: Union[str, datetime.datetime], path: str, @@ -317,6 +319,7 @@ def __init__( self._key_identifier: Optional[str] = None self._public_key: Optional[str] = None self._identifier = str(SimpleId(identifier)) + self._ledger_id = str(SimpleId(ledger_id)) self._not_before_string = not_before self._not_after_string = not_after self._not_before = self._parse_datetime(not_before) @@ -326,8 +329,8 @@ def __init__( self._parse_public_key(public_key) self._check_validation_boundaries() - @staticmethod - def _parse_datetime(obj: Union[str, datetime.datetime]) -> datetime.datetime: + @classmethod + def _parse_datetime(cls, obj: Union[str, datetime.datetime]) -> datetime.datetime: """ Parse datetime string. @@ -337,9 +340,9 @@ def _parse_datetime(obj: Union[str, datetime.datetime]) -> datetime.datetime: :return: a datetime.datetime instance. """ result = ( - obj - if isinstance(obj, datetime.datetime) - else datetime.datetime.fromisoformat(obj) + parse_datetime_from_str(obj) # type: ignore + if isinstance(obj, str) + else obj ) enforce(result.microsecond == 0, "Microsecond field not allowed.") return result @@ -385,6 +388,11 @@ def public_key(self) -> Optional[str]: """Get the public key.""" return self._public_key + @property + def ledger_id(self) -> str: + """Get the ledger id.""" + return self._ledger_id + @property def key_identifier(self) -> Optional[str]: """Get the key identifier.""" @@ -415,6 +423,7 @@ def json(self) -> Dict: """Compute the JSON representation.""" result = dict( identifier=self.identifier, + ledger_id=self.ledger_id, not_before=self._not_before_string, not_after=self._not_after_string, path=str(self.path), @@ -435,6 +444,7 @@ def __eq__(self, other): return ( isinstance(other, CertRequest) and self.identifier == other.identifier + and self.ledger_id == other.ledger_id and self.public_key == other.public_key and self.key_identifier == other.key_identifier and self.not_after == other.not_after diff --git a/aea/configurations/schemas/definitions.json b/aea/configurations/schemas/definitions.json index 26cb190a2b..83ac84c525 100644 --- a/aea/configurations/schemas/definitions.json +++ b/aea/configurations/schemas/definitions.json @@ -161,6 +161,14 @@ "cert_request": { "type": "object", "additionalProperties": false, + "required": [ + "public_key", + "identifier", + "ledger_id", + "not_before", + "not_after", + "path" + ], "properties": { "public_key": { "type": "string" @@ -168,6 +176,9 @@ "identifier": { "$ref": "definitions.json#/definitions/resource_name" }, + "ledger_id": { + "$ref": "definitions.json#/definitions/ledger_id" + }, "not_before": { "type": "string" }, diff --git a/aea/helpers/base.py b/aea/helpers/base.py index 481673ab56..5c30008dcd 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -19,6 +19,7 @@ """Miscellaneous helpers.""" import builtins import contextlib +import datetime import importlib.util import logging import os @@ -42,6 +43,7 @@ STRING_LENGTH_LIMIT = 128 +ISO_8601_FORMAT = "%G-%m-%dT%H-%M-%S%z" _default_logger = logging.getLogger(__name__) @@ -564,3 +566,13 @@ def ensure_dir(dir_path: str) -> None: os.makedirs(dir_path) else: enforce(os.path.isdir(dir_path), f"{dir_path} is not a directory!") + + +def parse_datetime_from_str(date_string: str) -> datetime.datetime: + """Parse datetime from string.""" + version = sys.version_info + major = version.major + minor = version.minor + if (major, minor) > (3, 6): + return datetime.datetime.fromisoformat(date_string) + return datetime.datetime.strptime(date_string, ISO_8601_FORMAT) diff --git a/tests/data/dummy_connection/connection.yaml b/tests/data/dummy_connection/connection.yaml index 5a007b5b5a..3df4ec5bbf 100644 --- a/tests/data/dummy_connection/connection.yaml +++ b/tests/data/dummy_connection/connection.yaml @@ -28,12 +28,14 @@ dependencies: version: <=1.11.12b2 is_abstract: false cert_requests: -- identifier: cert_id +- identifier: cert_id_1 + ledger_id: some_ledger_id not_after: '2020-01-02T00:00:00' not_before: '2020-01-01T00:00:00' path: some/path_1 public_key: key_id -- identifier: cert_id +- identifier: cert_id_2 + ledger_id: some_ledger_id not_after: '2020-01-02T00:00:00' not_before: '2020-01-01T00:00:00' path: some/path_2 diff --git a/tests/data/hashes.csv b/tests/data/hashes.csv index 39a639de56..f09d0a2ee2 100644 --- a/tests/data/hashes.csv +++ b/tests/data/hashes.csv @@ -1,6 +1,6 @@ dummy_author/agents/dummy_aea,QmPWxJVM9xnkRFtB41Wr1Xi2NGKpeRgDXUzbAzA8rferGh dummy_author/skills/dummy_skill,QmSmzcjGNhiz2KpSE6wSi19AeesCVjzupJm1NpXYJVnJuZ -fetchai/connections/dummy_connection,QmbHGef6NEiuZUJL3reGRieF415KVDnhzMZZmBmU593Bwz +fetchai/connections/dummy_connection,QmTCvVMLKiJFWHF3xb6JdGmz3HBdkiYuNrADd2WGnaDeV5 fetchai/contracts/dummy_contract,QmXhMKRF5upco1efaC9oNcesStEMkPWKhBUv6CK9Vngk2S fetchai/protocols/t_protocol,QmNhzDCznorQpySrvKt4F4sQFXjzF1ZecEd5inixiNavDH fetchai/protocols/t_protocol_no_ct,QmWwMwHxvuPZJaMh44Bd1d5aBzYiogep8TJswTKkRwLfBu diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index 832d5a0bff..394f0888c1 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -992,6 +992,7 @@ class BaseTestCertRequestError: PUBLIC_KEY = "a_public_key" IDENTIFIER = "an_identifier" + LEDGER_ID = "a_ledger_id" NOT_BEFORE = "2020-01-01T00:00:00+00:00" NOT_AFTER = "2020-01-02T00:00:00+00:00" PATH = "some/path" @@ -1003,6 +1004,7 @@ def test_error(self): CertRequest( self.PUBLIC_KEY, self.IDENTIFIER, + self.LEDGER_ID, self.NOT_BEFORE, self.NOT_AFTER, self.PATH, @@ -1025,6 +1027,15 @@ class TestCertRequestBadIdentifier(BaseTestCertRequestError): ) +class TestCertRequestBadLedgerId(BaseTestCertRequestError): + """Test instantiation of CertRequest class with bad ledger id.""" + + LEDGER_ID = "0bad_identifier" + ERROR_MESSAGE_PATTERN = ( + "Value 0bad_identifier does not match the regular expression.*" + ) + + class TestCertRequestBadNotBefore(BaseTestCertRequestError): """Test instantiation of CertRequest class with bad not_before date.""" @@ -1059,12 +1070,14 @@ def setup_class(cls): """Set up class.""" cls.expected_public_key = cls.PUBLIC_KEY cls.expected_identifier = "identifier" + cls.expected_ledger_id = "ledger_id" cls.not_before = "2020-01-01T00:00:00+00:00" cls.not_after = "2020-01-02T00:00:00+00:00" cls.expected_path = "some/path" cls.cert_request = CertRequest( cls.expected_public_key, cls.expected_identifier, + cls.expected_ledger_id, cls.not_before, cls.not_after, cls.expected_path, @@ -1075,6 +1088,7 @@ def test_instantiation(self): assert self.cert_request.public_key == self.EXPECTED_PUBLIC_KEY assert self.cert_request.key_identifier == self.EXPECTED_KEY_IDENTIFIER assert self.cert_request.identifier == self.expected_identifier + assert self.cert_request.ledger_id == self.expected_ledger_id expected_not_before = datetime.datetime( 2020, 1, 1, 0, 0, 0, 0, datetime.timezone.utc From 72882bcee605925fead321153185772c760c8b9e Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 22 Dec 2020 14:37:00 +0100 Subject: [PATCH 030/204] update date parsing --- aea/configurations/base.py | 164 +----------------- aea/helpers/base.py | 175 +++++++++++++++++++- tests/data/dummy_connection/connection.yaml | 8 +- tests/data/hashes.csv | 2 +- tests/test_configurations/test_base.py | 139 ---------------- tests/test_helpers/test_base.py | 140 +++++++++++++++- 6 files changed, 312 insertions(+), 316 deletions(-) diff --git a/aea/configurations/base.py b/aea/configurations/base.py index e3a31fac84..a0e1eb1aca 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -18,13 +18,11 @@ # ------------------------------------------------------------------------------ """Classes to handle AEA configurations.""" -import datetime import functools import pprint import re from abc import ABC, abstractmethod from collections import OrderedDict -from contextlib import suppress from copy import copy, deepcopy from enum import Enum from operator import attrgetter @@ -77,12 +75,12 @@ ) from aea.exceptions import enforce from aea.helpers.base import ( + CertRequest, RegexConstrainedString, STRING_LENGTH_LIMIT, SimpleId, SimpleIdOrStr, load_module, - parse_datetime_from_str, recursive_update, ) from aea.helpers.ipfs.base import IPFSHashOnly @@ -293,166 +291,6 @@ def dependencies_to_json(dependencies: Dependencies) -> Dict[str, Dict]: return result -class CertRequest: - """Certificate request for proof of representation.""" - - def __init__( - self, - public_key: str, - identifier: SimpleIdOrStr, - ledger_id: SimpleIdOrStr, - not_before: Union[str, datetime.datetime], - not_after: Union[str, datetime.datetime], - path: str, - ): - """ - Initialize the certificate request. - - :param public_key: the public key, or the key id. - :param identifier: certificate identifier. - :param not_before: specify the lower bound for certificate vailidity. - If it is a string, it must follow the format: 'YYYY-MM-DDTHH:MM:SS' - :param not_before: specify the lower bound for certificate vailidity. - if it is a string, it must follow the format: 'YYYY-MM-DDTHH:MM:SS' - :param path: the path to the certificate. - """ - self._key_identifier: Optional[str] = None - self._public_key: Optional[str] = None - self._identifier = str(SimpleId(identifier)) - self._ledger_id = str(SimpleId(ledger_id)) - self._not_before_string = not_before - self._not_after_string = not_after - self._not_before = self._parse_datetime(not_before) - self._not_after = self._parse_datetime(not_after) - self._path = Path(path) - - self._parse_public_key(public_key) - self._check_validation_boundaries() - - @classmethod - def _parse_datetime(cls, obj: Union[str, datetime.datetime]) -> datetime.datetime: - """ - Parse datetime string. - - It is expected to follow ISO 8601. - - :param obj: the input to parse. - :return: a datetime.datetime instance. - """ - result = ( - parse_datetime_from_str(obj) # type: ignore - if isinstance(obj, str) - else obj - ) - enforce(result.microsecond == 0, "Microsecond field not allowed.") - return result - - def _check_validation_boundaries(self): - """ - Check the validation boundaries are consistent. - - Namely, that not_before < not_after. - """ - enforce( - self._not_before < self._not_after, - f"Inconsistent certificate validity period: 'not_before' field '{self._not_before.isoformat()}' is not before than 'not_after' field '{self._not_after.isoformat()}'", - ValueError, - ) - - def _parse_public_key(self, public_key_str: str) -> None: - """ - Parse public key from string. - - It first tries to parse it as an identifier, - and in case of failure as a sequence of hexadecimals, starting with "0x". - """ - with suppress(ValueError): - # if this raises ValueError, we don't return - self._key_identifier = str(SimpleId(public_key_str)) - return - - with suppress(ValueError): - # this raises ValueError if the input is not a valid hexadecimal string. - int(public_key_str, 16) - self._public_key = public_key_str - return - - enforce( - False, - f"Public key field '{public_key_str}' is neither a valid identifier nor an address.", - exception_class=ValueError, - ) - - @property - def public_key(self) -> Optional[str]: - """Get the public key.""" - return self._public_key - - @property - def ledger_id(self) -> str: - """Get the ledger id.""" - return self._ledger_id - - @property - def key_identifier(self) -> Optional[str]: - """Get the key identifier.""" - return self._key_identifier - - @property - def identifier(self) -> str: - """Get the identifier.""" - return self._identifier - - @property - def not_before(self) -> datetime.datetime: - """Get the not_before field.""" - return self._not_before - - @property - def not_after(self) -> datetime.datetime: - """Get the not_after field.""" - return self._not_after - - @property - def path(self) -> Path: - """Get the path""" - return self._path - - @property - def json(self) -> Dict: - """Compute the JSON representation.""" - result = dict( - identifier=self.identifier, - ledger_id=self.ledger_id, - not_before=self._not_before_string, - not_after=self._not_after_string, - path=str(self.path), - ) - if self.public_key is not None: - result["public_key"] = self.public_key - elif self.key_identifier is not None: - result["public_key"] = self.key_identifier - return result - - @classmethod - def from_json(cls, obj: Dict) -> "CertRequest": - """Compute the JSON representation.""" - return cls(**obj) - - def __eq__(self, other): - """Check equality.""" - return ( - isinstance(other, CertRequest) - and self.identifier == other.identifier - and self.ledger_id == other.ledger_id - and self.public_key == other.public_key - and self.key_identifier == other.key_identifier - and self.not_after == other.not_after - and self.not_before == other.not_before - and self.path == other.path - ) - - VersionInfoClass = semver.VersionInfo PackageVersionLike = Union[str, semver.VersionInfo] diff --git a/aea/helpers/base.py b/aea/helpers/base.py index 5c30008dcd..835a4b7e4d 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -35,7 +35,7 @@ from functools import wraps from pathlib import Path from threading import RLock -from typing import Any, Callable, Deque, Dict, List, Set, TypeVar, Union +from typing import Any, Callable, Deque, Dict, List, Optional, Set, TypeVar, Union from dotenv import load_dotenv @@ -43,7 +43,7 @@ STRING_LENGTH_LIMIT = 128 -ISO_8601_FORMAT = "%G-%m-%dT%H-%M-%S%z" +ISO_8601_DATE_FORMAT = "%Y-%m-%d" _default_logger = logging.getLogger(__name__) @@ -570,9 +570,168 @@ def ensure_dir(dir_path: str) -> None: def parse_datetime_from_str(date_string: str) -> datetime.datetime: """Parse datetime from string.""" - version = sys.version_info - major = version.major - minor = version.minor - if (major, minor) > (3, 6): - return datetime.datetime.fromisoformat(date_string) - return datetime.datetime.strptime(date_string, ISO_8601_FORMAT) + result = datetime.datetime.strptime(date_string, ISO_8601_DATE_FORMAT) + result = result.replace(tzinfo=datetime.timezone.utc) + return result + + +class CertRequest: + """Certificate request for proof of representation.""" + + def __init__( + self, + public_key: str, + identifier: SimpleIdOrStr, + ledger_id: SimpleIdOrStr, + not_before: Union[str, datetime.datetime], + not_after: Union[str, datetime.datetime], + path: str, + ): + """ + Initialize the certificate request. + + :param public_key: the public key, or the key id. + :param identifier: certificate identifier. + :param not_before: specify the lower bound for certificate vailidity. + If it is a string, it must follow the format: 'YYYY-MM-DD'. It + will be interpreted as timezone UTC. + :param not_before: specify the lower bound for certificate vailidity. + if it is a string, it must follow the format: 'YYYY-MM-DD' It + will be interpreted as timezone UTC-0. + :param path: the path to the certificate. + """ + self._key_identifier: Optional[str] = None + self._public_key: Optional[str] = None + self._identifier = str(SimpleId(identifier)) + self._ledger_id = str(SimpleId(ledger_id)) + self._not_before_string = not_before + self._not_after_string = not_after + self._not_before = self._parse_datetime(not_before) + self._not_after = self._parse_datetime(not_after) + self._path = Path(path) + + self._parse_public_key(public_key) + self._check_validation_boundaries() + + @classmethod + def _parse_datetime(cls, obj: Union[str, datetime.datetime]) -> datetime.datetime: + """ + Parse datetime string. + + It is expected to follow ISO 8601. + + :param obj: the input to parse. + :return: a datetime.datetime instance. + """ + result = ( + parse_datetime_from_str(obj) # type: ignore + if isinstance(obj, str) + else obj + ) + enforce(result.microsecond == 0, "Microsecond field not allowed.") + return result + + def _check_validation_boundaries(self): + """ + Check the validation boundaries are consistent. + + Namely, that not_before < not_after. + """ + enforce( + self._not_before < self._not_after, + f"Inconsistent certificate validity period: 'not_before' field '{self._not_before.isoformat()}' is not before than 'not_after' field '{self._not_after.isoformat()}'", + ValueError, + ) + + def _parse_public_key(self, public_key_str: str) -> None: + """ + Parse public key from string. + + It first tries to parse it as an identifier, + and in case of failure as a sequence of hexadecimals, starting with "0x". + """ + with contextlib.suppress(ValueError): + # if this raises ValueError, we don't return + self._key_identifier = str(SimpleId(public_key_str)) + return + + with contextlib.suppress(ValueError): + # this raises ValueError if the input is not a valid hexadecimal string. + int(public_key_str, 16) + self._public_key = public_key_str + return + + enforce( + False, + f"Public key field '{public_key_str}' is neither a valid identifier nor an address.", + exception_class=ValueError, + ) + + @property + def public_key(self) -> Optional[str]: + """Get the public key.""" + return self._public_key + + @property + def ledger_id(self) -> str: + """Get the ledger id.""" + return self._ledger_id + + @property + def key_identifier(self) -> Optional[str]: + """Get the key identifier.""" + return self._key_identifier + + @property + def identifier(self) -> str: + """Get the identifier.""" + return self._identifier + + @property + def not_before(self) -> datetime.datetime: + """Get the not_before field.""" + return self._not_before + + @property + def not_after(self) -> datetime.datetime: + """Get the not_after field.""" + return self._not_after + + @property + def path(self) -> Path: + """Get the path""" + return self._path + + @property + def json(self) -> Dict: + """Compute the JSON representation.""" + result = dict( + identifier=self.identifier, + ledger_id=self.ledger_id, + not_before=self._not_before_string, + not_after=self._not_after_string, + path=str(self.path), + ) + if self.public_key is not None: + result["public_key"] = self.public_key + elif self.key_identifier is not None: + result["public_key"] = self.key_identifier + return result + + @classmethod + def from_json(cls, obj: Dict) -> "CertRequest": + """Compute the JSON representation.""" + return cls(**obj) + + def __eq__(self, other): + """Check equality.""" + return ( + isinstance(other, CertRequest) + and self.identifier == other.identifier + and self.ledger_id == other.ledger_id + and self.public_key == other.public_key + and self.key_identifier == other.key_identifier + and self.not_after == other.not_after + and self.not_before == other.not_before + and self.path == other.path + ) diff --git a/tests/data/dummy_connection/connection.yaml b/tests/data/dummy_connection/connection.yaml index 3df4ec5bbf..4598c6ac67 100644 --- a/tests/data/dummy_connection/connection.yaml +++ b/tests/data/dummy_connection/connection.yaml @@ -30,13 +30,13 @@ is_abstract: false cert_requests: - identifier: cert_id_1 ledger_id: some_ledger_id - not_after: '2020-01-02T00:00:00' - not_before: '2020-01-01T00:00:00' + not_after: '2020-01-02' + not_before: '2020-01-01' path: some/path_1 public_key: key_id - identifier: cert_id_2 ledger_id: some_ledger_id - not_after: '2020-01-02T00:00:00' - not_before: '2020-01-01T00:00:00' + not_after: '2020-01-02' + not_before: '2020-01-01' path: some/path_2 public_key: '0xABCDEF123456' diff --git a/tests/data/hashes.csv b/tests/data/hashes.csv index f09d0a2ee2..8d1d3cd01b 100644 --- a/tests/data/hashes.csv +++ b/tests/data/hashes.csv @@ -1,6 +1,6 @@ dummy_author/agents/dummy_aea,QmPWxJVM9xnkRFtB41Wr1Xi2NGKpeRgDXUzbAzA8rferGh dummy_author/skills/dummy_skill,QmSmzcjGNhiz2KpSE6wSi19AeesCVjzupJm1NpXYJVnJuZ -fetchai/connections/dummy_connection,QmTCvVMLKiJFWHF3xb6JdGmz3HBdkiYuNrADd2WGnaDeV5 +fetchai/connections/dummy_connection,QmWCzxyUTivdaDiouSepKwjyomyshdzocSHDStXG16ZHkL fetchai/contracts/dummy_contract,QmXhMKRF5upco1efaC9oNcesStEMkPWKhBUv6CK9Vngk2S fetchai/protocols/t_protocol,QmNhzDCznorQpySrvKt4F4sQFXjzF1ZecEd5inixiNavDH fetchai/protocols/t_protocol_no_ct,QmWwMwHxvuPZJaMh44Bd1d5aBzYiogep8TJswTKkRwLfBu diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index 394f0888c1..29d45eb061 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -18,11 +18,9 @@ # ------------------------------------------------------------------------------ """This module contains the tests for the aea.configurations.base module.""" -import datetime import re from copy import copy from pathlib import Path -from typing import Optional from unittest import TestCase, mock import pytest @@ -32,7 +30,6 @@ from aea.configurations.base import ( AgentConfig, CRUDCollection, - CertRequest, ComponentId, ComponentType, ConnectionConfig, @@ -985,139 +982,3 @@ def test_check_public_id_consistency_negative(): with pytest.raises(ValueError, match=f"Directory {random_dir_name} is not valid."): component_configuration = ProtocolConfig("name", "author") component_configuration.check_public_id_consistency(Path(random_dir_name)) - - -class BaseTestCertRequestError: - """Test errors when instantiating a CertRequest object.""" - - PUBLIC_KEY = "a_public_key" - IDENTIFIER = "an_identifier" - LEDGER_ID = "a_ledger_id" - NOT_BEFORE = "2020-01-01T00:00:00+00:00" - NOT_AFTER = "2020-01-02T00:00:00+00:00" - PATH = "some/path" - ERROR_MESSAGE_PATTERN = "" - - def test_error(self): - """Test error during instantiation..""" - with pytest.raises(ValueError, match=self.ERROR_MESSAGE_PATTERN): - CertRequest( - self.PUBLIC_KEY, - self.IDENTIFIER, - self.LEDGER_ID, - self.NOT_BEFORE, - self.NOT_AFTER, - self.PATH, - ) - - -class TestCertRequestBadPublicKey(BaseTestCertRequestError): - """Test instantiation of CertRequest class with bad public key.""" - - PUBLIC_KEY = "0a_bad_identifier" - ERROR_MESSAGE_PATTERN = "Public key field '0a_bad_identifier' is neither a valid identifier nor an address." - - -class TestCertRequestBadIdentifier(BaseTestCertRequestError): - """Test instantiation of CertRequest class with bad identifier.""" - - IDENTIFIER = "0bad_identifier" - ERROR_MESSAGE_PATTERN = ( - "Value 0bad_identifier does not match the regular expression.*" - ) - - -class TestCertRequestBadLedgerId(BaseTestCertRequestError): - """Test instantiation of CertRequest class with bad ledger id.""" - - LEDGER_ID = "0bad_identifier" - ERROR_MESSAGE_PATTERN = ( - "Value 0bad_identifier does not match the regular expression.*" - ) - - -class TestCertRequestBadNotBefore(BaseTestCertRequestError): - """Test instantiation of CertRequest class with bad not_before date.""" - - NOT_BEFORE = "bad-formatted-date" - ERROR_MESSAGE_PATTERN = "Invalid isoformat string: 'bad-formatted-date'" - - -class TestCertRequestBadNotAfter(BaseTestCertRequestError): - """Test instantiation of CertRequest class with bad not_after date.""" - - NOT_AFTER = "bad-formatted-date" - ERROR_MESSAGE_PATTERN = "Invalid isoformat string: 'bad-formatted-date'" - - -class TestCertRequestInconsistentDates(BaseTestCertRequestError): - """Test instantiation of CertRequest class when not_before >= not_after""" - - NOT_BEFORE = "1954-06-07T00:00:00+00:00" - NOT_AFTER = "1900-01-01T00:00:01+00:00" - ERROR_MESSAGE_PATTERN = r"Inconsistent certificate validity period: 'not_before' field '1954-06-07T00:00:00\+00:00' is not before than 'not_after' field '1900-01-01T00:00:01\+00:00'" - - -class BaseTestCertRequestInstantiation: - """Test (successful) instantiation of CertRequest class.""" - - PUBLIC_KEY: Optional[str] = "" - EXPECTED_PUBLIC_KEY: Optional[str] = "" - EXPECTED_KEY_IDENTIFIER: Optional[str] = "" - - @classmethod - def setup_class(cls): - """Set up class.""" - cls.expected_public_key = cls.PUBLIC_KEY - cls.expected_identifier = "identifier" - cls.expected_ledger_id = "ledger_id" - cls.not_before = "2020-01-01T00:00:00+00:00" - cls.not_after = "2020-01-02T00:00:00+00:00" - cls.expected_path = "some/path" - cls.cert_request = CertRequest( - cls.expected_public_key, - cls.expected_identifier, - cls.expected_ledger_id, - cls.not_before, - cls.not_after, - cls.expected_path, - ) - - def test_instantiation(self): - """Test instantiation.""" - assert self.cert_request.public_key == self.EXPECTED_PUBLIC_KEY - assert self.cert_request.key_identifier == self.EXPECTED_KEY_IDENTIFIER - assert self.cert_request.identifier == self.expected_identifier - assert self.cert_request.ledger_id == self.expected_ledger_id - - expected_not_before = datetime.datetime( - 2020, 1, 1, 0, 0, 0, 0, datetime.timezone.utc - ) - assert self.cert_request.not_before == expected_not_before - - expected_not_after = datetime.datetime( - 2020, 1, 2, 0, 0, 0, 0, datetime.timezone.utc - ) - assert self.cert_request.not_after == expected_not_after - - assert self.cert_request.path == Path(self.expected_path) - - def test_from_to_json(self): - """Test from-to json methods.""" - assert self.cert_request == self.cert_request.from_json(self.cert_request.json) - - -class TestCertRequestInstantiationWithKeyIdentifier(BaseTestCertRequestInstantiation): - """Test (successful) instantiation of CertRequest class.""" - - PUBLIC_KEY = "public_key" - EXPECTED_PUBLIC_KEY = None - EXPECTED_KEY_IDENTIFIER = PUBLIC_KEY - - -class TestCertRequestInstantiationWithKeyHex(BaseTestCertRequestInstantiation): - """Test (successful) instantiation of CertRequest class.""" - - PUBLIC_KEY = "0xABCDEF12345" - EXPECTED_PUBLIC_KEY = "0xABCDEF12345" - EXPECTED_KEY_IDENTIFIER = None diff --git a/tests/test_helpers/test_base.py b/tests/test_helpers/test_base.py index 53b65e99d8..cf919c1aa8 100644 --- a/tests/test_helpers/test_base.py +++ b/tests/test_helpers/test_base.py @@ -17,6 +17,7 @@ # # ------------------------------------------------------------------------------ """This module contains the tests for the helper module.""" +import datetime import os import platform import re @@ -26,13 +27,14 @@ from pathlib import Path from subprocess import Popen # nosec from tempfile import TemporaryDirectory -from typing import Dict, Set +from typing import Dict, Optional, Set from unittest.mock import patch import pytest from aea.exceptions import AEAEnforceError from aea.helpers.base import ( + CertRequest, MaxRetriesError, RegexConstrainedString, ensure_dir, @@ -396,3 +398,139 @@ def test_ensure_dir(): with pytest.raises(AEAEnforceError): ensure_dir(file_path) + + +class BaseTestCertRequestError: + """Test errors when instantiating a CertRequest object.""" + + PUBLIC_KEY = "a_public_key" + IDENTIFIER = "an_identifier" + LEDGER_ID = "a_ledger_id" + NOT_BEFORE = "2020-01-01" + NOT_AFTER = "2020-01-02" + PATH = "some/path" + ERROR_MESSAGE_PATTERN = "" + + def test_error(self): + """Test error during instantiation..""" + with pytest.raises(ValueError, match=self.ERROR_MESSAGE_PATTERN): + CertRequest( + self.PUBLIC_KEY, + self.IDENTIFIER, + self.LEDGER_ID, + self.NOT_BEFORE, + self.NOT_AFTER, + self.PATH, + ) + + +class TestCertRequestBadPublicKey(BaseTestCertRequestError): + """Test instantiation of CertRequest class with bad public key.""" + + PUBLIC_KEY = "0a_bad_identifier" + ERROR_MESSAGE_PATTERN = "Public key field '0a_bad_identifier' is neither a valid identifier nor an address." + + +class TestCertRequestBadIdentifier(BaseTestCertRequestError): + """Test instantiation of CertRequest class with bad identifier.""" + + IDENTIFIER = "0bad_identifier" + ERROR_MESSAGE_PATTERN = ( + "Value 0bad_identifier does not match the regular expression.*" + ) + + +class TestCertRequestBadLedgerId(BaseTestCertRequestError): + """Test instantiation of CertRequest class with bad ledger id.""" + + LEDGER_ID = "0bad_identifier" + ERROR_MESSAGE_PATTERN = ( + "Value 0bad_identifier does not match the regular expression.*" + ) + + +class TestCertRequestBadNotBefore(BaseTestCertRequestError): + """Test instantiation of CertRequest class with bad not_before date.""" + + NOT_BEFORE = "bad-formatted-date" + ERROR_MESSAGE_PATTERN = "Invalid isoformat string: 'bad-formatted-date'" + + +class TestCertRequestBadNotAfter(BaseTestCertRequestError): + """Test instantiation of CertRequest class with bad not_after date.""" + + NOT_AFTER = "bad-formatted-date" + ERROR_MESSAGE_PATTERN = "Invalid isoformat string: 'bad-formatted-date'" + + +class TestCertRequestInconsistentDates(BaseTestCertRequestError): + """Test instantiation of CertRequest class when not_before >= not_after""" + + NOT_BEFORE = "1954-06-07" + NOT_AFTER = "1900-01-01" + ERROR_MESSAGE_PATTERN = r"Inconsistent certificate validity period: 'not_before' field '1954-06-07T00:00:00\+00:00' is not before than 'not_after' field '1900-01-01T00:00:01\+00:00'" + + +class BaseTestCertRequestInstantiation: + """Test (successful) instantiation of CertRequest class.""" + + PUBLIC_KEY: Optional[str] = "" + EXPECTED_PUBLIC_KEY: Optional[str] = "" + EXPECTED_KEY_IDENTIFIER: Optional[str] = "" + + @classmethod + def setup_class(cls): + """Set up class.""" + cls.expected_public_key = cls.PUBLIC_KEY + cls.expected_identifier = "identifier" + cls.expected_ledger_id = "ledger_id" + cls.not_before = "2020-01-01" + cls.not_after = "2020-01-02" + cls.expected_path = "some/path" + cls.cert_request = CertRequest( + cls.expected_public_key, + cls.expected_identifier, + cls.expected_ledger_id, + cls.not_before, + cls.not_after, + cls.expected_path, + ) + + def test_instantiation(self): + """Test instantiation.""" + assert self.cert_request.public_key == self.EXPECTED_PUBLIC_KEY + assert self.cert_request.key_identifier == self.EXPECTED_KEY_IDENTIFIER + assert self.cert_request.identifier == self.expected_identifier + assert self.cert_request.ledger_id == self.expected_ledger_id + + expected_not_before = datetime.datetime( + 2020, 1, 1, 0, 0, 0, 0, datetime.timezone.utc + ) + assert self.cert_request.not_before == expected_not_before + + expected_not_after = datetime.datetime( + 2020, 1, 2, 0, 0, 0, 0, datetime.timezone.utc + ) + assert self.cert_request.not_after == expected_not_after + + assert self.cert_request.path == Path(self.expected_path) + + def test_from_to_json(self): + """Test from-to json methods.""" + assert self.cert_request == self.cert_request.from_json(self.cert_request.json) + + +class TestCertRequestInstantiationWithKeyIdentifier(BaseTestCertRequestInstantiation): + """Test (successful) instantiation of CertRequest class.""" + + PUBLIC_KEY = "public_key" + EXPECTED_PUBLIC_KEY = None + EXPECTED_KEY_IDENTIFIER = PUBLIC_KEY + + +class TestCertRequestInstantiationWithKeyHex(BaseTestCertRequestInstantiation): + """Test (successful) instantiation of CertRequest class.""" + + PUBLIC_KEY = "0xABCDEF12345" + EXPECTED_PUBLIC_KEY = "0xABCDEF12345" + EXPECTED_KEY_IDENTIFIER = None From f3772e931d75493be43ff8051a22c7c1cf657fec Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 22 Dec 2020 15:57:19 +0100 Subject: [PATCH 031/204] add issue-certificates command --- aea/cli/core.py | 2 + aea/cli/issue_certificates.py | 103 ++++++++++++++++++++++ aea/helpers/base.py | 16 +++- tests/test_cli/test_issue_certificates.py | 24 +++++ tests/test_helpers/test_base.py | 10 ++- 5 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 aea/cli/issue_certificates.py create mode 100644 tests/test_cli/test_issue_certificates.py diff --git a/aea/cli/core.py b/aea/cli/core.py index 901359bed3..16655e4335 100644 --- a/aea/cli/core.py +++ b/aea/cli/core.py @@ -42,6 +42,7 @@ from aea.cli.init import init from aea.cli.install import install from aea.cli.interact import interact +from aea.cli.issue_certificates import issue_certificates from aea.cli.launch import launch from aea.cli.list import list_command as _list from aea.cli.login import login @@ -137,6 +138,7 @@ def _init_gui() -> None: cli.add_command(init) cli.add_command(install) cli.add_command(interact) +cli.add_command(issue_certificates) cli.add_command(launch) cli.add_command(login) cli.add_command(logout) diff --git a/aea/cli/issue_certificates.py b/aea/cli/issue_certificates.py new file mode 100644 index 0000000000..830e7b1a4e --- /dev/null +++ b/aea/cli/issue_certificates.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2020 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Implementation of the 'aea issue_certificates' subcommand.""" +from pathlib import Path +from typing import cast + +import click +from click import ClickException + +from aea.cli.utils.config import load_item_config +from aea.cli.utils.context import Context +from aea.cli.utils.decorators import check_aea_project +from aea.cli.utils.package_utils import get_package_path_unified +from aea.configurations.base import ConnectionConfig, PublicId +from aea.configurations.constants import CONNECTION +from aea.crypto.registries import crypto_registry +from aea.exceptions import enforce +from aea.helpers.base import CertRequest + + +@click.command() +@click.pass_context +@check_aea_project +def issue_certificates(click_context): + """Issue certificates for connections that require them.""" + ctx = cast(Context, click_context.obj) + + for connection_id in ctx.agent_config.connections: + _process_connection(ctx, connection_id) + + click.echo("Done!") + + +def _process_certificate(ctx: Context, cert_request: CertRequest): + """Process a single certificate request.""" + click.echo(f"Issuing certificate '{cert_request.identifier}'...") + ledger_id = cert_request.ledger_id + output_path = cert_request.path + if cert_request.key_identifier is not None: + key_identifier = cert_request.key_identifier + connection_private_key_path = ctx.agent_config.connection_private_key_paths.read( + key_identifier + ) + if connection_private_key_path is None: + raise ClickException( + f"Cannot find connection private key with id '{key_identifier}'" + ) + connection_crypto = crypto_registry.make( + key_identifier, private_key_path=connection_private_key_path + ) + public_key = connection_crypto.public_key + else: + public_key = cast(str, cert_request.public_key) + enforce( + public_key is not None, + "Internal error - one of key_identifier or public_key must be not None.", + ) + public_key_bytes = public_key.encode("ascii") + identifier = cert_request.identifier.encode("ascii") + not_before = cert_request.not_before_string.encode("ascii") + not_after = cert_request.not_after_string.encode("ascii") + crytpo_private_key_path = ctx.agent_config.connection_private_key_paths.read( + ledger_id + ) + if crytpo_private_key_path is None: + raise ClickException(f"Cannot find private key with id '{ledger_id}'") + crypto = crypto_registry.make(ledger_id, private_key_path=crytpo_private_key_path) + message = public_key_bytes + identifier + not_before + not_after + signature = crypto.sign_message(message).encode("ascii") + Path(output_path).write_bytes(signature) + click.echo(f"Dumped certificate in '{output_path}'.") + + +def _process_connection(ctx: Context, connection_id: PublicId): + click.echo(f"Processing connection '{connection_id}'...") + path = get_package_path_unified(ctx, CONNECTION, connection_id) + connection_config = cast(ConnectionConfig, load_item_config(CONNECTION, Path(path))) + if ( + connection_config.cert_requests is None + or len(connection_config.cert_requests) == 0 + ): + click.echo("No certificates to process.") + return + + for cert_request in connection_config.cert_requests: + _process_certificate(ctx, cert_request) diff --git a/aea/helpers/base.py b/aea/helpers/base.py index 835a4b7e4d..dca2f8d049 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -583,8 +583,8 @@ def __init__( public_key: str, identifier: SimpleIdOrStr, ledger_id: SimpleIdOrStr, - not_before: Union[str, datetime.datetime], - not_after: Union[str, datetime.datetime], + not_before: str, + not_after: str, path: str, ): """ @@ -639,7 +639,7 @@ def _check_validation_boundaries(self): """ enforce( self._not_before < self._not_after, - f"Inconsistent certificate validity period: 'not_before' field '{self._not_before.isoformat()}' is not before than 'not_after' field '{self._not_after.isoformat()}'", + f"Inconsistent certificate validity period: 'not_before' field '{self._not_before_string}' is not before than 'not_after' field '{self._not_after_string}'", ValueError, ) @@ -687,6 +687,16 @@ def identifier(self) -> str: """Get the identifier.""" return self._identifier + @property + def not_before_string(self) -> str: + """Get the not_before field as string.""" + return self._not_before_string + + @property + def not_after_string(self) -> str: + """Get the not_after field as string.""" + return self._not_after_string + @property def not_before(self) -> datetime.datetime: """Get the not_before field.""" diff --git a/tests/test_cli/test_issue_certificates.py b/tests/test_cli/test_issue_certificates.py new file mode 100644 index 0000000000..4102fc9525 --- /dev/null +++ b/tests/test_cli/test_issue_certificates.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2020 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This test module contains tests for 'issue-certificates' command.""" +from aea.test_tools.test_cases import AEATestCaseEmpty + + +class TestIssueCertificatesPositive(AEATestCaseEmpty): + """Test 'issue-certificates', positive case.""" diff --git a/tests/test_helpers/test_base.py b/tests/test_helpers/test_base.py index cf919c1aa8..e800fe0330 100644 --- a/tests/test_helpers/test_base.py +++ b/tests/test_helpers/test_base.py @@ -453,14 +453,18 @@ class TestCertRequestBadNotBefore(BaseTestCertRequestError): """Test instantiation of CertRequest class with bad not_before date.""" NOT_BEFORE = "bad-formatted-date" - ERROR_MESSAGE_PATTERN = "Invalid isoformat string: 'bad-formatted-date'" + ERROR_MESSAGE_PATTERN = ( + "time data 'bad-formatted-date' does not match format '%Y-%m-%d'" + ) class TestCertRequestBadNotAfter(BaseTestCertRequestError): """Test instantiation of CertRequest class with bad not_after date.""" NOT_AFTER = "bad-formatted-date" - ERROR_MESSAGE_PATTERN = "Invalid isoformat string: 'bad-formatted-date'" + ERROR_MESSAGE_PATTERN = ( + "time data 'bad-formatted-date' does not match format '%Y-%m-%d'" + ) class TestCertRequestInconsistentDates(BaseTestCertRequestError): @@ -468,7 +472,7 @@ class TestCertRequestInconsistentDates(BaseTestCertRequestError): NOT_BEFORE = "1954-06-07" NOT_AFTER = "1900-01-01" - ERROR_MESSAGE_PATTERN = r"Inconsistent certificate validity period: 'not_before' field '1954-06-07T00:00:00\+00:00' is not before than 'not_after' field '1900-01-01T00:00:01\+00:00'" + ERROR_MESSAGE_PATTERN = r"Inconsistent certificate validity period: 'not_before' field '1954-06-07' is not before than 'not_after' field '1900-01-01'" class BaseTestCertRequestInstantiation: From 244032dd7069c583fdce7b3e31aa83adea46ab35 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 22 Dec 2020 16:18:22 +0100 Subject: [PATCH 032/204] change to 'save_path' --- aea/configurations/schemas/definitions.json | 4 +-- aea/helpers/base.py | 30 ++++++++++----------- tests/data/dummy_connection/connection.yaml | 4 +-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/aea/configurations/schemas/definitions.json b/aea/configurations/schemas/definitions.json index 83ac84c525..3ca8ac3d08 100644 --- a/aea/configurations/schemas/definitions.json +++ b/aea/configurations/schemas/definitions.json @@ -167,7 +167,7 @@ "ledger_id", "not_before", "not_after", - "path" + "save_path" ], "properties": { "public_key": { @@ -185,7 +185,7 @@ "not_after": { "type": "string" }, - "path": { + "save_path": { "type": "string" } } diff --git a/aea/helpers/base.py b/aea/helpers/base.py index dca2f8d049..0abb2f3583 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -59,7 +59,7 @@ def _get_module(spec): def locate(path: str) -> Any: - """Locate an object by name or dotted path, importing as necessary.""" + """Locate an object by name or dotted save_path, importing as necessary.""" parts = [part for part in path.split(".") if part] module, n = None, 0 while n < len(parts): @@ -95,7 +95,7 @@ def load_module(dotted_path: str, filepath: Path) -> types.ModuleType: """ Load a module. - :param dotted_path: the dotted path of the package/module. + :param dotted_path: the dotted save_path of the package/module. :param filepath: the file to the package/module. :return: None :raises ValueError: if the filepath provided is not a module. @@ -111,7 +111,7 @@ def load_env_file(env_file: str): """ Load the content of the environment file into the process environment. - :param env_file: path to the env file. + :param env_file: save_path to the env file. :return: None. """ load_dotenv(dotenv_path=Path(env_file), override=False) @@ -404,13 +404,13 @@ def _get_aea_logger_name_prefix(module_name: str, agent_name: str) -> str: """ Get the logger name prefix. - It consists of a dotted path with: + It consists of a dotted save_path with: - the name of the package, 'aea'; - the agent name; - - the rest of the dotted path. + - the rest of the dotted save_path. - >>> _get_aea_logger_name_prefix("aea.path.to.package", "myagent") - 'aea.myagent.path.to.package' + >>> _get_aea_logger_name_prefix("aea.save_path.to.package", "myagent") + 'aea.myagent.save_path.to.package' :param module_name: the module name. :param agent_name: the agent name. @@ -585,7 +585,7 @@ def __init__( ledger_id: SimpleIdOrStr, not_before: str, not_after: str, - path: str, + save_path: str, ): """ Initialize the certificate request. @@ -598,7 +598,7 @@ def __init__( :param not_before: specify the lower bound for certificate vailidity. if it is a string, it must follow the format: 'YYYY-MM-DD' It will be interpreted as timezone UTC-0. - :param path: the path to the certificate. + :param save_path: the save_path where to save the certificate. """ self._key_identifier: Optional[str] = None self._public_key: Optional[str] = None @@ -608,7 +608,7 @@ def __init__( self._not_after_string = not_after self._not_before = self._parse_datetime(not_before) self._not_after = self._parse_datetime(not_after) - self._path = Path(path) + self._save_path = Path(save_path) self._parse_public_key(public_key) self._check_validation_boundaries() @@ -708,9 +708,9 @@ def not_after(self) -> datetime.datetime: return self._not_after @property - def path(self) -> Path: - """Get the path""" - return self._path + def save_path(self) -> Path: + """Get the save_path""" + return self._save_path @property def json(self) -> Dict: @@ -720,7 +720,7 @@ def json(self) -> Dict: ledger_id=self.ledger_id, not_before=self._not_before_string, not_after=self._not_after_string, - path=str(self.path), + save_path=str(self.save_path), ) if self.public_key is not None: result["public_key"] = self.public_key @@ -743,5 +743,5 @@ def __eq__(self, other): and self.key_identifier == other.key_identifier and self.not_after == other.not_after and self.not_before == other.not_before - and self.path == other.path + and self.save_path == other.save_path ) diff --git a/tests/data/dummy_connection/connection.yaml b/tests/data/dummy_connection/connection.yaml index 4598c6ac67..5f68a2d453 100644 --- a/tests/data/dummy_connection/connection.yaml +++ b/tests/data/dummy_connection/connection.yaml @@ -32,11 +32,11 @@ cert_requests: ledger_id: some_ledger_id not_after: '2020-01-02' not_before: '2020-01-01' - path: some/path_1 + save_path: some/path_1 public_key: key_id - identifier: cert_id_2 ledger_id: some_ledger_id not_after: '2020-01-02' not_before: '2020-01-01' - path: some/path_2 + save_path: some/path_2 public_key: '0xABCDEF123456' From 1dfcd2507d592ddbed1d00803e7e78225112975b Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 22 Dec 2020 17:18:06 +0100 Subject: [PATCH 033/204] fix typo on crypto dictionary --- aea/cli/issue_certificates.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/aea/cli/issue_certificates.py b/aea/cli/issue_certificates.py index 830e7b1a4e..4a73acd532 100644 --- a/aea/cli/issue_certificates.py +++ b/aea/cli/issue_certificates.py @@ -27,6 +27,7 @@ from aea.cli.utils.config import load_item_config from aea.cli.utils.context import Context from aea.cli.utils.decorators import check_aea_project +from aea.cli.utils.loggers import logger from aea.cli.utils.package_utils import get_package_path_unified from aea.configurations.base import ConnectionConfig, PublicId from aea.configurations.constants import CONNECTION @@ -45,14 +46,14 @@ def issue_certificates(click_context): for connection_id in ctx.agent_config.connections: _process_connection(ctx, connection_id) - click.echo("Done!") + click.echo("All certificates have been issued.") def _process_certificate(ctx: Context, cert_request: CertRequest): """Process a single certificate request.""" click.echo(f"Issuing certificate '{cert_request.identifier}'...") ledger_id = cert_request.ledger_id - output_path = cert_request.path + output_path = cert_request.save_path if cert_request.key_identifier is not None: key_identifier = cert_request.key_identifier connection_private_key_path = ctx.agent_config.connection_private_key_paths.read( @@ -76,28 +77,28 @@ def _process_certificate(ctx: Context, cert_request: CertRequest): identifier = cert_request.identifier.encode("ascii") not_before = cert_request.not_before_string.encode("ascii") not_after = cert_request.not_after_string.encode("ascii") - crytpo_private_key_path = ctx.agent_config.connection_private_key_paths.read( - ledger_id - ) - if crytpo_private_key_path is None: + crypto_private_key_path = ctx.agent_config.private_key_paths.read(ledger_id) + if crypto_private_key_path is None: raise ClickException(f"Cannot find private key with id '{ledger_id}'") - crypto = crypto_registry.make(ledger_id, private_key_path=crytpo_private_key_path) + crypto = crypto_registry.make(ledger_id, private_key_path=crypto_private_key_path) message = public_key_bytes + identifier + not_before + not_after signature = crypto.sign_message(message).encode("ascii") Path(output_path).write_bytes(signature) - click.echo(f"Dumped certificate in '{output_path}'.") def _process_connection(ctx: Context, connection_id: PublicId): - click.echo(f"Processing connection '{connection_id}'...") path = get_package_path_unified(ctx, CONNECTION, connection_id) connection_config = cast(ConnectionConfig, load_item_config(CONNECTION, Path(path))) if ( connection_config.cert_requests is None or len(connection_config.cert_requests) == 0 ): - click.echo("No certificates to process.") + logger.debug("No certificates to process.") return + logger.debug(f"Processing connection '{connection_id}'...") for cert_request in connection_config.cert_requests: _process_certificate(ctx, cert_request) + click.echo( + f"Dumped certificate '{cert_request.identifier}' in '{cert_request.save_path}' for connection {connection_id}." + ) From 4369269e472802d0b818742f270a89f2f1684f74 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 22 Dec 2020 17:31:58 +0100 Subject: [PATCH 034/204] update docs --- aea/cli/issue_certificates.py | 4 +- docs/cli-commands.md | 1 + tests/data/dummy_connection/connection.yaml | 4 +- tests/data/hashes.csv | 2 +- tests/test_cli/test_misc.py | 73 ++++++++++----------- 5 files changed, 43 insertions(+), 41 deletions(-) diff --git a/aea/cli/issue_certificates.py b/aea/cli/issue_certificates.py index 4a73acd532..582f17bf6e 100644 --- a/aea/cli/issue_certificates.py +++ b/aea/cli/issue_certificates.py @@ -51,7 +51,6 @@ def issue_certificates(click_context): def _process_certificate(ctx: Context, cert_request: CertRequest): """Process a single certificate request.""" - click.echo(f"Issuing certificate '{cert_request.identifier}'...") ledger_id = cert_request.ledger_id output_path = cert_request.save_path if cert_request.key_identifier is not None: @@ -98,6 +97,9 @@ def _process_connection(ctx: Context, connection_id: PublicId): logger.debug(f"Processing connection '{connection_id}'...") for cert_request in connection_config.cert_requests: + click.echo( + f"Issuing certificate '{cert_request.identifier}' for connection {connection_config.public_id}..." + ) _process_certificate(ctx, cert_request) click.echo( f"Dumped certificate '{cert_request.identifier}' in '{cert_request.save_path}' for connection {connection_id}." diff --git a/docs/cli-commands.md b/docs/cli-commands.md index 738a6bbbb5..64dac6a086 100644 --- a/docs/cli-commands.md +++ b/docs/cli-commands.md @@ -23,6 +23,7 @@ | `install [-r ]` | Install the dependencies. (With `--install-deps` to install dependencies.) | | `init` | Initialize your AEA configurations. (With `--author` to define author.) | | `interact` | Interact with a running AEA via the stub connection. | +| `issue-certificates` | Issue the connection certificates. | | `launch [path_to_agent_project]...` | Launch many agents at the same time. | | `list [package_type]` | List the installed resources. | | `login USERNAME [--password password]` | Login to a registry account with credentials. | diff --git a/tests/data/dummy_connection/connection.yaml b/tests/data/dummy_connection/connection.yaml index 5f68a2d453..5dca25e586 100644 --- a/tests/data/dummy_connection/connection.yaml +++ b/tests/data/dummy_connection/connection.yaml @@ -32,11 +32,11 @@ cert_requests: ledger_id: some_ledger_id not_after: '2020-01-02' not_before: '2020-01-01' - save_path: some/path_1 public_key: key_id + save_path: some/path_1 - identifier: cert_id_2 ledger_id: some_ledger_id not_after: '2020-01-02' not_before: '2020-01-01' - save_path: some/path_2 public_key: '0xABCDEF123456' + save_path: some/path_2 diff --git a/tests/data/hashes.csv b/tests/data/hashes.csv index 8d1d3cd01b..0cdb6108e6 100644 --- a/tests/data/hashes.csv +++ b/tests/data/hashes.csv @@ -1,6 +1,6 @@ dummy_author/agents/dummy_aea,QmPWxJVM9xnkRFtB41Wr1Xi2NGKpeRgDXUzbAzA8rferGh dummy_author/skills/dummy_skill,QmSmzcjGNhiz2KpSE6wSi19AeesCVjzupJm1NpXYJVnJuZ -fetchai/connections/dummy_connection,QmWCzxyUTivdaDiouSepKwjyomyshdzocSHDStXG16ZHkL +fetchai/connections/dummy_connection,QmYn34pVcTpKazRLUpgafNW917ndLZidq9nDFFiRW7YzbT fetchai/contracts/dummy_contract,QmXhMKRF5upco1efaC9oNcesStEMkPWKhBUv6CK9Vngk2S fetchai/protocols/t_protocol,QmNhzDCznorQpySrvKt4F4sQFXjzF1ZecEd5inixiNavDH fetchai/protocols/t_protocol_no_ct,QmWwMwHxvuPZJaMh44Bd1d5aBzYiogep8TJswTKkRwLfBu diff --git a/tests/test_cli/test_misc.py b/tests/test_cli/test_misc.py index afc8e2594a..a9f4d3ea75 100644 --- a/tests/test_cli/test_misc.py +++ b/tests/test_cli/test_misc.py @@ -60,42 +60,41 @@ def test_flag_help(): --help Show this message and exit. Commands: - add Add a package to the agent. - add-key Add a private key to the wallet of the agent. - build Build the agent and its components. - config Read or modify a configuration of the agent. - create Create a new agent. - delete Delete an agent. - eject Eject a vendor package of the agent. - fetch Fetch an agent from the registry. - fingerprint Fingerprint a non-vendor package of the agent. - freeze Get the dependencies of the agent. - generate Generate a package for the agent. - generate-key Generate a private key and place it in a file. - generate-wealth Generate wealth for the agent on a test network. - get-address Get the address associated with a private key of the... - get-multiaddress Get the multiaddress associated with a private key or... - get-wealth Get the wealth associated with the private key of the... - gui Run the CLI GUI. - init Initialize your AEA configurations. - install Install the dependencies of the agent. - interact Interact with the running agent via the stub connection. - launch Launch many agents at the same time. - list List the installed packages of the agent. - login Login to the registry account. - logout Logout from the registry account. - publish Publish the agent to the registry. - push Push a non-vendor package of the agent to the registry. - register Create a new registry account. - remove Remove a package from the agent. - remove-key Remove a private key from the wallet of the agent. - reset_password Reset the password of the registry account. - run Run the agent. - scaffold Scaffold a package for the agent. - search Search for packages in the registry. - transfer Transfer wealth associated with a private key of the - agent... - - upgrade Upgrade the packages of the agent. + add Add a package to the agent. + add-key Add a private key to the wallet of the agent. + build Build the agent and its components. + config Read or modify a configuration of the agent. + create Create a new agent. + delete Delete an agent. + eject Eject a vendor package of the agent. + fetch Fetch an agent from the registry. + fingerprint Fingerprint a non-vendor package of the agent. + freeze Get the dependencies of the agent. + generate Generate a package for the agent. + generate-key Generate a private key and place it in a file. + generate-wealth Generate wealth for the agent on a test network. + get-address Get the address associated with a private key of the... + get-multiaddress Get the multiaddress associated with a private key or... + get-wealth Get the wealth associated with the private key of the... + gui Run the CLI GUI. + init Initialize your AEA configurations. + install Install the dependencies of the agent. + interact Interact with the running agent via the stub... + issue-certificates Issue certificates for connections that require them. + launch Launch many agents at the same time. + list List the installed packages of the agent. + login Login to the registry account. + logout Logout from the registry account. + publish Publish the agent to the registry. + push Push a non-vendor package of the agent to the... + register Create a new registry account. + remove Remove a package from the agent. + remove-key Remove a private key from the wallet of the agent. + reset_password Reset the password of the registry account. + run Run the agent. + scaffold Scaffold a package for the agent. + search Search for packages in the registry. + transfer Transfer wealth associated with a private key of the... + upgrade Upgrade the packages of the agent. """ ) From 520705a7f91af91c45736577152260923415aa95 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 22 Dec 2020 19:08:49 +0100 Subject: [PATCH 035/204] full coverage to 'aea issue-certificates' --- aea/cli/issue_certificates.py | 5 +- tests/test_cli/test_issue_certificates.py | 163 +++++++++++++++++++++- tests/test_helpers/test_base.py | 2 +- 3 files changed, 166 insertions(+), 4 deletions(-) diff --git a/aea/cli/issue_certificates.py b/aea/cli/issue_certificates.py index 582f17bf6e..f9eb17c62e 100644 --- a/aea/cli/issue_certificates.py +++ b/aea/cli/issue_certificates.py @@ -81,8 +81,9 @@ def _process_certificate(ctx: Context, cert_request: CertRequest): raise ClickException(f"Cannot find private key with id '{ledger_id}'") crypto = crypto_registry.make(ledger_id, private_key_path=crypto_private_key_path) message = public_key_bytes + identifier + not_before + not_after - signature = crypto.sign_message(message).encode("ascii") - Path(output_path).write_bytes(signature) + signature = crypto.sign_message(message).encode("ascii").hex() + click.echo(f"Generated signature: '{signature}'") + Path(output_path).write_bytes(signature.encode("ascii")) def _process_connection(ctx: Context, connection_id: PublicId): diff --git a/tests/test_cli/test_issue_certificates.py b/tests/test_cli/test_issue_certificates.py index 4102fc9525..102c8f3022 100644 --- a/tests/test_cli/test_issue_certificates.py +++ b/tests/test_cli/test_issue_certificates.py @@ -17,8 +17,169 @@ # # ------------------------------------------------------------------------------ """This test module contains tests for 'issue-certificates' command.""" +import os +import shutil +from pathlib import Path +from typing import List + +import pytest + +from aea.cli.utils.config import dump_item_config +from aea.helpers.base import CertRequest from aea.test_tools.test_cases import AEATestCaseEmpty +from tests.conftest import ( + CUR_PATH, + FETCHAI, + FETCHAI_PRIVATE_KEY_FILE, + FETCHAI_PRIVATE_KEY_PATH, +) +from tests.data.dummy_connection.connection import DummyConnection + + +class BaseTestIssueCertificates(AEATestCaseEmpty): + """Base test class for 'aea issue-certificates' tests.""" + + @classmethod + def setup_class(cls): + """Set up the class.""" + super().setup_class() + + # add dummy connection + shutil.copytree( + os.path.join(CUR_PATH, "data", "dummy_connection"), + os.path.join(cls.current_agent_context, "connections", "dummy"), + ) + agent_config = cls.load_agent_config(cls.agent_name) + agent_config.author = FETCHAI + agent_config.connections.add(DummyConnection.connection_id) + dump_item_config(agent_config, Path(cls.current_agent_context)) -class TestIssueCertificatesPositive(AEATestCaseEmpty): + @classmethod + def add_cert_requests(cls, cert_requests: List[CertRequest], connection_name: str): + """Add certificate requests to a target connection.""" + cls.nested_set_config( + f"connections.{connection_name}.cert_requests", cert_requests + ) + + +class TestIssueCertificatesPositive(BaseTestIssueCertificates): """Test 'issue-certificates', positive case.""" + + @classmethod + def setup_class(cls): + """Set up the class.""" + super().setup_class() + + cls.expected_path_1 = "path_1" + cls.expected_path_2 = "path_2" + cls.cert_id_1 = "cert_id_1" + cls.cert_id_2 = "cert_id_2" + cls.cert_request_1 = CertRequest( + identifier=cls.cert_id_1, + ledger_id=FETCHAI, + not_after="2020-01-02", + not_before="2020-01-01", + public_key=FETCHAI, + save_path=cls.expected_path_1, + ) + cls.cert_request_2 = CertRequest( + identifier=cls.cert_id_2, + ledger_id=FETCHAI, + not_after="2020-01-02", + not_before="2020-01-01", + public_key="0xABCDEF123456", + save_path=cls.expected_path_2, + ) + cls.add_cert_requests( + [cls.cert_request_1, cls.cert_request_2], DummyConnection.connection_id.name + ) + + # add fetchai key and connection key + shutil.copy( + FETCHAI_PRIVATE_KEY_PATH, + os.path.join(cls.current_agent_context, FETCHAI_PRIVATE_KEY_FILE), + ) + cls.add_private_key() + cls.add_private_key(connection=True) + + def test_issue_certificate(self): + """Test 'aea issue-certificates' in case of success.""" + result = self.run_cli_command("issue-certificates", cwd=self._get_cwd()) + self._check_signature(self.cert_id_1, self.expected_path_1, result.stdout) + self._check_signature(self.cert_id_2, self.expected_path_2, result.stdout) + + def _check_signature(self, cert_id, filename, stdout): + """Check signature has been generated correctly.""" + path = Path(self.current_agent_context, filename) + assert path.exists() + signature = path.read_text() + assert signature.isascii() + int(signature, 16) # this will fail if not hexadecimal + + cert_msg_1 = ( + f"Issuing certificate '{cert_id}' for connection fetchai/dummy:0.1.0..." + ) + cert_msg_3 = f"Generated signature: '{signature}'" + cert_msg_2 = f"Dumped certificate '{cert_id}' in '{filename}' for connection fetchai/dummy:0.1.0." + assert cert_msg_1 in stdout + assert cert_msg_2 in stdout + assert cert_msg_3 in stdout + + +class TestIssueCertificatesWrongConnectionKey(BaseTestIssueCertificates): + """Test 'aea issue-certificates' when a bad connection key id is provided.""" + + @classmethod + def setup_class(cls): + """Set up class.""" + super().setup_class() + cls.cert_id_1 = "cert_id_1" + cls.cert_request_1 = CertRequest( + identifier=cls.cert_id_1, + ledger_id=FETCHAI, + not_after="2020-01-02", + not_before="2020-01-01", + public_key="bad_ledger_id", + save_path="path", + ) + cls.add_cert_requests([cls.cert_request_1], DummyConnection.connection_id.name) + + def test_run(self): + """Run the test.""" + with pytest.raises( + Exception, + match="Exception: Cannot find connection private key with id 'bad_ledger_id'", + ): + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) + + +class TestIssueCertificatesWrongCryptoKey(BaseTestIssueCertificates): + """Test 'aea issue-certificates' when a bad crypto key id is provided.""" + + @classmethod + def setup_class(cls): + """Set up class.""" + super().setup_class() + cls.cert_id_1 = "cert_id_1" + cls.cert_request_1 = CertRequest( + identifier=cls.cert_id_1, + ledger_id="bad_ledger_id", + not_after="2020-01-02", + not_before="2020-01-01", + public_key=FETCHAI, + save_path="path", + ) + cls.add_cert_requests([cls.cert_request_1], DummyConnection.connection_id.name) + # add fetchai key and connection key + cls.generate_private_key() + cls.add_private_key() + cls.add_private_key(connection=True) + + def test_run(self): + """Run the test.""" + with pytest.raises( + Exception, + match="Exception: Cannot find private key with id 'bad_ledger_id'", + ): + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) diff --git a/tests/test_helpers/test_base.py b/tests/test_helpers/test_base.py index e800fe0330..6ae47d597d 100644 --- a/tests/test_helpers/test_base.py +++ b/tests/test_helpers/test_base.py @@ -517,7 +517,7 @@ def test_instantiation(self): ) assert self.cert_request.not_after == expected_not_after - assert self.cert_request.path == Path(self.expected_path) + assert self.cert_request.save_path == Path(self.expected_path) def test_from_to_json(self): """Test from-to json methods.""" From 625212d8857b872a66ef5902056d82d3a02708d7 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Wed, 23 Dec 2020 12:55:04 +0300 Subject: [PATCH 036/204] AgentConfigManager implementation --- aea/aea_builder.py | 6 +- aea/cli/config.py | 468 +++++++++--------- aea/cli/utils/config.py | 10 +- aea/cli/utils/constants.py | 3 +- aea/cli/utils/generic.py | 32 +- aea/configurations/base.py | 227 ++++++++- .../schemas/aea-config_schema.json | 6 + .../schemas/connection-config_schema.json | 3 + .../schemas/contract-config_schema.json | 3 + aea/configurations/schemas/definitions.json | 18 + .../schemas/protocol-config_schema.json | 3 + .../schemas/skill-config_schema.json | 3 + aea/configurations/validation.py | 49 +- aea/helpers/base.py | 21 +- aea/skills/base.py | 1 + aea/test_tools/test_cases.py | 12 +- tests/test_cli/test_build.py | 5 +- tests/test_cli/test_config.py | 92 ++-- tests/test_cli/test_generate_wealth.py | 9 +- tests/test_cli/test_remove_key.py | 8 +- tests/test_cli/test_upgrade.py | 2 +- tests/test_configurations/test_aea_config.py | 3 +- tests/test_configurations/test_base.py | 2 +- tests/test_configurations/test_loader.py | 16 +- tests/test_test_tools/test_test_cases.py | 6 +- 25 files changed, 628 insertions(+), 380 deletions(-) diff --git a/aea/aea_builder.py b/aea/aea_builder.py index 340fdabb03..85d31cb617 100644 --- a/aea/aea_builder.py +++ b/aea/aea_builder.py @@ -755,7 +755,8 @@ def add_component( self._check_can_add(configuration) # update dependency graph self._package_dependency_manager.add_component(configuration) - configuration.directory = directory + if not configuration.directory: + configuration.directory = directory return self @@ -1679,6 +1680,9 @@ def _overwrite_custom_configuration(self, configuration: ComponentConfiguration) new_configuration.component_id, {} ) new_configuration.update(custom_config) + if new_configuration.directory != configuration.directory: + raise Exception(configuration) + return new_configuration def _add_components_of_type( diff --git a/aea/cli/config.py b/aea/cli/config.py index 9310ded548..9127ef9a3f 100644 --- a/aea/cli/config.py +++ b/aea/cli/config.py @@ -17,35 +17,50 @@ # # ------------------------------------------------------------------------------ """Implementation of the 'aea config' subcommand.""" +import contextlib import json from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Dict, List, NewType, Optional, Tuple, Union, cast import click +from click.exceptions import ClickException +from aea.aea_builder import AEABuilder from aea.cli.utils.config import ( + _try_get_component_id_from_prefix, _try_get_configuration_object_from_aea_config, handle_dotted_path, ) from aea.cli.utils.constants import ( CONFIG_SUPPORTED_KEY_TYPES, - CONFIG_SUPPORTED_VALUE_TYPES, FALSE_EQUIVALENTS, FROM_STRING_TO_TYPE, ) from aea.cli.utils.context import Context from aea.cli.utils.decorators import check_aea_project, pass_ctx -from aea.cli.utils.generic import get_parent_object, load_yaml from aea.configurations.base import ( AgentConfig, + ComponentConfiguration, ComponentId, DEFAULT_AEA_CONFIG_FILE, - PACKAGE_TYPE_TO_CONFIG_CLASS, PackageType, - SkillConfig, ) -from aea.configurations.loader import ConfigLoader +from aea.configurations.loader import ConfigLoader, load_component_configuration +from aea.configurations.validation import ExtraPropertiesError from aea.exceptions import AEAException +from aea.helpers.storage.backends.base import JSON_TYPES + + +class VariableDoesNotExists(ValueError): + """Variable does not exiss in a config exception.""" + + +JsonPath = List[str] +VariablePath = Union[str, JsonPath] + + +NotExistsType = NewType("NotExistsType", object) +NotExists = NotExistsType(None) @click.group() @@ -60,300 +75,273 @@ def config(click_context): # pylint: disable=unused-argument @pass_ctx def get(ctx: Context, json_path: str): """Get a field.""" - value = ConfigGetSet(ctx, json_path).get() + try: + value = AgentConfigManager(ctx.agent_config, ctx.cwd).get_variable(json_path) + except (ValueError, AEAException) as e: + raise ClickException(*e.args) + + if isinstance(value, dict): + # turn it to json compatible string, not dict str representation + value = json.dumps(value, sort_keys=True) click.echo(value) @config.command(name="set") @click.option( "--type", - default="str", - type=click.Choice(CONFIG_SUPPORTED_KEY_TYPES), + "type_", + default=None, + type=click.Choice(CONFIG_SUPPORTED_KEY_TYPES + [None]), # type: ignore help="Specify the type of the value.", ) @click.argument("JSON_PATH", required=True) @click.argument("VALUE", required=True, type=str) @pass_ctx def set_command( - ctx: Context, - json_path: str, - value: str, - type: str, # pylint: disable=redefined-builtin + ctx: Context, json_path: str, value: str, type_: Optional[str], ): """Set a field.""" - ConfigGetSet(ctx, json_path).set(value, type) - + try: + agent_config_manager = AgentConfigManager(ctx.agent_config, ctx.cwd) + + current_value = None + with contextlib.suppress(VariableDoesNotExists): + current_value = agent_config_manager.get_variable(json_path) + + if type_ is None: + if current_value is not None: + try: + converted_value = AgentConfigManager.convert_value_str_to_type( + value, type(current_value).__name__ + ) + except ValueError: + converted_value = AgentConfigManager.convert_value_str_to_type( + value, "str" + ) + else: + converted_value = AgentConfigManager.convert_value_str_to_type( + value, "str" + ) + else: + converted_value = AgentConfigManager.convert_value_str_to_type( + value, cast(str, type_) + ) -class ConfigGetSet: - """Tool to get/set value in agent config.""" + agent_config_manager.set_variable(json_path, converted_value) + agent_config_manager.dump_config() + except ExtraPropertiesError as e: + raise ClickException(f"Field `{e.args[0][0]}` is not allowed to change!") + except (ValueError, AEAException) as e: + raise ClickException(*e.args) - def __init__(self, ctx: Context, dotted_path: str) -> None: - """Init tool. - :param ctx: click context - :param dotted_path: str with attribute path to get/set - """ - self.dotted_path = dotted_path - self.ctx = ctx - - ( - self.json_path, - self.path_to_resource_configuration, - self.config_loader, - self.component_id, - ) = self._handle_dotted_path() - - def _handle_dotted_path( - self, - ) -> Tuple[List[str], Path, ConfigLoader, Optional[ComponentId]]: - """Handle dotted path.""" - try: - return handle_dotted_path(self.dotted_path, self.agent_config.author) - except AEAException as e: - raise click.ClickException(*e.args) +class AgentConfigManager: + """AeaConfig manager.""" - @property - def parent_obj_path(self) -> List[str]: - """Get the parent object (dotted) path.""" - return self.json_path[:-1] + component_configurations = "component_configurations" - @property - def attr_name(self) -> str: - """Attribute name.""" - return self.json_path[-1] + def __init__( + self, agent_config: AgentConfig, aea_project_directory: Union[str, Path] + ) -> None: + """ + Init manager. - def get(self) -> Union[str, int]: - """Get config value.""" - if self.component_id: - return self._get_component_value() + :param agent_config: AgentConfig to manage. + :param aea_project_directory: directory where project for agent_config placed. + """ + self.agent_config = agent_config + self.aea_project_directory = aea_project_directory - return self._get_agent_value() + def load_component_configuration( + self, component_id: ComponentId, skip_consistency_check: bool = True, + ) -> ComponentConfiguration: + """ + Load component configuration from the project directory. - def _get_agent_value(self) -> Union[str, int]: - """Get config value for agent config.""" - configuration_object = self._load_configuration_object() - return self._get_value_from_configuration_object(configuration_object) + :param component_id: Id of the component to load config for. + :param skip_consistency_check: bool. - def _get_component_value(self) -> Union[str, int]: - """Get config value for component section in agent config or component package.""" - configuration_object_from_agent = self._get_configuration_object_from_agent() - try: - if not configuration_object_from_agent: - raise click.ClickException("") - return self._get_value_from_configuration_object( - configuration_object_from_agent - ) - except click.ClickException: - configuration_object = self._load_configuration_object() - return self._get_value_from_configuration_object(configuration_object) + :return: ComponentConfiguration + """ + path = AEABuilder.find_component_directory_from_component_id( + aea_project_directory=Path(self.aea_project_directory), + component_id=component_id, + ) + return load_component_configuration( + component_type=component_id.component_type, + directory=path, + skip_consistency_check=skip_consistency_check, + ) @property - def is_target_agent(self) -> bool: - """Is target of get/update is agent config.""" - return self.component_id is None - - def _load_configuration_object(self) -> Dict: - """Load configuration object for component/agent.""" - if self.is_target_agent: - configuration_object = self.agent_config.json - else: - configuration_object = load_yaml(str(self.path_to_resource_configuration)) - - self.config_loader.validate(configuration_object) - return configuration_object + def agent_config_file_path(self) -> Path: + """Return agent config file path.""" + return Path(self.aea_project_directory) / DEFAULT_AEA_CONFIG_FILE - def _get_configuration_object_from_agent( - self, - ) -> Optional[Dict[str, Union[str, int]]]: - """Get component configuration object from agent component configurations.""" - if not self.component_id: # pragma: nocover - raise ValueError("component in not set") + @classmethod + def load(cls, aea_project_path: str) -> "AgentConfigManager": + """Create AgentConfigManager instance from agent project path.""" + raise NotImplementedError - return _try_get_configuration_object_from_aea_config( - self.ctx, self.component_id - ) + def set_variable(self, path: VariablePath, value: JSON_TYPES) -> None: + """ + Set config variable. - def _get_value_from_configuration_object( - self, conf_obj: Dict[str, Union[str, int]] - ) -> Any: - """Get value from configuration object.""" - return self._get_parent_object(conf_obj).get(self.attr_name) + :param path: str dotted path or List[Union[ComponentId, str]] + :param value: one of the json friendly objects. - def _get_parent_object( - self, conf_obj: Dict[str, Union[str, int]] - ) -> Dict[str, Union[str, int]]: + :return: None """ - Get and validate parent object. + component_id, json_path = self._parse_path(path) + data = self._make_dict_for_path_and_value(json_path, value) + overrides = {} + if component_id: + overrides[self.component_configurations] = {component_id: data} + else: + # agent + overrides.update(data) - :param conf_obj: configuration object. + self.update_config(overrides) - :return: parent object. - :raises: ClickException if attribute is not valid. + @staticmethod + def _make_dict_for_path_and_value(json_path: JsonPath, value: JSON_TYPES) -> Dict: """ - parent_obj_path = self.parent_obj_path - attr_name = self.attr_name - try: - parent_obj = get_parent_object(conf_obj, parent_obj_path) - except ValueError as e: - raise click.ClickException(str(e)) - - if attr_name not in parent_obj: - raise click.ClickException("Attribute '{}' not found.".format(attr_name)) - if not isinstance(parent_obj.get(attr_name), CONFIG_SUPPORTED_VALUE_TYPES): - raise click.ClickException( # pragma: nocover - "Attribute '{}' is not of primitive type.".format(attr_name) - ) - return parent_obj + Turn json_path and value into overrides dict. - @property - def agent_config(self) -> AgentConfig: - """Return current context AgentConfig.""" - return self.ctx.agent_config + :param json_path: List[str] represents config variable path: + :param value: json friendly value - def _check_set_field_name(self) -> None: + :return: dict of overrides + """ + data: Dict = {} + nested = data + for key in json_path[:-1]: + nested[key] = {} + nested = nested[key] + nested[json_path[-1]] = value + return data + + def get_variable(self, path: VariablePath) -> JSON_TYPES: """ - Check field names on set operation. + Set config variable. - :return: None + :param path: str dotted path or List[Union[ComponentId, str]] - :raises: click.ClickException is field is not allowed to be changeed. + :return: json friendly value. """ - top_level_key = self.json_path[0] + component_id, json_path = self._parse_path(path) - if self.component_id: - config_class = PACKAGE_TYPE_TO_CONFIG_CLASS[self.component_id.package_type] + if component_id: + configrations_data = [ + _try_get_configuration_object_from_aea_config( + self.agent_config, component_id + ) + or {}, + self.load_component_configuration(component_id).json, + ] else: - config_class = type(self.agent_config) + configrations_data = [self.agent_config.json] - if top_level_key not in config_class.FIELDS_ALLOWED_TO_UPDATE: - raise click.ClickException( - f"Field `{top_level_key}` is not allowed to change!" - ) - if config_class == SkillConfig: - if top_level_key not in SkillConfig.FIELDS_WITH_NESTED_FIELDS: - return # pragma: nocover - if len(self.json_path) < 3: - path = ".".join(self.json_path) - raise click.ClickException(f"Path '{path}' not valid for skill.") - second_level_key = self.json_path[1] - third_level_key = self.json_path[2] - if third_level_key not in SkillConfig.NESTED_FIELDS_ALLOWED_TO_UPDATE: - raise click.ClickException( # pragma: nocover - f"Field `{top_level_key}.{second_level_key}.{third_level_key}` is not allowed to change!" - ) + for data in configrations_data: + value = self._get_value_for_json_path(data, json_path) + if value is not NotExists: + return cast(JSON_TYPES, value) - def _fix_component_id_version(self) -> None: - """Update self.component_id with actual version defined in agent instead of latest.""" - if not self.component_id: # pragma: nocover: check for mypy - raise ValueError("Component id is not set") - - component_id = None - - for component_id in self.agent_config.package_dependencies: - if ( - component_id.author == self.component_id.author - and component_id.package_type == self.component_id.package_type - and component_id.name == self.component_id.name - ): - break - else: # pragma: nocover # should be always ok, cause component has to be alrady registered - raise ValueError("component is not registered?") - - self.component_id = component_id - - def _parent_object_for_agent_component_configuration( - self, - ) -> Dict[str, Union[str, int]]: - if not self.component_id: # pragma: nocover: check for mypy - raise ValueError("no component specified") - configuration_object = self.agent_config.component_configurations.get( - self.component_id, {} + raise VariableDoesNotExists( + f"Variable `{'.'.join(json_path)}` for {'{}({}) config'.format(component_id.component_type,component_id.public_id) if component_id else 'AgentConfig'} does not exist" ) - self.agent_config.component_configurations[ - self.component_id - ] = configuration_object - parent_object = configuration_object - # get or create parent object in component configuration - for i in self.parent_obj_path: - if i not in parent_object: - parent_object[i] = {} - parent_object = parent_object[i] - return parent_object - - def set(self, value: str, type_str: str) -> None: + + @staticmethod + def _get_value_for_json_path( + data: Dict, json_path: JsonPath + ) -> Union[NotExistsType, JSON_TYPES]: """ - Set config value. + Get value by json path from the dict object. - :param value: value to set - :param type_str: name of the value type. + :param data: dict to get value from + :param json_path: List[str] - :return None + :return: one of the json values of NotExists if value not presents in data dict. + """ + value = json.loads(json.dumps(data)) # in case or ordered dict + prev_key = "" + for key in json_path: + if not isinstance(value, dict): + raise ValueError(f"Attribute '{prev_key}' is not a dictionary.") + + if key not in value: + return NotExists + value = value[key] + prev_key = key + return value + + def _parse_path(self, path: VariablePath) -> Tuple[Optional[ComponentId], JsonPath]: """ - # do a check across real configuration - self._check_set_field_name() + Get component_id and json path from dotted path or list of str with first optional component id. - configuration_object = self._load_configuration_object() - parent_configuration_object = self._get_parent_object(configuration_object) + :param path: dotted path str, list of str with first optional component id - if self.component_id: - # component. parent is component config in agent config - self._fix_component_id_version() - parent_configuration_object = ( - self._parent_object_for_agent_component_configuration() + :return: Tuple of optonal component id if path related to component and List[str] + """ + if isinstance(path, str): + json_path, *_, component_id = handle_dotted_path( + path, self.agent_config.author ) - self._update_object(parent_configuration_object, type_str, value) - agent_configuration_object = self.agent_config.json else: - # already agent - self._update_object(parent_configuration_object, type_str, value) - agent_configuration_object = configuration_object - self._dump_agent_configuration(agent_configuration_object) + if isinstance(path[0], ComponentId): + json_path = path[1:] + component_id = path[0] + else: + component_id = None + json_path = path + if component_id: + component_id = _try_get_component_id_from_prefix( + set(self.agent_config.all_components_id), component_id.component_prefix + ) + return component_id, json_path - def _update_object(self, parent_object: Dict, type_str: str, value: str) -> None: + def update_config(self, overrides: Dict) -> None: """ - Update dict with value converted to type. + Apply overrides for agent config. - :param parent_object: dict where value should be updated, - :param: type_str: type name to convert value on update. - :param value: str of the value to set. + Validates and applies agent config and component overrides. + Does not save it on the disc! + + :param overrides: overrided values dictionary :return: None """ - type_ = FROM_STRING_TO_TYPE[type_str] - try: - if type_ == bool: - parent_object[self.attr_name] = value not in FALSE_EQUIVALENTS - elif type_ is None: - parent_object[self.attr_name] = None - elif type_ in (dict, list): - parent_object[self.attr_name] = json.loads(value) - else: - parent_object[self.attr_name] = type_(value) - except (ValueError, json.decoder.JSONDecodeError): # pragma: no cover - raise click.ClickException( - "Cannot convert {} to type {}".format(value, type_) - ) + for component_id, obj in overrides.get("component_configurations", {}).items(): + component_configuration = self.load_component_configuration(component_id) + component_configuration.check_overrides_valid(obj) - @property - def agent_config_loader(self) -> ConfigLoader: - """Return agent config loader.""" - return ConfigLoader.from_configuration_type(PackageType.AGENT) + self.agent_config.update(overrides) @property - def agent_config_file_path(self) -> Path: - """Return agent config file path.""" - return Path(".") / DEFAULT_AEA_CONFIG_FILE + def json(self) -> Dict: + """Return current agent config json representation.""" + return self.agent_config.json + + def dump_config(self) -> None: + """Save agent config on the disc.""" + config_data = self.json + self.agent_config.validate_config_data(config_data) + with open(self.agent_config_file_path, "w") as file_pointer: + ConfigLoader.from_configuration_type(PackageType.AGENT).dump( + self.agent_config, file_pointer + ) - def _dump_agent_configuration( - self, agent_configuration_object: Dict[str, Union[str, int]] - ) -> None: - """Save agent configuration.""" + @staticmethod + def convert_value_str_to_type(value: str, type_str: str) -> JSON_TYPES: + """Convert value by type name to native python type.""" try: - configuration_obj = self.agent_config_loader.configuration_class.from_json( - agent_configuration_object - ) - configuration_obj.validate_config_data(configuration_obj.json) - with open(self.agent_config_file_path, "w") as file_pointer: - self.agent_config_loader.dump(configuration_obj, file_pointer) - except Exception as e: # pragma: nocover - raise click.ClickException(f"Attribute or value not valid. {e}") + type_ = FROM_STRING_TO_TYPE[type_str] + if type_ == bool: + return value not in FALSE_EQUIVALENTS + if type_ is None: + return None + if type_ in (dict, list): + return json.loads(value) + return type_(value) + except (ValueError, json.decoder.JSONDecodeError): # pragma: no cover + raise ValueError("Cannot convert {} to type {}".format(value, type_)) diff --git a/aea/cli/utils/config.py b/aea/cli/utils/config.py index e7fae977d9..8579d6a420 100644 --- a/aea/cli/utils/config.py +++ b/aea/cli/utils/config.py @@ -55,6 +55,7 @@ from aea.cli.utils.exceptions import AEAConfigException from aea.cli.utils.generic import load_yaml from aea.configurations.base import ( + AgentConfig, ComponentId, ComponentType, PackageConfiguration, @@ -64,6 +65,7 @@ ) from aea.configurations.constants import AGENT, AGENTS, DEFAULT_AEA_CONFIG_FILE, VENDOR from aea.configurations.loader import ConfigLoader, ConfigLoaders +from aea.configurations.validation import ExtraPropertiesError from aea.exceptions import AEAEnforceError, AEAException, enforce @@ -95,7 +97,7 @@ def try_to_load_agent_config( DEFAULT_AEA_CONFIG_FILE ) ) - except jsonschema.exceptions.ValidationError: + except (jsonschema.exceptions.ValidationError, ExtraPropertiesError): if is_exit_on_except: raise click.ClickException( "Agent configuration file '{}' is invalid. Please check the documentation.".format( @@ -363,7 +365,7 @@ def validate_item_config(item_type: str, package_path: Path) -> None: def _try_get_configuration_object_from_aea_config( - ctx: Context, component_id: ComponentId + agent_config: AgentConfig, component_id: ComponentId ) -> Optional[Dict]: """ Try to get the configuration object in the AEA config. @@ -383,12 +385,12 @@ def _try_get_configuration_object_from_aea_config( component_id.author, component_id.name, ) - component_ids = set(ctx.agent_config.component_configurations.keys()) + component_ids = set(agent_config.component_configurations.keys()) true_component_id = _try_get_component_id_from_prefix( component_ids, (type_, author, name) ) if true_component_id is not None: - return ctx.agent_config.component_configurations.get(true_component_id) + return agent_config.component_configurations.get(true_component_id) return None diff --git a/aea/cli/utils/constants.py b/aea/cli/utils/constants.py index ce96134432..4ad2a8e4d3 100644 --- a/aea/cli/utils/constants.py +++ b/aea/cli/utils/constants.py @@ -64,7 +64,6 @@ str=str, int=int, bool=bool, float=float, dict=dict, list=list, none=None, ) CONFIG_SUPPORTED_KEY_TYPES = list(FROM_STRING_TO_TYPE.keys()) -CONFIG_SUPPORTED_VALUE_TYPES = (str, int, bool, float, dict, list, type(None)) ALLOWED_PATH_ROOTS = [ AGENT, @@ -80,6 +79,6 @@ CONNECTIONS: DEFAULT_CONNECTION_CONFIG_FILE, CONTRACTS: DEFAULT_CONTRACT_CONFIG_FILE, } # type: Dict[str, str] -FALSE_EQUIVALENTS = ["f", "false", "False"] +FALSE_EQUIVALENTS = ["f", "false", "False", "0"] REQUIREMENTS = "requirements.txt" diff --git a/aea/cli/utils/generic.py b/aea/cli/utils/generic.py index cb87d7a4ee..3d1d673ef8 100644 --- a/aea/cli/utils/generic.py +++ b/aea/cli/utils/generic.py @@ -18,42 +18,12 @@ # ------------------------------------------------------------------------------ """Module with generic utils of the aea cli.""" import os -from typing import Dict, List +from typing import Dict import yaml from click import ClickException -def get_parent_object(obj: Dict, dotted_path: List[str]): - """ - Given a nested dictionary, return the object denoted by the dotted path (if any). - - In particular if dotted_path = [], it returns the same object. - - :param obj: the dictionary. - :param dotted_path: the path to the object. - :return: the target dictionary - :raise ValueError: if the path is not valid. - """ - index = 0 - current_object = obj - while index < len(dotted_path): - current_attribute_name = dotted_path[index] - current_object = current_object.get(current_attribute_name, None) - # if the dictionary does not have the key we want, fail. - if current_object is None: - raise ValueError(f"Cannot get attribute '{current_attribute_name}'.") - if not isinstance(current_object, dict): - raise ValueError( - f"Attribute '{current_attribute_name}' is not a dictionary." - ) - index += 1 - # if we are not at the last step and the attribute value is not a dictionary, fail. - if isinstance(current_object, dict): - return current_object - raise ValueError("The target object is not a dictionary.") # pragma: nocover - - def load_yaml(filepath: str) -> Dict: """ Read content from yaml file. diff --git a/aea/configurations/base.py b/aea/configurations/base.py index adfa05712d..12ffde6c7e 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -31,6 +31,7 @@ Optional, Sequence, Set, + Tuple, Type, TypeVar, Union, @@ -72,7 +73,13 @@ ) from aea.configurations.validation import ConfigValidator from aea.exceptions import enforce -from aea.helpers.base import SimpleId, SimpleIdOrStr, load_module, recursive_update +from aea.helpers.base import ( + SimpleId, + SimpleIdOrStr, + dict_to_path_value, + load_module, + recursive_update, +) from aea.helpers.ipfs.base import IPFSHashOnly @@ -200,6 +207,7 @@ class PackageConfiguration(Configuration, ABC): package_type: PackageType FIELDS_ALLOWED_TO_UPDATE: FrozenSet[str] = frozenset() schema: str + CHECK_EXCLUDES: List[Tuple[str]] = [] def __init__( self, @@ -309,6 +317,13 @@ def update(self, data: Dict) -> None: :param data: the data to replace. :return: None """ + if not data: # do nothing if nothing to update + return + + self.check_overrides_valid(data) + self._create_or_update_from_json( + obj=self.make_resulting_config_data(data), instance=self + ) @classmethod def validate_config_data(cls, json_data: Dict) -> None: @@ -327,6 +342,75 @@ def _create_or_update_from_json( """Create new config object or updates existing one from json data.""" raise NotImplementedError + @staticmethod + def _compare_data_to_pattern( + data: dict, pattern: dict, excludes: Optional[List[Tuple[str]]] = None + ) -> List[str]: + excludes = excludes or [] + pattern_path_value = { + tuple(path): value for path, value in dict_to_path_value(pattern) + } + data_path_value = { + tuple(path): value for path, value in dict_to_path_value(data) + } + errors = [] + + def check_excludes(path): + for exclude in excludes: + if len(exclude) > len(path): + continue + + if path[: len(exclude)] == exclude: + return True + return False + + for path, new_value in data_path_value.items(): + if check_excludes(path): + continue + if path not in pattern_path_value: + errors.append(f"Field `{'.'.join(path)}` is not allowed to be updated!") + continue + + current_value = data_path_value[path] + + if current_value is None: + # not possible to determine data type for optional value not set + # it will be checked with jsonschema later + continue + + if not issubclass(type(new_value), type(current_value)): + errors.append( + f"For {'.'.join(path)} `{type(current_value)}` data type is expected, but `{type(new_value)}` was provided!" + ) + return errors + + def make_resulting_config_data(self, overrides: Dict) -> Dict: + """Make config data with overrides applied.""" + current_config = self.json + recursive_update(current_config, overrides, allow_new_values=True) + return current_config + + def check_overrides_valid(self, overrides: Dict) -> None: + """Check overrides is correct, return list of errors if present.""" + # check for permited overrides + self._check_overrides_corresponds_to_overridable(overrides) + # check resulting config with applied overrides passes validation + + result_config = self.make_resulting_config_data(overrides) + self.validate_config_data(result_config) + + def _check_overrides_corresponds_to_overridable(self, overrides: Dict) -> None: + """Check overrides is correct, return list of errors if present.""" + errors_list = self._compare_data_to_pattern( + overrides, self.get_overridable(), excludes=self.CHECK_EXCLUDES + ) + if errors_list: + raise ValueError(errors_list[0]) + + def get_overridable(self) -> dict: + """Get dictionary of values that can be updated for this config.""" + return {k: self.json.get(k) for k in self.FIELDS_ALLOWED_TO_UPDATE} + class ComponentConfiguration(PackageConfiguration, ABC): """Class to represent an agent component configuration.""" @@ -566,6 +650,7 @@ def _create_or_update_from_json( cls, obj: Dict, instance: Optional["ConnectionConfig"] = None ) -> "ConnectionConfig": """Create new config object or updates existing one from json data.""" + obj = {**(instance.json if instance else {}), **copy(obj)} restricted_to_protocols = obj.get("restricted_to_protocols", set()) restricted_to_protocols = { PublicId.from_str(id_) for id_ in restricted_to_protocols @@ -575,6 +660,7 @@ def _create_or_update_from_json( dependencies = dependencies_from_json(obj.get("dependencies", {})) protocols = {PublicId.from_str(id_) for id_ in obj.get(PROTOCOLS, set())} connections = {PublicId.from_str(id_) for id_ in obj.get(CONNECTIONS, set())} + params = dict( name=cast(str, obj.get("name")), author=cast(str, obj.get("author")), @@ -597,10 +683,16 @@ def _create_or_update_from_json( is_abstract=obj.get("is_abstract", False), **cast(dict, obj.get("config", {})), ) + directory = instance.directory if instance else None + if instance is None: instance = cls(**params) # type: ignore else: instance.__init__(**params) # type: ignore + + if not instance.directory: + instance.directory = directory + return instance def update(self, data: Dict) -> None: @@ -613,7 +705,7 @@ def update(self, data: Dict) -> None: :return: None """ new_config = data.get("config", {}) - recursive_update(self.config, new_config) + recursive_update(self.config, new_config, allow_new_values=True) self.is_abstract = data.get("is_abstract", self.is_abstract) @@ -683,6 +775,7 @@ def _create_or_update_from_json( cls, obj: Dict, instance: "ProtocolConfig" = Optional[None] ) -> "ProtocolConfig": """Initialize from a JSON object.""" + obj = {**(instance.json if instance else {}), **copy(obj)} dependencies = dependencies_from_json(obj.get("dependencies", {})) params = dict( name=cast(str, obj.get("name")), @@ -699,10 +792,16 @@ def _create_or_update_from_json( dependencies=dependencies, description=cast(str, obj.get("description", "")), ) + directory = instance.directory if instance else None + if instance is None: instance = cls(**params) # type: ignore else: instance.__init__(**params) # type: ignore + + if not instance.directory: + instance.directory = directory + return instance @@ -735,6 +834,7 @@ def _create_or_update_from_json( cls, obj: Dict, instance: Optional["SkillComponentConfiguration"] = None ) -> "SkillComponentConfiguration": """Initialize from a JSON object.""" + obj = {**(instance.json if instance else {}), **copy(obj)} class_name = cast(str, obj.get("class_name")) params = dict(class_name=class_name, **obj.get("args", {})) @@ -871,6 +971,7 @@ def _create_or_update_from_json( cls, obj: Dict, instance: "SkillConfig" = Optional[None] ) -> "SkillConfig": """Initialize from a JSON object.""" + obj = {**(instance.json if instance else {}), **copy(obj)} name = cast(str, obj.get("name")) author = cast(str, obj.get("author")) version = cast(str, obj.get("version")) @@ -903,14 +1004,19 @@ def _create_or_update_from_json( dependencies=dependencies, description=description, is_abstract=obj.get("is_abstract", False), - build_directory=cast(Optional[str], obj.get("build_directory")), + build_directory=obj.get("build_directory"), ) + directory = instance.directory if instance else None + if instance is None: instance = cls(**params) # type: ignore else: instance.__init__(**params) # type: ignore + if not instance.directory: + instance.directory = directory + for behaviour_id, behaviour_data in obj.get("behaviours", {}).items(): behaviour_config = SkillComponentConfiguration.from_json(behaviour_data) instance.behaviours.create(behaviour_id, behaviour_config) @@ -925,7 +1031,14 @@ def _create_or_update_from_json( return instance - def update(self, data: Dict) -> None: + def make_resulting_config_data(self, overrides: Dict) -> Dict: + """Make config data with overrides applied.""" + self._update(overrides) + current_config = self.json + recursive_update(current_config, overrides, allow_new_values=True) + return current_config + + def _update(self, data: Dict) -> None: """ Update configuration with other data. @@ -945,6 +1058,9 @@ def _update_skill_component_config(type_plural: str, data: Dict): new_component_config = data.get(type_plural, {}) all_component_names = dict(registry.read_all()) + if not isinstance(new_component_config, dict): + raise ValueError(f"Path '{type_plural}' not valid for skill.") + new_skill_component_names = set(new_component_config.keys()).difference( set(all_component_names.keys()) ) @@ -965,13 +1081,34 @@ def _update_skill_component_config(type_plural: str, data: Dict): raise ValueError( f"These fields of skill component configuration '{component_name}' of skill '{self.public_id}' are not allowed to change: {unallowed_keys}." ) - recursive_update(component_config.args, component_data.get("args", {})) + recursive_update( + component_config.args, + component_data.get("args", {}), + check_data_type=False, + ) _update_skill_component_config("behaviours", data) _update_skill_component_config("handlers", data) _update_skill_component_config("models", data) self.is_abstract = data.get("is_abstract", self.is_abstract) + def get_overridable(self) -> dict: + """Get overrideable confg data.""" + result = {} + current_config_data = self.json + if "is_abstract" in current_config_data: + result["is_abstract"] = current_config_data["is_abstract"] + + for field in self.FIELDS_WITH_NESTED_FIELDS: + result[field] = {} + for name in current_config_data[field].keys(): + result[field][name] = {} + for nested_field in self.NESTED_FIELDS_ALLOWED_TO_UPDATE: + result[field][name][nested_field] = current_config_data[field][ + name + ][nested_field] + return result + class AgentConfig(PackageConfiguration): """Class to represent the agent configuration file.""" @@ -1001,6 +1138,10 @@ class AgentConfig(PackageConfiguration): "storage_uri", ] ) + CHECK_EXCLUDES = [ + ("private_key_paths",), + ("default_routing",), + ] def __init__( self, @@ -1234,6 +1375,7 @@ def json(self) -> Dict: @classmethod def _create_or_update_from_json(cls, obj: Dict, instance=None) -> "AgentConfig": """Create new config object or updates existing one from json data.""" + obj = {**(instance.json if instance else {}), **copy(obj)} params = dict( agent_name=cast(str, obj.get("agent_name")), author=cast(str, obj.get("author")), @@ -1266,12 +1408,17 @@ def _create_or_update_from_json(cls, obj: Dict, instance=None) -> "AgentConfig": storage_uri=cast(str, obj.get("storage_uri")), component_configurations=None, ) + directory = instance.directory if instance else None + if instance is None: - # create - agent_config = AgentConfig(**params) # type: ignore + instance = cls(**params) # type: ignore else: - agent_config = instance - agent_config.__init__(**params) # type: ignore + instance.__init__(**params) # type: ignore + + if not instance.directory: + instance.directory = directory + + agent_config = instance #  parse private keys for crypto_id, path in obj.get("private_key_paths", {}).items(): @@ -1306,6 +1453,22 @@ def _create_or_update_from_json(cls, obj: Dict, instance=None) -> "AgentConfig": return agent_config + @property + def all_components_id(self) -> List[ComponentId]: + """Get list of the all components for this agent config.""" + component_type_to_set = { + ComponentType.PROTOCOL: self.protocols, + ComponentType.CONNECTION: self.connections, + ComponentType.CONTRACT: self.contracts, + ComponentType.SKILL: self.skills, + } + result = [] + for component_type, public_ids in component_type_to_set.items(): + for public_id in public_ids: + result.append(ComponentId(component_type, public_id)) + + return result + def update(self, data: Dict) -> None: """ Update configuration with other data. @@ -1319,26 +1482,23 @@ def update(self, data: Dict) -> None: data = copy(data) # update component parts new_component_configurations: Dict = data.pop("component_configurations", {}) - updated_component_configuration: Dict[ComponentId, Dict] = copy( + updated_component_configurations: Dict[ComponentId, Dict] = copy( self.component_configurations ) for component_id, obj in new_component_configurations.items(): - if component_id not in updated_component_configuration: - updated_component_configuration[component_id] = obj + if component_id not in updated_component_configurations: + updated_component_configurations[component_id] = obj else: - recursive_update(updated_component_configuration[component_id], obj) - - self.component_configurations = updated_component_configuration - - # update other fields - for item_id, value in data.get("private_key_paths", {}).items(): - self.private_key_paths.update(item_id, value) - - for item_id, value in data.get("connection_private_key_paths", {}).items(): - self.connection_private_key_paths.update(item_id, value) + recursive_update( + updated_component_configurations[component_id], + obj, + allow_new_values=True, + ) - self.logging_config = data.get("logging_config", self.logging_config) - self.registry_path = data.get("registry_path", self.registry_path) + self.check_overrides_valid(data) + super().update(data) + self.validate_config_data(self.json) + self.component_configurations = updated_component_configurations class SpeechActContentConfig(Configuration): @@ -1429,6 +1589,7 @@ def _create_or_update_from_json( # type: ignore cls, obj: Dict, instance: Optional["ProtocolSpecification"] = None ) -> "ProtocolSpecification": """Initialize from a JSON object.""" + obj = {**(instance.json if instance else {}), **copy(obj)} params = dict( name=cast(str, obj.get("name")), author=cast(str, obj.get("author")), @@ -1438,12 +1599,17 @@ def _create_or_update_from_json( # type: ignore description=cast(str, obj.get("description", "")), ) + directory = instance.directory if instance else None + if instance is None: - protocol_specification = cls(**params) # type: ignore + instance = cls(**params) # type: ignore else: - protocol_specification = instance - protocol_specification.__init__(**params) # type: ignore + instance.__init__(**params) # type: ignore + + if not instance.directory: + instance.directory = directory + protocol_specification = instance for speech_act, speech_act_content in obj.get("speech_acts", {}).items(): speech_act_content_config = SpeechActContentConfig.from_json( speech_act_content @@ -1529,6 +1695,7 @@ def _create_or_update_from_json( cls, obj: Dict, instance: "ContractConfig" = Optional[None] ) -> "ContractConfig": """Initialize from a JSON object.""" + obj = {**(instance.json if instance else {}), **copy(obj)} dependencies = cast( Dependencies, dependencies_from_json(obj.get("dependencies", {})) ) @@ -1551,10 +1718,16 @@ def _create_or_update_from_json( ), class_name=obj.get("class_name", ""), ) + directory = instance.directory if instance else None + if instance is None: instance = cls(**params) # type: ignore else: instance.__init__(**params) # type: ignore + + if not instance.directory: + instance.directory = directory + return instance diff --git a/aea/configurations/schemas/aea-config_schema.json b/aea/configurations/schemas/aea-config_schema.json index a8a495c46c..4db7cac10e 100644 --- a/aea/configurations/schemas/aea-config_schema.json +++ b/aea/configurations/schemas/aea-config_schema.json @@ -43,6 +43,9 @@ "build_entrypoint": { "$ref": "definitions.json#/definitions/build_entrypoint" }, + "build_directory": { + "$ref": "definitions.json#/definitions/build_directory" + }, "registry_path": { "type": "string" }, @@ -107,6 +110,9 @@ "description": { "$ref": "definitions.json#/definitions/description" }, + "logging_config" : { + "$ref": "definitions.json#/definitions/logging_config" + }, "period": { "$ref": "definitions.json#/definitions/period" }, diff --git a/aea/configurations/schemas/connection-config_schema.json b/aea/configurations/schemas/connection-config_schema.json index ee94c0eaed..6e1babce62 100644 --- a/aea/configurations/schemas/connection-config_schema.json +++ b/aea/configurations/schemas/connection-config_schema.json @@ -43,6 +43,9 @@ "build_entrypoint": { "$ref": "definitions.json#/definitions/build_entrypoint" }, + "build_directory": { + "$ref": "definitions.json#/definitions/build_directory" + }, "class_name": { "type": "string" }, diff --git a/aea/configurations/schemas/contract-config_schema.json b/aea/configurations/schemas/contract-config_schema.json index fa26c7bdf0..956779215f 100644 --- a/aea/configurations/schemas/contract-config_schema.json +++ b/aea/configurations/schemas/contract-config_schema.json @@ -40,6 +40,9 @@ "build_entrypoint": { "$ref": "definitions.json#/definitions/build_entrypoint" }, + "build_directory": { + "$ref": "definitions.json#/definitions/build_directory" + }, "dependencies": { "$ref": "definitions.json#/definitions/dependencies" }, diff --git a/aea/configurations/schemas/definitions.json b/aea/configurations/schemas/definitions.json index b71039b5c5..f4c4af9ba0 100644 --- a/aea/configurations/schemas/definitions.json +++ b/aea/configurations/schemas/definitions.json @@ -109,6 +109,9 @@ "build_entrypoint": { "type": "string" }, + "build_directory": { + "type": "string" + }, "ledger_id": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]{0,127}$" @@ -150,6 +153,21 @@ }, "keep_terminal_state_dialogues": { "type": "boolean" + }, + "logging_config": { + "type": "object", + "additionalProperties": false, + "properties": { + "disable_existing_loggers": { + "type": ["boolean", "null"] + }, + "version": { + "type": "number" + }, + "handlers": {"type": "object"}, + "formatters": {"type": "object"}, + "loggers": {"type": "object"} + } } } } diff --git a/aea/configurations/schemas/protocol-config_schema.json b/aea/configurations/schemas/protocol-config_schema.json index 55792479d4..093bd40631 100644 --- a/aea/configurations/schemas/protocol-config_schema.json +++ b/aea/configurations/schemas/protocol-config_schema.json @@ -39,6 +39,9 @@ "build_entrypoint": { "$ref": "definitions.json#/definitions/build_entrypoint" }, + "build_directory": { + "$ref": "definitions.json#/definitions/build_directory" + }, "dependencies": { "$ref": "definitions.json#/definitions/dependencies" }, diff --git a/aea/configurations/schemas/skill-config_schema.json b/aea/configurations/schemas/skill-config_schema.json index e5ce9dda11..db13ef890c 100644 --- a/aea/configurations/schemas/skill-config_schema.json +++ b/aea/configurations/schemas/skill-config_schema.json @@ -42,6 +42,9 @@ "build_entrypoint": { "$ref": "definitions.json#/definitions/build_entrypoint" }, + "build_directory": { + "$ref": "definitions.json#/definitions/build_directory" + }, "connections": { "type": "array", "additionalProperties": false, diff --git a/aea/configurations/validation.py b/aea/configurations/validation.py index b30c8513f7..32d55feb56 100644 --- a/aea/configurations/validation.py +++ b/aea/configurations/validation.py @@ -26,6 +26,10 @@ import jsonschema from jsonschema import Draft4Validator +from jsonschema._utils import find_additional_properties +from jsonschema._validators import additionalProperties +from jsonschema.exceptions import ValidationError +from jsonschema.validators import extend from aea.configurations.constants import AGENT from aea.configurations.data_types import ComponentId, ComponentType, PublicId @@ -69,6 +73,33 @@ def _get_path_to_custom_config_schema_from_type(component_type: ComponentType) - return str(full_path) +class ExtraPropertiesError(ValueError): + """Extra properties exception.""" + + def __str__(self) -> str: + """Get string representation of the object.""" + return ( + f"ExtraPropertiesError: properties not expected: {', '.join(self.args[0])}" + ) + + def __repr__(self) -> str: + """Get string representation of the object.""" + return str(self) + + +def ownAdditionalProperties(validator, aP, instance, schema): + """Additioinal properties validator.""" + for _ in additionalProperties(validator, aP, instance, schema): + raise ExtraPropertiesError(list(find_additional_properties(instance, schema))) + return iter(()) + + +OwnDraft4Validator = extend( + validator=Draft4Validator, + validators={"additionalProperties": ownAdditionalProperties}, +) + + class ConfigValidator: """Configuration validator implementation.""" @@ -82,7 +113,7 @@ def __init__(self, schema_filename: str): self._schema = json.load((base_uri / schema_filename).open()) root_path = make_jsonschema_base_uri(base_uri) self._resolver = jsonschema.RefResolver(root_path, self._schema) - self._validator = Draft4Validator(self._schema, resolver=self._resolver) + self._validator = OwnDraft4Validator(self._schema, resolver=self._resolver) @staticmethod def split_component_id_and_config( @@ -135,7 +166,7 @@ def validate_component_configuration( **configuration, ) ) - except jsonschema.ValidationError as e: + except (ExtraPropertiesError, jsonschema.ValidationError) as e: raise ValueError( f"Configuration of component {component_id} is not valid. {e}" ) from e @@ -165,9 +196,19 @@ def validate(self, json_data: Dict) -> None: ) # validate agent config - self._validator.validate(instance=json_data_copy) + self._validate_data(json_data_copy) else: - self._validator.validate(instance=json_data) + self._validate_data(json_data) + + def _validate_data(self, data): + try: + self._validator.validate(instance=data) + except ValidationError as e: + if e.validator == "additionalProperties": + extras = list(find_additional_properties(data, self._schema)) + if extras: + raise ExtraPropertiesError(extras) + raise @property def required_fields(self) -> List[str]: diff --git a/aea/helpers/base.py b/aea/helpers/base.py index a44188d997..d3152bb0f5 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -369,7 +369,12 @@ def _is_dict_like(obj: Any) -> bool: return type(obj) in {dict, OrderedDict} -def recursive_update(to_update: Dict, new_values: Dict) -> None: +def recursive_update( + to_update: Dict, + new_values: Dict, + allow_new_values: bool = False, + check_data_type: bool = True, +) -> None: """ Update a dictionary by replacing conflicts with the new values. @@ -386,11 +391,15 @@ def recursive_update(to_update: Dict, new_values: Dict) -> None: :return: None """ for key, value in new_values.items(): - if key not in to_update: + if (not allow_new_values) and key not in to_update: raise ValueError( f"Key '{key}' is not contained in the dictionary to update." ) + if key not in to_update and allow_new_values: + to_update[key] = value + continue + value_to_update = to_update[key] value_type = type(value) value_to_update_type = type(value_to_update) @@ -400,13 +409,14 @@ def recursive_update(to_update: Dict, new_values: Dict) -> None: and value_type != value_to_update_type and value is not None and value_to_update is not None + and check_data_type ): raise ValueError( f"Trying to replace value '{value_to_update}' with value '{value}' which is of different type." ) if both_are_dict: - recursive_update(value_to_update, value) + recursive_update(value_to_update, value, allow_new_values) else: to_update[key] = value @@ -585,10 +595,9 @@ def dict_to_path_value( """Convert dict to sequence of terminal path build of keys and value.""" path = path or [] for key, value in data.items(): - path.append(key) if not isinstance(value, Mapping): # terminal value - yield path, value + yield path + [key], value else: - for p, v in dict_to_path_value(value, path): + for p, v in dict_to_path_value(value, path + [key]): yield p, v diff --git a/aea/skills/base.py b/aea/skills/base.py index b00df5953b..2fdac58034 100644 --- a/aea/skills/base.py +++ b/aea/skills/base.py @@ -835,6 +835,7 @@ def from_config( :param agent_context: the agent context. :return: the skill. """ + if configuration.directory is None: # pragma: nocover raise ValueError("Configuration must be associated with a directory.") diff --git a/aea/test_tools/test_cases.py b/aea/test_tools/test_cases.py index 03073ae5b8..f8903577ff 100644 --- a/aea/test_tools/test_cases.py +++ b/aea/test_tools/test_cases.py @@ -107,7 +107,9 @@ def unset_agent_context(cls): cls.current_agent_context = "" @classmethod - def set_config(cls, dotted_path: str, value: Any, type_: str = "str") -> Result: + def set_config( + cls, dotted_path: str, value: Any, type_: Optional[str] = None + ) -> Result: """ Set a config. @@ -119,6 +121,9 @@ def set_config(cls, dotted_path: str, value: Any, type_: str = "str") -> Result: :return: Result """ + if type_ is None: + type_ = type(value).__name__ + return cls.run_cli_command( "config", "set", @@ -164,7 +169,10 @@ def run_cli_command(cls, *args: str, cwd: str = ".") -> Result: """ with cd(cwd): result = cls.runner.invoke( - cli, [*CLI_LOG_OPTION, *args], standalone_mode=False + cli, + [*CLI_LOG_OPTION, *args], + standalone_mode=False, + catch_exceptions=False, ) cls.last_cli_runner_result = result if result.exit_code != 0: diff --git a/tests/test_cli/test_build.py b/tests/test_cli/test_build.py index 74f50714d5..ea7c4d34e3 100644 --- a/tests/test_cli/test_build.py +++ b/tests/test_cli/test_build.py @@ -17,15 +17,16 @@ # # ------------------------------------------------------------------------------ + """This test module contains the tests for the `aea build` sub-command.""" import re from pathlib import Path from unittest import mock import pytest +from click.exceptions import ClickException from aea.configurations.constants import DEFAULT_VERSION -from aea.test_tools.exceptions import AEATestingException from aea.test_tools.test_cases import AEATestCaseEmpty from tests.common.utils import run_aea_subprocess @@ -107,5 +108,5 @@ class TestAEABuildEntrypointNegative(AEATestCaseEmpty): ) def test_build_exception(self, *_mock): """Test build exception.""" - with pytest.raises(AEATestingException, match="some error."): + with pytest.raises(ClickException, match="some error."): self.run_cli_command("build", cwd=self._get_cwd()) diff --git a/tests/test_cli/test_config.py b/tests/test_cli/test_config.py index 993d92ab3d..5694e6515f 100644 --- a/tests/test_cli/test_config.py +++ b/tests/test_cli/test_config.py @@ -155,14 +155,20 @@ def test_resource_not_existing(self): def test_attribute_not_found(self): """Test that the 'get' fails because the attribute is not found.""" - result = self.runner.invoke( - cli, - [*CLI_LOG_OPTION, "config", "get", "skills.dummy.non_existing_attribute"], - standalone_mode=False, - ) - assert result.exit_code == 1 - s = "Attribute 'non_existing_attribute' not found." - assert result.exception.message == s + with pytest.raises( + ClickException, match=r"Variable `.* for .* config does not exist" + ): + self.runner.invoke( + cli, + [ + *CLI_LOG_OPTION, + "config", + "get", + "skills.dummy.non_existing_attribute", + ], + standalone_mode=False, + catch_exceptions=False, + ) def test_get_whole_dict(self): """Test that getting the 'dummy' skill behaviours works.""" @@ -174,7 +180,7 @@ def test_get_whole_dict(self): assert result.exit_code == 0 assert ( result.output - == "{'dummy': {'args': {'behaviour_arg_1': 1, 'behaviour_arg_2': '2'}, 'class_name': 'DummyBehaviour'}}\n" + == '{"dummy": {"args": {"behaviour_arg_1": 1, "behaviour_arg_2": "2"}, "class_name": "DummyBehaviour"}}\n' ) def test_get_list(self): @@ -194,19 +200,20 @@ def test_get_list(self): def test_get_fails_when_getting_nested_object(self): """Test that getting a nested object in 'dummy' skill fails because path is not valid.""" - result = self.runner.invoke( - cli, - [ - *CLI_LOG_OPTION, - "config", - "get", - "skills.dummy.non_existing_attribute.dummy", - ], - standalone_mode=False, - ) - assert result.exit_code == 1 - s = "Cannot get attribute 'non_existing_attribute'." - assert result.exception.message == s + with pytest.raises( + ClickException, match=r"Variable `.* for .* config does not exist" + ): + self.runner.invoke( + cli, + [ + *CLI_LOG_OPTION, + "config", + "get", + "skills.dummy.non_existing_attribute.dummy", + ], + standalone_mode=False, + catch_exceptions=False, + ) def test_get_fails_when_getting_non_dict_attribute(self): """Test that the get fails because the path point to a non-dict object.""" @@ -283,6 +290,7 @@ def test_set_agent_logging_options(self): "set", "agent.logging_config.disable_existing_loggers", "True", + "--type=bool", ], standalone_mode=False, catch_exceptions=False, @@ -304,7 +312,7 @@ def test_set_agent_logging_options(self): def test_set_agent_incorrect_value(self): """Test setting the agent name.""" with pytest.raises( - ClickException, match="Field `not_agent_name` is not allowed to change!" + ClickException, match="Field `not_agent_name` is not allowed to be updated!" ): self.runner.invoke( cli, @@ -326,6 +334,7 @@ def test_set_type_bool(self): "--type=bool", ], standalone_mode=False, + catch_exceptions=False, ) assert result.exit_code == 0 @@ -342,6 +351,7 @@ def test_set_type_none(self): "--type=none", ], standalone_mode=False, + catch_exceptions=False, ) assert result.exit_code == 0 @@ -358,6 +368,7 @@ def test_set_type_dict(self): "--type=dict", ], standalone_mode=False, + catch_exceptions=False, ) assert result.exit_code == 0 @@ -374,6 +385,7 @@ def test_set_type_list(self): "--type=list", ], standalone_mode=False, + catch_exceptions=False, ) assert result.exit_code == 0 @@ -434,7 +446,7 @@ def test_set_nested_attribute_not_allowed(self): assert result.exit_code == 1 assert ( result.exception.message - == "Field `behaviours.dummy.config` is not allowed to change!" + == "Field `behaviours.dummy.config.behaviour_arg_1` is not allowed to be updated!" ) def test_no_recognized_root(self): @@ -498,7 +510,8 @@ def test_resource_not_existing(self): def test_attribute_not_found(self): """Test that the 'set' fails because the attribute is not found.""" with pytest.raises( - ClickException, match="Field `.*` is not allowed to change!" + ClickException, + match="Field `non_existing_attribute` is not allowed to be updated!", ): self.runner.invoke( cli, @@ -515,19 +528,21 @@ def test_attribute_not_found(self): def test_set_fails_when_setting_non_primitive_type(self): """Test that setting the 'dummy' skill behaviours fails because not a primitive type.""" - result = self.runner.invoke( - cli, - [*CLI_LOG_OPTION, "config", "set", "skills.dummy.behaviours", "value"], - standalone_mode=False, - ) - assert result.exit_code == 1 - s = "Path 'behaviours' not valid for skill." - assert result.exception.message == s + with pytest.raises( + ClickException, match="Field `behaviours` is not allowed to be updated!" + ): + self.runner.invoke( + cli, + [*CLI_LOG_OPTION, "config", "set", "skills.dummy.behaviours", "value"], + standalone_mode=False, + catch_exceptions=False, + ) def test_get_fails_when_setting_nested_object(self): """Test that setting a nested object in 'dummy' skill fails because path is not valid.""" with pytest.raises( - ClickException, match=r"Field `.*` is not allowed to change!" + ClickException, + match=r"Field `non_existing_attribute.dummy` is not allowed to be updated!", ): self.runner.invoke( cli, @@ -595,7 +610,9 @@ def teardown(self): def test_set_get_incorrect_path(self): """Fail on incorrect attribute tryed to be updated.""" - with pytest.raises(ClickException, match="Attribute .* not found."): + with pytest.raises( + ClickException, match="Variable `.*` for .* config does not exist" + ): self.runner.invoke( cli, [*CLI_LOG_OPTION, "config", "get", self.INCORRECT_PATH], @@ -603,7 +620,10 @@ def test_set_get_incorrect_path(self): catch_exceptions=False, ) - with pytest.raises(ClickException, match="Attribute '.*' not found."): + with pytest.raises( + ClickException, + match="Field `behaviours.dummy.args.behaviour_arg_100500` is not allowed to be updated!", + ): self.runner.invoke( cli, [ diff --git a/tests/test_cli/test_generate_wealth.py b/tests/test_cli/test_generate_wealth.py index ad20e03466..9afcbd1659 100644 --- a/tests/test_cli/test_generate_wealth.py +++ b/tests/test_cli/test_generate_wealth.py @@ -17,14 +17,13 @@ # # ------------------------------------------------------------------------------ """This test module contains the tests for commands in aea.cli.generate_wealth module.""" - from unittest import TestCase, mock import pytest from aea.cli import cli from aea.cli.generate_wealth import _try_generate_wealth -from aea.test_tools.exceptions import AEATestingException +from aea.exceptions import AEAException from aea.test_tools.test_cases import AEATestCaseMany from tests.conftest import ( @@ -108,7 +107,7 @@ def test_wealth_commands_negative(self): settings = {"unsupported_crypto": "path"} self.nested_set_config("agent.private_key_paths", settings) - with pytest.raises(AEATestingException) as excinfo: + with pytest.raises( + AEAException, match="Item not registered with id 'unsupported_crypto'." + ): self.generate_wealth() - - assert "Item not registered with id 'unsupported_crypto'." in str(excinfo.value) diff --git a/tests/test_cli/test_remove_key.py b/tests/test_cli/test_remove_key.py index c79c4ce2b5..383dc75595 100644 --- a/tests/test_cli/test_remove_key.py +++ b/tests/test_cli/test_remove_key.py @@ -16,11 +16,10 @@ # limitations under the License. # # ------------------------------------------------------------------------------ - """This test module contains the tests for the `aea remove-key` sub-command.""" import pytest +from click.exceptions import ClickException -from aea.test_tools.exceptions import AEATestingException from aea.test_tools.test_cases import AEATestCaseEmpty from tests.conftest import FETCHAI_PRIVATE_KEY_PATH @@ -66,11 +65,8 @@ class BaseTestRemovePrivateKeyNegative(AEATestCaseEmpty): def test_remove(self): """Test remove.""" - with pytest.raises(AEATestingException): + with pytest.raises(ClickException, match=self.EXPECTED_ERROR_MSG): self.remove_private_key(connection=self.WITH_CONNECTION) - assert self.last_cli_runner_result.exit_code == 1 - error_msg = str(self.last_cli_runner_result.exception) - assert error_msg == self.EXPECTED_ERROR_MSG class TestRemoveCryptoPrivateKeyNegative(BaseTestRemovePrivateKeyNegative): diff --git a/tests/test_cli/test_upgrade.py b/tests/test_cli/test_upgrade.py index 65c852d965..47bcc65c31 100644 --- a/tests/test_cli/test_upgrade.py +++ b/tests/test_cli/test_upgrade.py @@ -898,7 +898,7 @@ def test_default_routing_updated_correctly(self): ) assert ( result.stdout - == f"{{'{DefaultMessage.protocol_id}': '{StubConnection.connection_id}'}}\n" + == f'{{"{DefaultMessage.protocol_id}": "{StubConnection.connection_id}"}}\n' ) def test_default_connection_updated_correctly(self): diff --git a/tests/test_configurations/test_aea_config.py b/tests/test_configurations/test_aea_config.py index c7c91a5860..f3104185c3 100644 --- a/tests/test_configurations/test_aea_config.py +++ b/tests/test_configurations/test_aea_config.py @@ -440,6 +440,7 @@ def test_agent_configuration_dump_multipage_fails_bad_component_configuration(): ] = "not in specs!" fp = io.StringIO() with pytest.raises( - ValueError, match="Configuration of component .* is not valid..*'BAD FIELD'" + ValueError, + match="Configuration of component .* is not valid. ExtraPropertiesError: properties not expected: BAD FIELD", ): loader.dump(agent_config, fp) diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index 052d129a32..3569111453 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -284,7 +284,7 @@ def test_update_method_raises_error_if_we_try_to_change_classname_of_skill_compo with pytest.raises( ValueError, - match="These fields of skill component configuration 'error_handler' of skill 'fetchai/error:0.10.0' are not allowed to change: {'class_name'}.", + match="Field `handlers.error_handler.class_name` is not allowed to be updated!", ): skill_config.update(new_configurations) diff --git a/tests/test_configurations/test_loader.py b/tests/test_configurations/test_loader.py index d798ce7cf7..2c7b750601 100644 --- a/tests/test_configurations/test_loader.py +++ b/tests/test_configurations/test_loader.py @@ -30,7 +30,7 @@ import aea from aea.configurations.base import PackageType, ProtocolSpecification from aea.configurations.loader import ConfigLoader -from aea.configurations.validation import make_jsonschema_base_uri +from aea.configurations.validation import OwnDraft4Validator, make_jsonschema_base_uri from aea.protocols.generator.common import load_protocol_specification from tests.conftest import protocol_specification_files @@ -57,8 +57,8 @@ def test_config_loader_dump_component(): """Test ConfigLoader.dump""" config_loader = ConfigLoader.from_configuration_type(PackageType.PROTOCOL) configuration = MagicMock() - with mock.patch.object(aea.configurations.loader, "yaml_dump"), mock.patch( - "jsonschema.Draft4Validator.validate" + with mock.patch.object(aea.configurations.loader, "yaml_dump"), mock.patch.object( + OwnDraft4Validator, "validate" ), mock.patch("builtins.open"): config_loader.dump(configuration, open("foo")) @@ -67,9 +67,9 @@ def test_config_loader_dump_agent_config(): """Test ConfigLoader.dump""" config_loader = ConfigLoader.from_configuration_type(PackageType.AGENT) configuration = MagicMock(ordered_json={"component_configurations": []}) - with mock.patch.object(aea.configurations.loader, "yaml_dump_all"), mock.patch( - "jsonschema.Draft4Validator.validate" - ), mock.patch("builtins.open"): + with mock.patch.object( + aea.configurations.loader, "yaml_dump_all" + ), mock.patch.object(OwnDraft4Validator, "validate"), mock.patch("builtins.open"): config_loader.dump(configuration, open("foo")) @@ -92,7 +92,7 @@ def test_load_protocol_specification_only_first_part(): ) with mock.patch.object( yaml, "safe_load_all", return_value=[valid_protocol_specification] - ), mock.patch("builtins.open"), mock.patch("jsonschema.Draft4Validator.validate"): + ), mock.patch("builtins.open"), mock.patch.object(OwnDraft4Validator, "validate"): load_protocol_specification("foo") @@ -110,7 +110,7 @@ def test_load_protocol_specification_two_parts(): yaml, "safe_load_all", return_value=[valid_protocol_specification, valid_protocol_specification], - ), mock.patch("builtins.open"), mock.patch("jsonschema.Draft4Validator.validate"): + ), mock.patch("builtins.open"), mock.patch.object(OwnDraft4Validator, "validate"): load_protocol_specification("foo") diff --git a/tests/test_test_tools/test_test_cases.py b/tests/test_test_tools/test_test_cases.py index 882d7c0fca..00cf342c49 100644 --- a/tests/test_test_tools/test_test_cases.py +++ b/tests/test_test_tools/test_test_cases.py @@ -115,15 +115,15 @@ def test_agent_nested_set_connection_dependency(self): def test_agent_set(self): """Test agent test set from path.""" - value = "testvalue" + value = True key_name = "agent.logging_config.disable_existing_loggers" self.set_config(key_name, value) result = self.run_cli_command("config", "get", key_name, cwd=self._get_cwd()) - assert value in str(result.stdout_bytes) + assert str(value) in str(result.stdout_bytes) def test_agent_get_exception(self): """Test agent test get non exists key.""" - with pytest.raises(AEATestingException, match=".*bad_key.*"): + with pytest.raises(Exception, match=".*bad_key.*"): self.run_cli_command("config", "get", "agent.bad_key", cwd=self._get_cwd()) From fc06f777bdff8641e3260e954a21769dea2bddc2 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Wed, 23 Dec 2020 10:11:21 +0000 Subject: [PATCH 037/204] small misc fixes in docs --- docs/agent-oriented-development.md | 2 +- docs/app-areas.md | 2 +- docs/demos.md | 2 +- docs/glossary.md | 6 ++- docs/identity.md | 2 +- docs/index.md | 4 +- docs/interaction-protocol.md | 0 docs/language-agnostic-definition.md | 62 +++++++++++++++------------- docs/oef-ledger.md | 6 +-- docs/questions-and-answers.md | 7 ++-- docs/version.md | 2 +- docs/vision.md | 21 +++++----- 12 files changed, 61 insertions(+), 55 deletions(-) create mode 100644 docs/interaction-protocol.md diff --git a/docs/agent-oriented-development.md b/docs/agent-oriented-development.md index 006da42b37..1d7ac3d4f4 100644 --- a/docs/agent-oriented-development.md +++ b/docs/agent-oriented-development.md @@ -18,7 +18,7 @@ Note, this is not to say that facilitators and middlemen have no place in a mult ## Conflicting Environment -As discussed above, the notion of decentralisation extends as far as ownership and control. Therefore, the different components that make up a decentralised system may each be owned by a different entity, designed according to very different principles and standards, with heterogeneous software and hardware, and each with internal objectives that may be fundamentally inconsistent, worst yet contradictory, with those of others. +As discussed above, the notion of decentralisation extends as far as ownership and control. Therefore, the different components that make up a decentralised system may each be owned by a different entity, designed according to very different principles and standards, with heterogeneous software and hardware, and each with internal objectives that may be fundamentally inconsistent, worse yet contradictory, with those of others. As such, a distinctive characteristic of a multi-agent environment, is that it is inhabited by more than one agent (as the name suggests), where each agent may be owned potentially by a different stakeholder (individual, company, government). Since by design, each agent represents and looks after the interests of its owner(s), and because different stakeholders may have unaligned, conflicting, or contradictory interests, it is very common to have multi-agent systems in which the agents' objectives, values and preferences are unaligned, conflicting, or contradictory. diff --git a/docs/app-areas.md b/docs/app-areas.md index 1ccfcce5f9..880b586667 100644 --- a/docs/app-areas.md +++ b/docs/app-areas.md @@ -1,4 +1,4 @@ -An AEA is an intelligent agent whose goal is generating economic value for its owner. It can represent machines, humans, or data. +An AEA is an intelligent agent whose goal is generating economic value for its owner. It can represent machines, humans (individuals or organisations), or data. ## General application areas diff --git a/docs/demos.md b/docs/demos.md index 6d33a2fb6f..0c9d0bd737 100644 --- a/docs/demos.md +++ b/docs/demos.md @@ -1,5 +1,5 @@ We provide demo guides for multiple use-cases, each one involving several AEAs interacting in a different scenario. -These demos serve to highlight the concept of AEAs as well as provide inspiration for developers. +These demos serve to highlight the concept of AEAs as well as provide inspiration for developers. Demos should not be taken as production ready software, although every care is taken to fix bugs when reported. Demos are alphabetically sorted, we recommend you start with the weather skills demo. \ No newline at end of file diff --git a/docs/glossary.md b/docs/glossary.md index d9b632c063..09a5076d3e 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -1,5 +1,7 @@ This glossary defines a number of common terms used throughout the documentation. For the definition of framework components consult the API docs. -* AEA: an Autonomous Economic Agent (AEA) is a "an intelligent agent acting on an owner's behalf, with limited or no interference, and whose goal is to generate economic value to its owner". AEAs are a special type of agent. +* AEA: an Autonomous Economic Agent (AEA) is a "an intelligent agent acting on an owner's behalf, with limited or no interference, and whose goal is to generate economic value to its owner". AEAs are a special type of agent. [more] -* (Software) Agent: a software agent is a computer program that acts on behalf of an entity (e.g. individual, organisation, business). +* (Software) Agent: a software agent is a computer program that acts on behalf of an entity (e.g. individual, organisation, business). [more] + +* sOEF (Simple-OEF): The Simple-OEF, or soef, is a search and discovery mechanism for autonomous economic agents. [more] diff --git a/docs/identity.md b/docs/identity.md index 45cacf18df..b3db86b33c 100644 --- a/docs/identity.md +++ b/docs/identity.md @@ -8,7 +8,7 @@ The AEAs currently use the addresses associated with their private-public key pa Keys of an AEA -To learn how to generate a private-public key pair check out this section. +To learn how to generate a private-public key pair check out the CLI section. To learn more about public-key cryptography check out Wikipedia. diff --git a/docs/index.md b/docs/index.md index dc8a1321ac..ed73641528 100644 --- a/docs/index.md +++ b/docs/index.md @@ -18,7 +18,7 @@ AEAs act independently of constant input from their owner and autonomously execu AEAs are not: -* just any agents: AEAs' purpose is to generate economic value in a multi-stakeholder environment with competing incentives between agents. +* just any agents: AEAs' purpose is to generate economic value in a multi-stakeholder environment with competing incentives between agents. The represent humans, organisations or things. * APIs or sensors which do not have agency. * smart contracts which do not display any proactiveness and are purely reactive to external requests (=contract calls). * artificial general intelligence (AGI): AEAs can have a very narrow goal directed focus involving some economic gain and implemented via simple conditional logic. @@ -36,7 +36,7 @@ AEAs achieve their goals with the help of the Open Economic The AEA framework provides the developer with a number of features which cannot be found in this combination anywhere else: * the peer-to-peer agent communication network allows your AEAs to interact with all other AEAs over the public internet -* the search and discovery system sOEF allows your AEAs find other agents +* the search and discovery system sOEF allows your AEAs to find other agents * the AEA registry enables code sharing and re-use * the framework's crypto and ledger APIs make it possible for AEAs to interact with blockchains * reusability of code for interacting with blockchains is enabled via the contract packages diff --git a/docs/interaction-protocol.md b/docs/interaction-protocol.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/language-agnostic-definition.md b/docs/language-agnostic-definition.md index 28e13396e5..c5ba27371a 100644 --- a/docs/language-agnostic-definition.md +++ b/docs/language-agnostic-definition.md @@ -1,12 +1,12 @@ An Autonomous Economic Agent is, in technical terms, defined by the following characteristics:
    -
  • It MUST be capable of receiving and sending `Envelopes` which satisfy the following protobuf schema: +
  • It MUST be capable of receiving and sending Envelopes which satisfy the following protobuf schema: ``` proto syntax = "proto3"; -package fetch.aea; +package aea; message Envelope{ string to = 1; @@ -17,44 +17,47 @@ message Envelope{ } ``` -The format for the above fields, except `message`, is specified below. +The format for the above fields, except message, is specified below.
      -
    • to and sender: an address derived from the private key of a secp256k1-compatible elliptic curve
      • -
    • protocol_id: this must match a defined regular expression (see below) -
    • bytes: a bytes string representing a serialized message in the specified protocol
      • -
    • URI: this syntax
      • +
    • to and sender: an address derived from the private key of a secp256k1-compatible elliptic curve
      • +
    • protocol_id: this must match a defined regular expression (see below) +
    • message: a bytes string representing a serialized message in the specified protocol
      • +
    • uri: this syntax
    • -
  • It MUST implement each protocol's message with the required meta-fields: +
  • It MUST implement each protocol's message with the required meta-fields: ``` proto +import "google/protobuf/struct.proto"; + +message DialogueMessage { + int32 message_id = 1; + string dialogue_starter_reference = 2; + string dialogue_responder_reference = 3; + int32 target = 4; + bytes content = 5; +} - message DialogueMessage { - int32 message_id = 1; - string dialogue_starter_reference = 2; - string dialogue_responder_reference = 3; - int32 target = 4; - bytes content = 5; +message Message { + oneof message { + google.protobuf.Struct body = 1; + DialogueMessage dialogue_message = 2; } +} ``` - where `content` is replaced with the protocol specific content (see here for details). + where content is replaced with the protocol specific content (see here for details).
  • It MUST implement protocols according to their specification (see here for details). -
    -

    Note

    -

    This section is incomplete, and will be updated soon!

    -
    -
    • -
  • It SHOULD implement the `fetchai/default:0.10.0` protocol which satisfies the following protobuf schema: +
  • It SHOULD implement the fetchai/default:0.10.0 protocol which satisfies the following protobuf schema: ``` proto syntax = "proto3"; -package fetch.aea.Default; +package aea.fetchai.default; message DefaultMessage{ @@ -82,25 +85,28 @@ message DefaultMessage{ map error_data = 3; } + message End_Performative{} + oneof performative{ Bytes_Performative bytes = 5; - Error_Performative error = 6; + End_Performative end = 6; + Error_Performative error = 7; } } ```
    • -
  • The protocol id MUST match the following regular expression: ^[a-zA-Z0-9_]*/[a-zA-Z_][a-zA-Z0-9_]*:(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$
    • -
  • It is recommended that it processes `Envelopes` asynchronously. Note, the specification regarding the processing of messages does not impose any particular implementation, and the AEA can be designed to process envelopes either synchronously and asynchronously. However, asynchronous message handling enables the agent to be more responsive and scalable in maintaining many concurrent dialogues with its peers. +
  • The protocol id MUST match the following regular expression: ^[a-zA-Z0-9_]*/[a-zA-Z_][a-zA-Z0-9_]*:(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$
    • +
  • It is recommended that it processes Envelopes asynchronously. Note, the specification regarding the processing of messages does not impose any particular implementation, and the AEA can be designed to process envelopes either synchronously and asynchronously. However, asynchronous message handling enables the agent to be more responsive and scalable in maintaining many concurrent dialogues with its peers.
  • It MUST have an identity in the form of, at a minimum, an address derived from a public key and its associated private key (where the eliptic curve must be of type SECP256k1).
    • -
  • It SHOULD implement handling of errors using the `fetchai/default:0.10.0` protocol. The protobuf schema is given above. +
  • It SHOULD implement handling of errors using the fetchai/default:0.10.0 protocol. The protobuf schema is given above.
  • It MUST implement the following principles when handling messages:
      -
    • It MUST ALWAYS handle incoming envelopes/messages and NEVER raise an exception. This ensures another AEA cannot cause the agent to fail by sending a malicious envelope/message.
      • -
    • It MUST NEVER handle outgoing messages and ALWAYS raise an exception if this rule is violated, as this would imply that the handler is resolving a bug in the implementation.
      • +
    • It MUST ALWAYS handle incoming envelopes/messages and NEVER raise an exception when decoding and validating the message. This ensures another AEA cannot cause the agent to fail by sending a malicious envelope/message.
      • +
    • It MUST NEVER handle outgoing messages and ALWAYS raise an exception when validating the message. An exception implies that the handler is resolving a bug in the implementation.
diff --git a/docs/oef-ledger.md b/docs/oef-ledger.md index 3d9d0c2a6a..91f604bd36 100644 --- a/docs/oef-ledger.md +++ b/docs/oef-ledger.md @@ -14,9 +14,9 @@ The 'Open Economic Framework' (OEF) consists of protocols, languages and market At present, the OEF's capabilities are fulfilled by two components: -- a permissionless, public peer to peer (agent to agent) communication network, called the Agent Communication Network; -- a set of agent interaction protocols; and -- a centralized search and discovery system. +- a permissionless, public peer to peer (agent to agent) communication network, called the Agent Communication Network; +- a set of agent interaction protocols; and +- a centralized search and discovery system. The latter will be decentralized over time. diff --git a/docs/questions-and-answers.md b/docs/questions-and-answers.md index cee0990be8..a6ebcc2145 100644 --- a/docs/questions-and-answers.md +++ b/docs/questions-and-answers.md @@ -1,5 +1,5 @@
What is the Open Economic Framework (OEF)? -The 'Open Economic Framework' (OEF) is a node that enables search, discovery and communicate with possible clients or services. +The Open Economic Framework' (OEF) consists of protocols, languages and market mechanisms agents use to search and find each other, communicate with as well as trade with each other. As such the OEF defines the decentralised virtual environment that supplies and supports APIs for autonomous third-party software agents, also known as Autonomous Economic Agents (AEAs).

You can read more about the ledgers and the OEF here
@@ -21,15 +21,14 @@ You can read more about the Search and Discovery hereHow does an AEA use blockchain? The AEA framework enables the agents to interact with public blockchains to complete transactions. Currently, the framework supports -two different networks natively: the `Fetch.ai` network and the `Ethereum` network. +two different networks natively: the Fetch.ai network and the Ethereum network.

You can read more about the intergration of ledger here
How does one install third party libraries? -The framework supports the use of third-party libraries hosted on PyPI we can directly reference the external dependencies. -The `aea install` command will install each dependency that the specific AEA needs and is listed in the skill's YAML file. +The framework supports the use of third-party libraries hosted on PyPI. We can directly reference the external dependencies in an AEA package's configuration file. The aea install command will install each dependency that the specific AEA needs and is listed in the one of it's packages configuration files.
How does one connect to a database? diff --git a/docs/version.md b/docs/version.md index 7111169c3d..bbba76c672 100644 --- a/docs/version.md +++ b/docs/version.md @@ -1,3 +1,3 @@ -The current version of the Python based Autonomous Economic Agent framework is PyPI. The framework is under rapid development with frequent breaking changes in the run-up to `v1` due in Q4 2020. +The current version of the Python based Autonomous Economic Agent framework is PyPI. The framework is under rapid development with frequent breaking changes in the run-up to `v1` due in Q1 2021. The Python based AEA framework is in principle compatible with any AEA framework, independent of the language it is implemented in. The language agnostic definition provides details on the aspects an implementation has to satisfy to qualify as an AEA framework. diff --git a/docs/vision.md b/docs/vision.md index cda6727f73..5b919ef665 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -7,23 +7,22 @@ We are creating infrastructure for developers to build their own agent-based sol AEA users are, among others: -* Data scientists. -* Economists. -* Researchers (Artificial Intelligence, Machine Learning, Multi-Agent Systems). -* Engineers. -* Machine learning experts. -* Independent developers. -* Students and academics. -* Crypto connoisseurs and enthusiasts. -* Web developers. +* Data scientists +* Economists +* Researchers (Artificial Intelligence, Machine Learning, Multi-Agent Systems) +* Engineers +* Machine learning experts +* Independent developers +* Students and academics +* Crypto connoisseurs and enthusiasts +* Web developers ## Decentralised agent economy for businesses We envisage the AEA framework to be used by businesses of all sizes to deploy multi-agent solutions into the decentralized agent economy cultivated by Fetch.ai. -With start up grants we will kick start solutions while testing product-market fit and identifying our user base. - +With start up grants we kick start solutions while testing product-market fit and identifying our user base.
From c0ee2788140d9cb2ac515298dc53396322ad0563 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Wed, 23 Dec 2020 10:31:58 +0000 Subject: [PATCH 038/204] fix cert tests, refactor --- aea/cli/issue_certificates.py | 6 +----- aea/helpers/base.py | 10 ++++++++++ tests/test_cli/test_issue_certificates.py | 7 ++++++- tests/test_cli/test_misc.py | 4 ++-- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/aea/cli/issue_certificates.py b/aea/cli/issue_certificates.py index f9eb17c62e..6c26d9c8cd 100644 --- a/aea/cli/issue_certificates.py +++ b/aea/cli/issue_certificates.py @@ -72,15 +72,11 @@ def _process_certificate(ctx: Context, cert_request: CertRequest): public_key is not None, "Internal error - one of key_identifier or public_key must be not None.", ) - public_key_bytes = public_key.encode("ascii") - identifier = cert_request.identifier.encode("ascii") - not_before = cert_request.not_before_string.encode("ascii") - not_after = cert_request.not_after_string.encode("ascii") crypto_private_key_path = ctx.agent_config.private_key_paths.read(ledger_id) if crypto_private_key_path is None: raise ClickException(f"Cannot find private key with id '{ledger_id}'") crypto = crypto_registry.make(ledger_id, private_key_path=crypto_private_key_path) - message = public_key_bytes + identifier + not_before + not_after + message = cert_request.get_message(public_key) signature = crypto.sign_message(message).encode("ascii").hex() click.echo(f"Generated signature: '{signature}'") Path(output_path).write_bytes(signature.encode("ascii")) diff --git a/aea/helpers/base.py b/aea/helpers/base.py index 0abb2f3583..fb52013771 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -712,6 +712,16 @@ def save_path(self) -> Path: """Get the save_path""" return self._save_path + def get_message(self, public_key: str) -> bytes: + """Get the message to sign.""" + message = ( + public_key.encode("ascii") + + self.identifier.encode("ascii") + + self.not_before_string.encode("ascii") + + self.not_after_string.encode("ascii") + ) + return message + @property def json(self) -> Dict: """Compute the JSON representation.""" diff --git a/tests/test_cli/test_issue_certificates.py b/tests/test_cli/test_issue_certificates.py index 102c8f3022..fe18606814 100644 --- a/tests/test_cli/test_issue_certificates.py +++ b/tests/test_cli/test_issue_certificates.py @@ -114,7 +114,12 @@ def _check_signature(self, cert_id, filename, stdout): path = Path(self.current_agent_context, filename) assert path.exists() signature = path.read_text() - assert signature.isascii() + + def is_ascii(s): + """Check isascii method for all Python 3 versions""" + return all(ord(c) < 128 for c in s) + + assert is_ascii(signature) int(signature, 16) # this will fail if not hexadecimal cert_msg_1 = ( diff --git a/tests/test_cli/test_misc.py b/tests/test_cli/test_misc.py index a9f4d3ea75..f6aed6bdb3 100644 --- a/tests/test_cli/test_misc.py +++ b/tests/test_cli/test_misc.py @@ -79,14 +79,14 @@ def test_flag_help(): gui Run the CLI GUI. init Initialize your AEA configurations. install Install the dependencies of the agent. - interact Interact with the running agent via the stub... + interact Interact with the running agent via the stub connection. issue-certificates Issue certificates for connections that require them. launch Launch many agents at the same time. list List the installed packages of the agent. login Login to the registry account. logout Logout from the registry account. publish Publish the agent to the registry. - push Push a non-vendor package of the agent to the... + push Push a non-vendor package of the agent to the registry. register Create a new registry account. remove Remove a package from the agent. remove-key Remove a private key from the wallet of the agent. From e4c978c08cdd2122676769cc69c6e8e7884bb7ff Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Wed, 23 Dec 2020 13:05:20 +0100 Subject: [PATCH 039/204] fix aea_version field on 'aea create' --- aea/cli/create.py | 4 +++- aea/helpers/base.py | 21 +++++++++++++++++++++ tests/test_cli/test_create.py | 11 ++++++++++- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/aea/cli/create.py b/aea/cli/create.py index 5974d981eb..a39a12b060 100644 --- a/aea/cli/create.py +++ b/aea/cli/create.py @@ -24,6 +24,7 @@ from typing import Optional, cast import click +from packaging.version import Version import aea from aea.cli.add import add_item @@ -54,6 +55,7 @@ STATE_UPDATE_PROTOCOL, VENDOR, ) +from aea.helpers.base import compute_specifier_from_version @click.command() @@ -177,7 +179,7 @@ def _crete_agent_config(ctx: Context, agent_name: str, set_author: str) -> Agent """ agent_config = AgentConfig( agent_name=agent_name, - aea_version=aea.__version__, + aea_version=compute_specifier_from_version(Version(aea.__version__)), author=set_author, version=DEFAULT_VERSION, license_=DEFAULT_LICENSE, diff --git a/aea/helpers/base.py b/aea/helpers/base.py index fb52013771..acd8cb784f 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -38,6 +38,7 @@ from typing import Any, Callable, Deque, Dict, List, Optional, Set, TypeVar, Union from dotenv import load_dotenv +from packaging.version import Version from aea.exceptions import enforce @@ -755,3 +756,23 @@ def __eq__(self, other): and self.not_before == other.not_before and self.save_path == other.save_path ) + + +def compute_specifier_from_version(version: Version) -> str: + """ + Compute the specifier set from a version, by varying only on the patch number. + + I.e. from "{major}.{minor}.{patch}", return + + ">={major}.{minor}.0, <{major}.{minor + 1}.0" + + :param version: the version + :return: the specifier set + """ + new_major = version.major + new_minor_low = version.minor + new_minor_high = new_minor_low + 1 + lower_bound = Version(f"{new_major}.{new_minor_low}.0") + upper_bound = Version(f"{new_major}.{new_minor_high}.0") + specifier_set = f">={lower_bound}, <{upper_bound}" + return specifier_set diff --git a/tests/test_cli/test_create.py b/tests/test_cli/test_create.py index b8c6a6052c..a0a66c1fe3 100644 --- a/tests/test_cli/test_create.py +++ b/tests/test_cli/test_create.py @@ -33,6 +33,7 @@ import pytest import yaml from jsonschema import Draft4Validator +from packaging.version import Version import aea from aea.cli import cli @@ -134,7 +135,15 @@ def test_configuration_file_is_compliant_to_schema(self): def test_aea_version_is_correct(self): """Check that the aea version in the configuration file is correct, i.e. the same of the installed package.""" - assert self.agent_config["aea_version"] == aea.__version__ + expected_aea_version = Version(aea.__version__) + version_no_micro = Version( + f"{expected_aea_version.major}.{expected_aea_version.minor}.0" + ) + version_next_minor = Version( + f"{expected_aea_version.major}.{expected_aea_version.minor + 1}.0" + ) + version_range = f">={version_no_micro}, <{version_next_minor}" + assert self.agent_config["aea_version"] == version_range def test_agent_name_is_correct(self): """Check that the agent name in the configuration file is correct.""" From 8ee3cf5dfb78a272ec71e0239cecf0bb9c4422f0 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Wed, 23 Dec 2020 14:03:21 +0100 Subject: [PATCH 040/204] add tests --- scripts/bump_aea_version.py | 21 +-------------------- tests/test_helpers/test_base.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/scripts/bump_aea_version.py b/scripts/bump_aea_version.py index b4610a1147..09a9c776f0 100644 --- a/scripts/bump_aea_version.py +++ b/scripts/bump_aea_version.py @@ -34,6 +34,7 @@ DEFAULT_PROTOCOL_CONFIG_FILE, DEFAULT_SKILL_CONFIG_FILE, ) +from aea.helpers.base import compute_specifier_from_version from scripts.generate_ipfs_hashes import update_hashes @@ -103,26 +104,6 @@ def update_version_for_aea(new_version: str) -> str: return current_version -def compute_specifier_from_version(version: Version) -> str: - """ - Compute the specifier set from a version, by varying only on the patch number. - - I.e. from "{major}.{minor}.{patch}", return - - ">={major}.{minor}.0, <{major}.{minor + 1}.0" - - :param version: the version - :return: the specifier set - """ - new_major = version.major - new_minor_low = version.minor - new_minor_high = new_minor_low + 1 - lower_bound = Version(f"{new_major}.{new_minor_low}.0") - upper_bound = Version(f"{new_major}.{new_minor_high}.0") - specifier_set = f">={lower_bound}, <{upper_bound}" - return specifier_set - - def update_version_for_file(path: Path, current_version: str, new_version: str) -> None: """ Update version for file. diff --git a/tests/test_helpers/test_base.py b/tests/test_helpers/test_base.py index 6ae47d597d..4f926fbff9 100644 --- a/tests/test_helpers/test_base.py +++ b/tests/test_helpers/test_base.py @@ -31,12 +31,14 @@ from unittest.mock import patch import pytest +from packaging.version import Version from aea.exceptions import AEAEnforceError from aea.helpers.base import ( CertRequest, MaxRetriesError, RegexConstrainedString, + compute_specifier_from_version, ensure_dir, exception_log_and_reraise, find_topological_order, @@ -538,3 +540,15 @@ class TestCertRequestInstantiationWithKeyHex(BaseTestCertRequestInstantiation): PUBLIC_KEY = "0xABCDEF12345" EXPECTED_PUBLIC_KEY = "0xABCDEF12345" EXPECTED_KEY_IDENTIFIER = None + + +def test_compute_specifier_from_version(): + """Test function 'compute_specifier_from_version'.""" + + version = "0.1.5" + expected_range = ">=0.1.0, <0.2.0" + assert expected_range == compute_specifier_from_version(Version(version)) + + version = "1.1.5" + expected_range = ">=1.1.0, <1.2.0" + assert expected_range == compute_specifier_from_version(Version(version)) From e43815ee27157732b2f8399a0b75cbf8c0ef7dd0 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Wed, 23 Dec 2020 16:13:18 +0300 Subject: [PATCH 041/204] fixes --- aea/aea_builder.py | 5 +- aea/cli/config.py | 19 +-- aea/configurations/base.py | 173 ++++++-------------- aea/configurations/schemas/definitions.json | 2 +- aea/configurations/validation.py | 15 +- aea/helpers/base.py | 12 +- tests/test_cli/test_config.py | 21 +-- tests/test_configurations/test_base.py | 4 +- 8 files changed, 83 insertions(+), 168 deletions(-) diff --git a/aea/aea_builder.py b/aea/aea_builder.py index 85d31cb617..8bac447ade 100644 --- a/aea/aea_builder.py +++ b/aea/aea_builder.py @@ -755,8 +755,7 @@ def add_component( self._check_can_add(configuration) # update dependency graph self._package_dependency_manager.add_component(configuration) - if not configuration.directory: - configuration.directory = directory + configuration.directory = directory return self @@ -1680,8 +1679,6 @@ def _overwrite_custom_configuration(self, configuration: ComponentConfiguration) new_configuration.component_id, {} ) new_configuration.update(custom_config) - if new_configuration.directory != configuration.directory: - raise Exception(configuration) return new_configuration diff --git a/aea/cli/config.py b/aea/cli/config.py index 9127ef9a3f..fb9a46f4dc 100644 --- a/aea/cli/config.py +++ b/aea/cli/config.py @@ -108,21 +108,18 @@ def set_command( with contextlib.suppress(VariableDoesNotExists): current_value = agent_config_manager.get_variable(json_path) + # type was not specified, tried to auto determine if type_ is None: + # apply str as default type + converted_value = AgentConfigManager.convert_value_str_to_type(value, "str") if current_value is not None: - try: + # try to convert to original value's type + with contextlib.suppress(Exception): converted_value = AgentConfigManager.convert_value_str_to_type( value, type(current_value).__name__ ) - except ValueError: - converted_value = AgentConfigManager.convert_value_str_to_type( - value, "str" - ) - else: - converted_value = AgentConfigManager.convert_value_str_to_type( - value, "str" - ) else: + # convert to type specified by user converted_value = AgentConfigManager.convert_value_str_to_type( value, cast(str, type_) ) @@ -130,7 +127,7 @@ def set_command( agent_config_manager.set_variable(json_path, converted_value) agent_config_manager.dump_config() except ExtraPropertiesError as e: - raise ClickException(f"Field `{e.args[0][0]}` is not allowed to change!") + raise ClickException(f"Attribute `{e.args[0][0]}` is not allowed to change!") except (ValueError, AEAException) as e: raise ClickException(*e.args) @@ -248,7 +245,7 @@ def get_variable(self, path: VariablePath) -> JSON_TYPES: return cast(JSON_TYPES, value) raise VariableDoesNotExists( - f"Variable `{'.'.join(json_path)}` for {'{}({}) config'.format(component_id.component_type,component_id.public_id) if component_id else 'AgentConfig'} does not exist" + f"Attribute `{'.'.join(json_path)}` for {'{}({}) config'.format(component_id.component_type,component_id.public_id) if component_id else 'AgentConfig'} does not exist" ) @staticmethod diff --git a/aea/configurations/base.py b/aea/configurations/base.py index 12ffde6c7e..a902c6da5e 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -368,7 +368,9 @@ def check_excludes(path): if check_excludes(path): continue if path not in pattern_path_value: - errors.append(f"Field `{'.'.join(path)}` is not allowed to be updated!") + errors.append( + f"Attribute `{'.'.join(path)}` is not allowed to be updated!" + ) continue current_value = data_path_value[path] @@ -385,7 +387,10 @@ def check_excludes(path): return errors def make_resulting_config_data(self, overrides: Dict) -> Dict: - """Make config data with overrides applied.""" + """Make config data with overrides applied. + + Does not update config, just creates json representation + """ current_config = self.json recursive_update(current_config, overrides, allow_new_values=True) return current_config @@ -411,6 +416,25 @@ def get_overridable(self) -> dict: """Get dictionary of values that can be updated for this config.""" return {k: self.json.get(k) for k in self.FIELDS_ALLOWED_TO_UPDATE} + @classmethod + def _apply_params_to_instance( + cls, params: dict, instance: Optional["PackageConfiguration"] + ) -> "PackageConfiguration": + """Constructs or update instance with params provided.""" + directory = ( + instance.directory if instance and hasattr(instance, "directory") else None + ) + + if instance is None: + instance = cls(**params) # type: ignore + else: + instance.__init__(**params) # type: ignore + + if directory and not instance.directory: + instance.directory = directory + + return instance + class ComponentConfiguration(PackageConfiguration, ABC): """Class to represent an agent component configuration.""" @@ -683,15 +707,10 @@ def _create_or_update_from_json( is_abstract=obj.get("is_abstract", False), **cast(dict, obj.get("config", {})), ) - directory = instance.directory if instance else None - - if instance is None: - instance = cls(**params) # type: ignore - else: - instance.__init__(**params) # type: ignore - if not instance.directory: - instance.directory = directory + instance = cast( + ConnectionConfig, cls._apply_params_to_instance(params, instance) + ) return instance @@ -792,15 +811,7 @@ def _create_or_update_from_json( dependencies=dependencies, description=cast(str, obj.get("description", "")), ) - directory = instance.directory if instance else None - - if instance is None: - instance = cls(**params) # type: ignore - else: - instance.__init__(**params) # type: ignore - - if not instance.directory: - instance.directory = directory + instance = cast(ProtocolConfig, cls._apply_params_to_instance(params, instance)) return instance @@ -838,11 +849,21 @@ def _create_or_update_from_json( class_name = cast(str, obj.get("class_name")) params = dict(class_name=class_name, **obj.get("args", {})) + instance = cast( + SkillComponentConfiguration, cls._apply_params_to_instance(params, instance) + ) + + return instance + + @classmethod + def _apply_params_to_instance( + cls, params: dict, instance: Optional["SkillComponentConfiguration"] + ) -> "SkillComponentConfiguration": + """Constructs or update instance with params provided.""" if instance is None: instance = cls(**params) # type: ignore else: instance.__init__(**params) # type: ignore - return instance @@ -852,6 +873,7 @@ class SkillConfig(ComponentConfiguration): default_configuration_filename = DEFAULT_SKILL_CONFIG_FILE package_type = PackageType.SKILL schema = "skill-config_schema.json" + abstract_field_name = "is_abstract" FIELDS_ALLOWED_TO_UPDATE: FrozenSet[str] = frozenset( ["behaviours", "handlers", "models", "is_abstract"] @@ -1007,15 +1029,7 @@ def _create_or_update_from_json( build_directory=obj.get("build_directory"), ) - directory = instance.directory if instance else None - - if instance is None: - instance = cls(**params) # type: ignore - else: - instance.__init__(**params) # type: ignore - - if not instance.directory: - instance.directory = directory + instance = cast(SkillConfig, cls._apply_params_to_instance(params, instance)) for behaviour_id, behaviour_data in obj.get("behaviours", {}).items(): behaviour_config = SkillComponentConfiguration.from_json(behaviour_data) @@ -1031,75 +1045,18 @@ def _create_or_update_from_json( return instance - def make_resulting_config_data(self, overrides: Dict) -> Dict: - """Make config data with overrides applied.""" - self._update(overrides) - current_config = self.json - recursive_update(current_config, overrides, allow_new_values=True) - return current_config - - def _update(self, data: Dict) -> None: - """ - Update configuration with other data. - - :param data: the data to replace. - :return: None - """ - - def _update_skill_component_config(type_plural: str, data: Dict): - """ - Update skill component configurations with new data. - - Also check that there are not undeclared components. - """ - registry: CRUDCollection[SkillComponentConfiguration] = getattr( - self, type_plural - ) - new_component_config = data.get(type_plural, {}) - all_component_names = dict(registry.read_all()) - - if not isinstance(new_component_config, dict): - raise ValueError(f"Path '{type_plural}' not valid for skill.") - - new_skill_component_names = set(new_component_config.keys()).difference( - set(all_component_names.keys()) - ) - if len(new_skill_component_names) > 0: - raise ValueError( - f"The custom configuration for skill {self.public_id} includes new {type_plural}: {new_skill_component_names}. This is not allowed." - ) - - for component_name, component_data in data.get(type_plural, {}).items(): - component_config = cast( - SkillComponentConfiguration, registry.read(component_name) - ) - component_data_keys = set(component_data.keys()) - unallowed_keys = component_data_keys.difference( - SkillConfig.NESTED_FIELDS_ALLOWED_TO_UPDATE - ) - if len(unallowed_keys) > 0: - raise ValueError( - f"These fields of skill component configuration '{component_name}' of skill '{self.public_id}' are not allowed to change: {unallowed_keys}." - ) - recursive_update( - component_config.args, - component_data.get("args", {}), - check_data_type=False, - ) - - _update_skill_component_config("behaviours", data) - _update_skill_component_config("handlers", data) - _update_skill_component_config("models", data) - self.is_abstract = data.get("is_abstract", self.is_abstract) - def get_overridable(self) -> dict: """Get overrideable confg data.""" result = {} current_config_data = self.json - if "is_abstract" in current_config_data: - result["is_abstract"] = current_config_data["is_abstract"] + if self.abstract_field_name in current_config_data: + result[self.abstract_field_name] = current_config_data[ + self.abstract_field_name + ] for field in self.FIELDS_WITH_NESTED_FIELDS: + if not current_config_data.get(field, {}): + continue result[field] = {} for name in current_config_data[field].keys(): result[field][name] = {} @@ -1408,15 +1365,7 @@ def _create_or_update_from_json(cls, obj: Dict, instance=None) -> "AgentConfig": storage_uri=cast(str, obj.get("storage_uri")), component_configurations=None, ) - directory = instance.directory if instance else None - - if instance is None: - instance = cls(**params) # type: ignore - else: - instance.__init__(**params) # type: ignore - - if not instance.directory: - instance.directory = directory + instance = cast(AgentConfig, cls._apply_params_to_instance(params, instance)) agent_config = instance @@ -1599,15 +1548,9 @@ def _create_or_update_from_json( # type: ignore description=cast(str, obj.get("description", "")), ) - directory = instance.directory if instance else None - - if instance is None: - instance = cls(**params) # type: ignore - else: - instance.__init__(**params) # type: ignore - - if not instance.directory: - instance.directory = directory + instance = cast( + ProtocolSpecification, cls._apply_params_to_instance(params, instance) + ) protocol_specification = instance for speech_act, speech_act_content in obj.get("speech_acts", {}).items(): @@ -1718,15 +1661,7 @@ def _create_or_update_from_json( ), class_name=obj.get("class_name", ""), ) - directory = instance.directory if instance else None - - if instance is None: - instance = cls(**params) # type: ignore - else: - instance.__init__(**params) # type: ignore - - if not instance.directory: - instance.directory = directory + instance = cast(ContractConfig, cls._apply_params_to_instance(params, instance)) return instance diff --git a/aea/configurations/schemas/definitions.json b/aea/configurations/schemas/definitions.json index f4c4af9ba0..ddb05dfe8e 100644 --- a/aea/configurations/schemas/definitions.json +++ b/aea/configurations/schemas/definitions.json @@ -156,7 +156,7 @@ }, "logging_config": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "disable_existing_loggers": { "type": ["boolean", "null"] diff --git a/aea/configurations/validation.py b/aea/configurations/validation.py index 32d55feb56..298867408e 100644 --- a/aea/configurations/validation.py +++ b/aea/configurations/validation.py @@ -28,7 +28,6 @@ from jsonschema import Draft4Validator from jsonschema._utils import find_additional_properties from jsonschema._validators import additionalProperties -from jsonschema.exceptions import ValidationError from jsonschema.validators import extend from aea.configurations.constants import AGENT @@ -196,19 +195,9 @@ def validate(self, json_data: Dict) -> None: ) # validate agent config - self._validate_data(json_data_copy) + self._validator.validate(instance=json_data_copy) else: - self._validate_data(json_data) - - def _validate_data(self, data): - try: - self._validator.validate(instance=data) - except ValidationError as e: - if e.validator == "additionalProperties": - extras = list(find_additional_properties(data, self._schema)) - if extras: - raise ExtraPropertiesError(extras) - raise + self._validator.validate(instance=json_data) @property def required_fields(self) -> List[str]: diff --git a/aea/helpers/base.py b/aea/helpers/base.py index d3152bb0f5..dc73a42e18 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -370,10 +370,7 @@ def _is_dict_like(obj: Any) -> bool: def recursive_update( - to_update: Dict, - new_values: Dict, - allow_new_values: bool = False, - check_data_type: bool = True, + to_update: Dict, new_values: Dict, allow_new_values: bool = False, ) -> None: """ Update a dictionary by replacing conflicts with the new values. @@ -409,7 +406,6 @@ def recursive_update( and value_type != value_to_update_type and value is not None and value_to_update is not None - and check_data_type ): raise ValueError( f"Trying to replace value '{value_to_update}' with value '{value}' which is of different type." @@ -595,9 +591,9 @@ def dict_to_path_value( """Convert dict to sequence of terminal path build of keys and value.""" path = path or [] for key, value in data.items(): - if not isinstance(value, Mapping): + if isinstance(value, Mapping) and value: # terminal value - yield path + [key], value - else: for p, v in dict_to_path_value(value, path + [key]): yield p, v + else: + yield path + [key], value diff --git a/tests/test_cli/test_config.py b/tests/test_cli/test_config.py index 5694e6515f..14ad03da04 100644 --- a/tests/test_cli/test_config.py +++ b/tests/test_cli/test_config.py @@ -156,7 +156,7 @@ def test_resource_not_existing(self): def test_attribute_not_found(self): """Test that the 'get' fails because the attribute is not found.""" with pytest.raises( - ClickException, match=r"Variable `.* for .* config does not exist" + ClickException, match=r"Attribute `.* for .* config does not exist" ): self.runner.invoke( cli, @@ -201,7 +201,7 @@ def test_get_list(self): def test_get_fails_when_getting_nested_object(self): """Test that getting a nested object in 'dummy' skill fails because path is not valid.""" with pytest.raises( - ClickException, match=r"Variable `.* for .* config does not exist" + ClickException, match=r"Attribute `.* for .* config does not exist" ): self.runner.invoke( cli, @@ -312,7 +312,8 @@ def test_set_agent_logging_options(self): def test_set_agent_incorrect_value(self): """Test setting the agent name.""" with pytest.raises( - ClickException, match="Field `not_agent_name` is not allowed to be updated!" + ClickException, + match="Attribute `not_agent_name` is not allowed to be updated!", ): self.runner.invoke( cli, @@ -417,7 +418,7 @@ def test_set_skill_name_should_fail(self): def test_set_nested_attribute(self): """Test setting a nested attribute.""" path = "skills.dummy.behaviours.dummy.args.behaviour_arg_1" - new_value = "new_dummy_name" + new_value = "10" # cause old value is int result = self.runner.invoke( cli, [*CLI_LOG_OPTION, "config", "set", path, new_value], @@ -446,7 +447,7 @@ def test_set_nested_attribute_not_allowed(self): assert result.exit_code == 1 assert ( result.exception.message - == "Field `behaviours.dummy.config.behaviour_arg_1` is not allowed to be updated!" + == "Attribute `behaviours.dummy.config.behaviour_arg_1` is not allowed to be updated!" ) def test_no_recognized_root(self): @@ -511,7 +512,7 @@ def test_attribute_not_found(self): """Test that the 'set' fails because the attribute is not found.""" with pytest.raises( ClickException, - match="Field `non_existing_attribute` is not allowed to be updated!", + match="Attribute `non_existing_attribute` is not allowed to be updated!", ): self.runner.invoke( cli, @@ -529,7 +530,7 @@ def test_attribute_not_found(self): def test_set_fails_when_setting_non_primitive_type(self): """Test that setting the 'dummy' skill behaviours fails because not a primitive type.""" with pytest.raises( - ClickException, match="Field `behaviours` is not allowed to be updated!" + ClickException, match="Attribute `behaviours` is not allowed to be updated!" ): self.runner.invoke( cli, @@ -542,7 +543,7 @@ def test_get_fails_when_setting_nested_object(self): """Test that setting a nested object in 'dummy' skill fails because path is not valid.""" with pytest.raises( ClickException, - match=r"Field `non_existing_attribute.dummy` is not allowed to be updated!", + match=r"Attribute `non_existing_attribute.dummy` is not allowed to be updated!", ): self.runner.invoke( cli, @@ -611,7 +612,7 @@ def teardown(self): def test_set_get_incorrect_path(self): """Fail on incorrect attribute tryed to be updated.""" with pytest.raises( - ClickException, match="Variable `.*` for .* config does not exist" + ClickException, match="Attribute `.*` for .* config does not exist" ): self.runner.invoke( cli, @@ -622,7 +623,7 @@ def test_set_get_incorrect_path(self): with pytest.raises( ClickException, - match="Field `behaviours.dummy.args.behaviour_arg_100500` is not allowed to be updated!", + match="Attribute `behaviours.dummy.args.behaviour_arg_100500` is not allowed to be updated!", ): self.runner.invoke( cli, diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index 3569111453..a7ee851b45 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -260,7 +260,7 @@ def test_update_method_raises_error_if_skill_component_not_allowed(self): with pytest.raises( ValueError, - match="The custom configuration for skill fetchai/error:0.10.0 includes new behaviours: {'new_behaviour'}. This is not allowed.", + match="Attribute `behaviours.new_behaviour.args` is not allowed to be updated!", ): skill_config.update(new_configurations) @@ -284,7 +284,7 @@ def test_update_method_raises_error_if_we_try_to_change_classname_of_skill_compo with pytest.raises( ValueError, - match="Field `handlers.error_handler.class_name` is not allowed to be updated!", + match="Attribute `handlers.error_handler.class_name` is not allowed to be updated!", ): skill_config.update(new_configurations) From ff6ec9a224db9ad145a5bc4d9cfcd3500dee3063 Mon Sep 17 00:00:00 2001 From: ali Date: Wed, 23 Dec 2020 14:16:05 +0000 Subject: [PATCH 042/204] tests for confirmation_aw2 skill --- .../skills/confirmation_aw2/handlers.py | 4 +- .../confirmation_aw2/registration_db.py | 4 +- .../skills/confirmation_aw2/skill.yaml | 8 +- .../skills/confirmation_aw2/strategy.py | 5 +- packages/hashes.csv | 2 +- .../test_confirmation_aw2/__init__.py | 20 + .../test_confirmation_aw2/test_handlers.py | 253 ++++++++++ .../test_registration_db.py | 475 ++++++++++++++++++ .../test_confirmation_aw2/test_strategy.py | 249 +++++++++ 9 files changed, 1008 insertions(+), 12 deletions(-) create mode 100644 tests/test_packages/test_skills/test_confirmation_aw2/__init__.py create mode 100644 tests/test_packages/test_skills/test_confirmation_aw2/test_handlers.py create mode 100644 tests/test_packages/test_skills/test_confirmation_aw2/test_registration_db.py create mode 100644 tests/test_packages/test_skills/test_confirmation_aw2/test_strategy.py diff --git a/packages/fetchai/skills/confirmation_aw2/handlers.py b/packages/fetchai/skills/confirmation_aw2/handlers.py index 7c79a538ba..0f4c30aef3 100644 --- a/packages/fetchai/skills/confirmation_aw2/handlers.py +++ b/packages/fetchai/skills/confirmation_aw2/handlers.py @@ -70,7 +70,7 @@ def handle(self, message: Message) -> None: default_dialogues = cast(DefaultDialogues, self.context.default_dialogues) default_dialogue = cast(DefaultDialogue, default_dialogues.update(default_msg)) if default_dialogue is None: - self._handle_unidentified_dialogue(default_dialogue) + self._handle_unidentified_dialogue(default_msg) return # handle message @@ -83,7 +83,7 @@ def _handle_unidentified_dialogue(self, default_msg: DefaultMessage) -> None: """ Handle an unidentified dialogue. - :param fipa_msg: the message + :param default_msg: the message """ self.context.logger.info( f"received invalid default message={default_msg}, unidentified dialogue." diff --git a/packages/fetchai/skills/confirmation_aw2/registration_db.py b/packages/fetchai/skills/confirmation_aw2/registration_db.py index 156b8d4f18..f13f22cd70 100644 --- a/packages/fetchai/skills/confirmation_aw2/registration_db.py +++ b/packages/fetchai/skills/confirmation_aw2/registration_db.py @@ -117,7 +117,7 @@ def is_registered(self, address: str) -> bool: result = self._execute_single_sql(command, variables) return len(result) != 0 - def is_allowed_to_trade(self, address: str, mininum_hours_between_txs: int) -> bool: + def is_allowed_to_trade(self, address: str, minimum_hours_between_txs: int) -> bool: """Check if an address is registered.""" record = self.get_trade_table(address) if record is None: @@ -136,7 +136,7 @@ def is_allowed_to_trade(self, address: str, mininum_hours_between_txs: int) -> b first_trade, "%Y-%m-%d %H:%M:%S.%f" ) is_allowed_to_trade_ = now - first_trade_dt > datetime.timedelta( - hours=mininum_hours_between_txs + hours=minimum_hours_between_txs ) if not is_allowed_to_trade_: self.context.logger.info( diff --git a/packages/fetchai/skills/confirmation_aw2/skill.yaml b/packages/fetchai/skills/confirmation_aw2/skill.yaml index d75eeb3979..b8862de77a 100644 --- a/packages/fetchai/skills/confirmation_aw2/skill.yaml +++ b/packages/fetchai/skills/confirmation_aw2/skill.yaml @@ -11,9 +11,9 @@ fingerprint: __init__.py: QmdNXiy1gZ6GQvnesh9ZJBeS2T8aiGk1DsbdKLvRrRDiCz behaviours.py: QmWRRAXwJf7mEcV4L2DdG43zkyg55PBViiLnpy1Chj28xn dialogues.py: QmcUgBjxeytE5aAx3VvPyna5EcBuqck9KazG3HygCWjawv - handlers.py: QmYKoLkvxdpD1NFn5DUKrjQwPRUtm4jQxiywDhxPGqo7ff - registration_db.py: Qmf4oT8r3ZZ26TREt7VStZmAyXMKaoCWwSLBBshGHYhLdW - strategy.py: QmayZEjxiCcg1Vu18vKwsVvXxGSeKR1ySej5bMjshGXHVS + handlers.py: QmVeJuN78rMn3kWFbntfBUk6v88DYBW88ZQZ129KZo84wp + registration_db.py: QmWJkpgWRnyxYqfG5BqheU2UiuZVk59z76QX5YfzSZTASF + strategy.py: QmafVgQyrcetTPmURWWzqqFgkU2yY2DJp2G9FWwvxcZTmW fingerprint_ignore_patterns: [] connections: - fetchai/ledger:0.11.0 @@ -83,8 +83,8 @@ models: max_quantity: 100 max_tx_fee: 1 max_unit_price: 20 + minimum_hours_between_txs: 4 minimum_minutes_since_last_attempt: 2 - mininum_hours_between_txs: 4 search_query: constraint_type: == search_key: seller_service diff --git a/packages/fetchai/skills/confirmation_aw2/strategy.py b/packages/fetchai/skills/confirmation_aw2/strategy.py index 350a3ebd1f..8acbd6ad5a 100644 --- a/packages/fetchai/skills/confirmation_aw2/strategy.py +++ b/packages/fetchai/skills/confirmation_aw2/strategy.py @@ -39,7 +39,7 @@ def __init__(self, **kwargs) -> None: if aw1_aea is None: raise ValueError("aw1_aea must be provided!") self.aw1_aea = aw1_aea - self.mininum_hours_between_txs = kwargs.pop("mininum_hours_between_txs", 4) + self.minimum_hours_between_txs = kwargs.pop("mininum_hours_between_txs", 4) self.minimum_minutes_since_last_attempt = kwargs.pop( "minimum_minutes_since_last_attempt", 2 ) @@ -93,7 +93,7 @@ def is_valid_counterparty(self, counterparty: str) -> bool: return False self.last_attempt[counterparty] = datetime.datetime.now() if not registration_db.is_allowed_to_trade( - counterparty, self.mininum_hours_between_txs + counterparty, self.minimum_hours_between_txs ): return False return True @@ -120,7 +120,6 @@ def register_counterparty(self, counterparty: str, developer_handle: str) -> Non :param counterparty: the counterparty address :param developer_handle: the developer handle - :param data: the data :return: False """ registration_db = cast(RegistrationDB, self.context.registration_db) diff --git a/packages/hashes.csv b/packages/hashes.csv index f20cd2319f..af91e911b2 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -69,7 +69,7 @@ fetchai/skills/carpark_client,QmWabiH48yVv4hNaufU3jtCqe3GSEsmhMe5DEPeciTYTP3 fetchai/skills/carpark_detection,QmYKRWjC3VEU8QatkptXoGyyL3P9uR8reoDdmgLW2r318x fetchai/skills/coin_price,QmXRHQim1XGbij8QnJ4CPBksdGxmBmpbmPmHdrPAw3z3pE fetchai/skills/confirmation_aw1,QmQbqko8t8utxmfywvDqRhpE1gACVdLxrZNnv98U1aAYyR -fetchai/skills/confirmation_aw2,QmSsnfnQo35F1ryWnqWNgvcjR3PNwiHc7aPKP4Y5js6pU3 +fetchai/skills/confirmation_aw2,QmbzgZnGsRMP5JLvQ52iTkwztoG3a6Ya1iUSQ7K5MZs4Dn fetchai/skills/confirmation_aw3,QmPkTAzgHivkAs3fViiW3z8q7zkqAN8DS3DcKne992sS2u fetchai/skills/echo,QmTir9ngs1i9dLweJs5gB8Q5NLqDufuJ5A3A2bznibKPMk fetchai/skills/erc1155_client,QmTTuStPpw2zygu9dd9cNkJnzjSVihiNHu1qACSYBT5fvL diff --git a/tests/test_packages/test_skills/test_confirmation_aw2/__init__.py b/tests/test_packages/test_skills/test_confirmation_aw2/__init__.py new file mode 100644 index 0000000000..156d382aaf --- /dev/null +++ b/tests/test_packages/test_skills/test_confirmation_aw2/__init__.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""The tests module contains the tests of the packages/skills/confirmation_aw2 dir.""" diff --git a/tests/test_packages/test_skills/test_confirmation_aw2/test_handlers.py b/tests/test_packages/test_skills/test_confirmation_aw2/test_handlers.py new file mode 100644 index 0000000000..15f8f80463 --- /dev/null +++ b/tests/test_packages/test_skills/test_confirmation_aw2/test_handlers.py @@ -0,0 +1,253 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the tests of the handler classes of the confirmation aw2 skill.""" + +import logging +from pathlib import Path +from typing import cast +from unittest.mock import patch + +from aea.crypto.ledger_apis import LedgerApis +from aea.protocols.dialogue.base import DialogueMessage +from aea.test_tools.test_skill import BaseSkillTestCase + +from packages.fetchai.protocols.default.dialogues import DefaultDialogue +from packages.fetchai.protocols.default.message import DefaultMessage +from packages.fetchai.protocols.register.message import RegisterMessage +from packages.fetchai.skills.confirmation_aw2.dialogues import DefaultDialogues +from packages.fetchai.skills.confirmation_aw2.handlers import DefaultHandler +from packages.fetchai.skills.confirmation_aw2.strategy import Strategy + +from tests.conftest import ROOT_DIR + + +class TestDefaultHandler(BaseSkillTestCase): + """Test default handler of confirmation aw2.""" + + path_to_skill = Path(ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw2") + + @classmethod + def setup(cls): + """Setup the test class.""" + cls.aw1_aea = "some_aw1_aea" + config_overrides = {"models": {"strategy": {"args": {"aw1_aea": cls.aw1_aea}}}} + + super().setup(config_overrides=config_overrides) + cls.default_handler = cast( + DefaultHandler, cls._skill.skill_context.handlers.default_handler + ) + cls.logger = cls._skill.skill_context.logger + cls.strategy = cast(Strategy, cls._skill.skill_context.strategy) + + cls.default_dialogues = cast( + DefaultDialogues, cls._skill.skill_context.default_dialogues + ) + + cls.list_of_default_messages = ( + DialogueMessage( + DefaultMessage.Performative.BYTES, {"content": b"some_content"} + ), + ) + + cls.confirmed_aea = b"ConfirmedAEA" + cls.developer_handle = b"DeveloperHandle" + + def test_setup(self): + """Test the setup method of the default handler.""" + assert self.default_handler.setup() is None + self.assert_quantity_in_outbox(0) + + def test_handle_unidentified_dialogue(self): + """Test the _handle_unidentified_dialogue method of the register handler.""" + # setup + incorrect_dialogue_reference = ("", "") + incoming_message = self.build_incoming_message( + message_type=DefaultMessage, + dialogue_reference=incorrect_dialogue_reference, + performative=DefaultMessage.Performative.BYTES, + content=b"some_content", + ) + + # operation + with patch.object(self.logger, "log") as mock_logger: + self.default_handler.handle(incoming_message) + + # after + mock_logger.assert_any_call( + logging.INFO, + f"received invalid default message={incoming_message}, unidentified dialogue.", + ) + + def test_handle_bytes_i(self): + """Test the _handle_bytes method of the default handler where the sender IS aw1_aea.""" + # setup + incoming_message = cast( + RegisterMessage, + self.build_incoming_message( + message_type=DefaultMessage, + performative=DefaultMessage.Performative.BYTES, + content=self.confirmed_aea + b"_" + self.developer_handle, + sender=self.aw1_aea, + ), + ) + + # operation + with patch.object(LedgerApis, "is_valid_address", return_value=True): + with patch.object(self.logger, "log") as mock_logger: + with patch.object( + self.strategy, "register_counterparty" + ) as mock_register: + self.default_handler.handle(incoming_message) + + # after + mock_register.called_once() + + mock_logger.assert_any_call( + logging.INFO, + f"adding confirmed_aea={self.confirmed_aea.decode('utf-8')} with developer_handle={self.developer_handle.decode('utf-8')} to db.", + ) + + def test_handle_bytes_ii(self): + """Test the _handle_bytes method of the default handler where the content is undecodable.""" + # setup + incorrect_content = "some_incorrect_content" + + incoming_message = cast( + RegisterMessage, + self.build_incoming_message( + message_type=DefaultMessage, + performative=DefaultMessage.Performative.BYTES, + content=incorrect_content, + sender=self.aw1_aea, + ), + ) + + # operation + with patch.object(LedgerApis, "is_valid_address", return_value=True): + with patch.object(self.logger, "log") as mock_logger: + with patch.object( + self.strategy, "register_counterparty" + ) as mock_register: + self.default_handler.handle(incoming_message) + + # after + mock_register.called_once() + + mock_logger.assert_any_call( + logging.WARNING, "received invalid developer_handle=." + ) + + def test_handle_bytes_iii(self): + """Test the _handle_bytes method of the default handler where is_valid_address is False.""" + # setup + incoming_message = cast( + RegisterMessage, + self.build_incoming_message( + message_type=DefaultMessage, + performative=DefaultMessage.Performative.BYTES, + content=self.confirmed_aea + b"_" + self.developer_handle, + sender=self.aw1_aea, + ), + ) + + # operation + with patch.object(LedgerApis, "is_valid_address", return_value=False): + with patch.object(self.logger, "log") as mock_logger: + with patch.object( + self.strategy, "register_counterparty" + ) as mock_register: + self.default_handler.handle(incoming_message) + + # after + mock_register.called_once() + + default_dialogue = cast( + DefaultDialogue, self.default_dialogues.get_dialogue(incoming_message) + ) + + mock_logger.assert_any_call( + logging.WARNING, + f"received invalid address={self.confirmed_aea.decode('utf-8')} in dialogue={default_dialogue}.", + ) + + def test_handle_bytes_iv(self): + """Test the _handle_bytes method of the default handler where the sender is NOT aw1_aea.""" + # setup + incoming_message = cast( + RegisterMessage, + self.build_incoming_message( + message_type=DefaultMessage, + performative=DefaultMessage.Performative.BYTES, + content=self.confirmed_aea + b"_" + self.developer_handle, + sender="some_other_aea", + ), + ) + + # operation + with patch.object(self.logger, "log") as mock_logger: + with patch.object(self.strategy, "register_counterparty") as mock_register: + self.default_handler.handle(incoming_message) + + # after + mock_register.called_once() + + default_dialogue = cast( + DefaultDialogue, self.default_dialogues.get_dialogue(incoming_message) + ) + + mock_logger.assert_any_call( + logging.WARNING, + f"cannot handle default message of performative={incoming_message.performative} in dialogue={default_dialogue}. Invalid sender={incoming_message.sender}", + ) + + def test_handle_invalid(self): + """Test the _handle_invalid method of the default handler.""" + # setup + default_dialogue = cast( + DefaultDialogue, + self.prepare_skill_dialogue( + dialogues=self.default_dialogues, + messages=self.list_of_default_messages[:1], + ), + ) + incoming_message = cast( + RegisterMessage, + self.build_incoming_message_for_skill_dialogue( + dialogue=default_dialogue, + performative=DefaultMessage.Performative.ERROR, + error_code=DefaultMessage.ErrorCode.DECODING_ERROR, + error_msg="some_error_message", + error_data={"some_key": b"some_value"}, + ), + ) + + # operation + with patch.object(self.logger, "log") as mock_logger: + self.default_handler.handle(incoming_message) + + # after + mock_logger.assert_any_call( + logging.WARNING, + f"cannot handle default message of performative={incoming_message.performative} in dialogue={default_dialogue}.", + ) + + def test_teardown(self): + """Test the teardown method of the default handler.""" + assert self.default_handler.teardown() is None + self.assert_quantity_in_outbox(0) diff --git a/tests/test_packages/test_skills/test_confirmation_aw2/test_registration_db.py b/tests/test_packages/test_skills/test_confirmation_aw2/test_registration_db.py new file mode 100644 index 0000000000..0ecd7ce794 --- /dev/null +++ b/tests/test_packages/test_skills/test_confirmation_aw2/test_registration_db.py @@ -0,0 +1,475 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the tests of the RegistrationDB class of the confirmation aw2 skill.""" +import datetime +import json +import logging +from pathlib import Path +from unittest.mock import Mock, patch + +from aea.test_tools.test_skill import BaseSkillTestCase + +from packages.fetchai.skills.confirmation_aw2.registration_db import RegistrationDB + +from tests.conftest import ROOT_DIR + + +class TestStrategy(BaseSkillTestCase): + """Test RegistrationDB of confirmation aw2.""" + + path_to_skill = Path(ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw2") + + @classmethod + def setup(cls): + """Setup the test class.""" + cls.aw1_aea = "some_aw1_aea" + config_overrides = {"models": {"strategy": {"args": {"aw1_aea": cls.aw1_aea}}}} + super().setup(config_overrides=config_overrides) + cls.custom_path = None + cls.db = RegistrationDB( + custom_path=cls.custom_path, + name="strategy", + skill_context=cls._skill.skill_context, + ) + cls.address = "some_address" + cls.logger = cls._skill.skill_context.logger + + cls.timestamp = datetime.datetime.now() + cls.data = {"some_key_1": "some_value_1", "some_key_2": "some_value_2"} + + cls.developer_handle = "developer_handle" + + cls.first_trade = "2020-12-22 18:30:00.000000" + cls.second_trade = "second_trade" + cls.first_info = "first_info" + cls.second_info = "second_info" + + def test_set_trade_i(self): + """Test the set_trade method of the RegistrationDB class where record IS None.""" + # operation + with patch.object( + self.db, "get_trade_table", return_value=None + ) as mock_get_trade_table: + with patch.object(self.db, "_execute_single_sql") as mock_exe: + self.db.set_trade( + self.address, self.timestamp, self.data, + ) + + # after + mock_get_trade_table.assert_called_once() + mock_exe.assert_any_call( + "INSERT INTO trade_table(address, first_trade, second_trade, first_info, second_info) values(?, ?, ?, ?, ?)", + (self.address, self.timestamp, None, json.dumps(self.data), None), + ) + + def test_set_trade_ii(self): + """Test the set_trade method of the RegistrationDB class where record is NOT None.""" + # setup + record = ( + self.address, + self.first_trade, + None, + self.first_info, + self.second_info, + ) + + # operation + with patch.object( + self.db, "get_trade_table", return_value=record + ) as mock_get_trade_table: + with patch.object(self.db, "_execute_single_sql") as mock_exe: + self.db.set_trade( + self.address, self.timestamp, self.data, + ) + + # after + mock_get_trade_table.assert_called_once() + + mock_exe.assert_any_call( + "INSERT or REPLACE into trade_table(address, first_trade, second_trade, first_info, second_info) values(?, ?, ?, ?, ?)", + ( + self.address, + self.first_trade, + self.timestamp, + self.first_info, + json.dumps(self.data), + ), + ) + + def test_get_trade_table(self): + """Test the get_trade_table method of the RegistrationDB class.""" + # setup + trade_table = ("something_1", "something_2") + result = [trade_table, ("something_3",)] + + # operation + with patch.object( + self.db, "_execute_single_sql", return_value=result + ) as mock_exe: + actual_trade_table = self.db.get_trade_table(self.address) + + # after + mock_exe.assert_any_call( + "SELECT * FROM trade_table where address=?", (self.address,) + ) + assert actual_trade_table == trade_table + + def test_set_registered_i(self): + """Test the set_registered method of the RegistrationDB class where is_registeredis is False.""" + # operation + with patch.object( + self.db, "is_registered", return_value=False + ) as mock_is_registered: + with patch.object(self.db, "_execute_single_sql") as mock_exe: + self.db.set_registered( + self.address, self.developer_handle, + ) + + # after + mock_is_registered.assert_called_once() + mock_exe.assert_any_call( + "INSERT OR REPLACE INTO registered_table(address, ethereum_address, ethereum_signature, fetchai_signature, developer_handle, tweet) values(?, ?, ?, ?, ?, ?)", + (self.address, "", "", "", self.developer_handle, ""), + ) + + def test_set_registered_ii(self): + """Test the set_registered method of the RegistrationDB class where is_registeredis is True.""" + # operation + with patch.object( + self.db, "is_registered", return_value=True + ) as mock_is_registered: + with patch.object(self.db, "_execute_single_sql") as mock_exe: + self.db.set_registered( + self.address, self.developer_handle, + ) + + # after + mock_is_registered.assert_called_once() + mock_exe.assert_not_called() + + def test_is_registered_i(self): + """Test the is_registered method of the RegistrationDB class where result is NOT empty.""" + # setup + result = [["1"], ["2", "3"]] + + # operation + with patch.object( + self.db, "_execute_single_sql", return_value=result + ) as mock_exe: + is_registered = self.db.is_registered(self.address) + + # after + mock_exe.assert_any_call( + "SELECT * FROM registered_table WHERE address=?", (self.address,) + ) + assert is_registered + + def test_is_registered_ii(self): + """Test the is_registered method of the RegistrationDB class where result is empty.""" + # setup + result = [] + + # operation + with patch.object( + self.db, "_execute_single_sql", return_value=result + ) as mock_exe: + is_registered = self.db.is_registered(self.address) + + # after + mock_exe.assert_any_call( + "SELECT * FROM registered_table WHERE address=?", (self.address,) + ) + assert not is_registered + + def test_is_allowed_to_trade_i(self): + """Test the is_allowed_to_trade method of the RegistrationDB class where record IS None.""" + # setup + mininum_hours_between_txs = 1 + record = None + + # operation + with patch.object( + self.db, "get_trade_table", return_value=record + ) as mock_get_trade: + with patch.object(self.db, "_execute_single_sql") as mock_exe: + is_registered = self.db.is_allowed_to_trade( + self.address, mininum_hours_between_txs + ) + + # after + mock_get_trade.assert_called_once() + mock_exe.assert_not_called() + assert is_registered + + def test_is_allowed_to_trade_ii(self): + """Test the is_allowed_to_trade method of the RegistrationDB class where first_trade and second_trade are NOT present.""" + # setup + minimum_hours_between_txs = 1 + record = (self.address, None, None, self.first_info, self.second_info) + + # operation + with patch.object( + self.db, "get_trade_table", return_value=record + ) as mock_get_trade: + is_registered = self.db.is_allowed_to_trade( + self.address, minimum_hours_between_txs + ) + + # after + mock_get_trade.assert_called_once() + assert is_registered + + def test_is_allowed_to_trade_iii(self): + """Test the is_allowed_to_trade method of the RegistrationDB class where is_allowed_to_trade_ is True.""" + # setup + minimum_hours_between_txs = 1 + record = ( + self.address, + self.first_trade, + None, + self.first_info, + self.second_info, + ) + + mocked_now_greater_than_minimum = "2020-12-22 20:30:00.000000" + datetime_mock = Mock(wraps=datetime.datetime) + datetime_mock.now.return_value = datetime.datetime.strptime( + mocked_now_greater_than_minimum, "%Y-%m-%d %H:%M:%S.%f" + ) + + # operation + with patch("datetime.datetime", new=datetime_mock): + with patch.object( + self.db, "get_trade_table", return_value=record + ) as mock_get_trade: + is_registered = self.db.is_allowed_to_trade( + self.address, minimum_hours_between_txs + ) + + # after + mock_get_trade.assert_called_once() + assert is_registered + + def test_is_allowed_to_trade_iv(self): + """Test the is_allowed_to_trade method of the RegistrationDB class where is_allowed_to_trade_ is False.""" + # setup + minimum_hours_between_txs = 1 + record = ( + self.address, + self.first_trade, + None, + self.first_info, + self.second_info, + ) + + mocked_now_less_than_minimum = "2020-12-22 18:31:00.000000" + + # operation + datetime_mock = Mock(wraps=datetime.datetime) + datetime_mock.now.return_value = datetime.datetime.strptime( + mocked_now_less_than_minimum, "%Y-%m-%d %H:%M:%S.%f" + ) + with patch("datetime.datetime", new=datetime_mock): + with patch.object( + self.db, "get_trade_table", return_value=record + ) as mock_get_trade: + with patch.object(self.logger, "log") as mock_logger: + is_registered = self.db.is_allowed_to_trade( + self.address, minimum_hours_between_txs + ) + + # after + mock_get_trade.assert_called_once() + mock_logger.assert_any_call( + logging.INFO, + f"Invalid attempt for counterparty={self.address}, not enough time since last trade!", + ) + assert not is_registered + + def test_is_allowed_to_trade_v(self): + """Test the is_allowed_to_trade method of the RegistrationDB class where second_trade IS present.""" + # setup + minimum_hours_between_txs = 1 + record = ( + self.address, + self.first_trade, + self.second_trade, + self.first_info, + self.second_info, + ) + + mocked_now = "2020-12-22 18:31:00.000000" + # operation + datetime_mock = Mock(wraps=datetime.datetime) + datetime_mock.now.return_value = datetime.datetime.strptime( + mocked_now, "%Y-%m-%d %H:%M:%S.%f" + ) + with patch("datetime.datetime", new=datetime_mock): + with patch.object( + self.db, "get_trade_table", return_value=record + ) as mock_get_trade: + with patch.object(self.logger, "log") as mock_logger: + is_registered = self.db.is_allowed_to_trade( + self.address, minimum_hours_between_txs + ) + + # after + mock_get_trade.assert_called_once() + mock_logger.assert_any_call( + logging.INFO, + f"Invalid attempt for counterparty={self.address}, already completed 2 trades!", + ) + assert not is_registered + + def test_has_completed_two_trades_i(self): + """Test the has_completed_two_trades method of the RegistrationDB class where first and second trade are present.""" + # setup + record = ( + self.address, + self.first_trade, + self.second_trade, + self.first_info, + self.second_info, + ) + + # operation + with patch.object( + self.db, "get_trade_table", return_value=record + ) as mock_get_trade: + has_completed = self.db.has_completed_two_trades(self.address) + + # after + mock_get_trade.assert_called_once() + assert has_completed + + def test_has_completed_two_trades_ii(self): + """Test the has_completed_two_trades method of the RegistrationDB class where first and second trade are NOT present.""" + # setup + record = (self.address, None, None, self.first_info, self.second_info) + + # operation + with patch.object( + self.db, "get_trade_table", return_value=record + ) as mock_get_trade: + has_completed = self.db.has_completed_two_trades(self.address) + + # after + mock_get_trade.assert_called_once() + assert not has_completed + + def test_has_completed_two_trades_iii(self): + """Test the has_completed_two_trades method of the RegistrationDB class where first trade is NOT and second trade IS present.""" + # setup + record = ( + self.address, + None, + self.second_trade, + self.first_info, + self.second_info, + ) + + # operation + with patch.object( + self.db, "get_trade_table", return_value=record + ) as mock_get_trade: + has_completed = self.db.has_completed_two_trades(self.address) + + # after + mock_get_trade.assert_called_once() + assert not has_completed + + def test_has_completed_two_trades_iv(self): + """Test the has_completed_two_trades method of the RegistrationDB class where first trade IS and second trade is NOT present.""" + # setup + record = ( + self.address, + self.first_trade, + None, + self.first_info, + self.second_info, + ) + + # operation + with patch.object( + self.db, "get_trade_table", return_value=record + ) as mock_get_trade: + has_completed = self.db.has_completed_two_trades(self.address) + + # after + mock_get_trade.assert_called_once() + assert not has_completed + + def test_has_completed_two_trades_v(self): + """Test the has_completed_two_trades method of the RegistrationDB class where record IS None.""" + # setup + record = None + + # operation + with patch.object( + self.db, "get_trade_table", return_value=record + ) as mock_get_trade: + has_completed = self.db.has_completed_two_trades(self.address) + + # after + mock_get_trade.assert_called_once() + assert not has_completed + + def test_completed_two_trades(self): + """Test the completed_two_trades method of the RegistrationDB class.""" + # setup + row_1 = ( + "address_1", + "ethereum_address_1", + "something2_1", + "something3_1", + "developer_handle_1", + ) + row_2 = ( + "address_2", + "ethereum_address_2", + "something2_2", + "something3_2", + "developer_handle_2", + ) + row_3 = ( + "address_3", + "ethereum_address_3", + "something2_3", + "something3_3", + "developer_handle_3", + ) + result = [row_1, row_2, row_3] + + has_completed = [True, False, True] + + # operation + with patch.object( + self.db, "_execute_single_sql", return_value=result + ) as mock_exe: + with patch.object( + self.db, "has_completed_two_trades", side_effect=has_completed + ): + actual_completed = self.db.completed_two_trades() + + # after + mock_exe.assert_any_call("SELECT * FROM registered_table", ()) + assert actual_completed == [ + ("address_1", "ethereum_address_1", "developer_handle_1"), + ("address_3", "ethereum_address_3", "developer_handle_3"), + ] diff --git a/tests/test_packages/test_skills/test_confirmation_aw2/test_strategy.py b/tests/test_packages/test_skills/test_confirmation_aw2/test_strategy.py new file mode 100644 index 0000000000..13bd901e6e --- /dev/null +++ b/tests/test_packages/test_skills/test_confirmation_aw2/test_strategy.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the tests of the strategy class of the confirmation aw2 skill.""" + +import datetime +import logging +from pathlib import Path +from typing import cast +from unittest.mock import Mock, patch + +from aea.test_tools.test_skill import BaseSkillTestCase + +from packages.fetchai.skills.confirmation_aw2.registration_db import RegistrationDB +from packages.fetchai.skills.confirmation_aw2.strategy import Strategy + +from tests.conftest import ROOT_DIR + + +class TestStrategy(BaseSkillTestCase): + """Test Strategy of confirmation aw2.""" + + path_to_skill = Path(ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw2") + + @classmethod + def setup(cls): + """Setup the test class.""" + cls.aw1_aea = "some_aw1_aea" + config_overrides = {"models": {"strategy": {"args": {"aw1_aea": cls.aw1_aea}}}} + super().setup(config_overrides=config_overrides) + + cls.minimum_hours_between_txs = 4 + cls.minimum_minutes_since_last_attempt = 2 + cls.strategy = Strategy( + aw1_aea="some_aw1_aea", + mininum_hours_between_txs=cls.minimum_hours_between_txs, + minimum_minutes_since_last_attempt=cls.minimum_minutes_since_last_attempt, + name="strategy", + skill_context=cls._skill.skill_context, + ) + + cls.address = "some_address" + cls.info = { + "ethereum_address": "some_value", + "signature_of_ethereum_address": "some_signature_of_ethereum_address", + "signature_of_fetchai_address": "some_signature_of_fetchai_address", + "developer_handle": "some_developer_handle", + "tweet": "some_tweet", + } + cls.logger = cls._skill.skill_context.logger + cls.db = cast(RegistrationDB, cls._skill.skill_context.registration_db) + + cls.counterparty = "couterparty_1" + + def test__init__(self): + """Test the __init__ of Strategy class.""" + assert self.strategy.aw1_aea == self.aw1_aea + assert self.strategy.minimum_hours_between_txs == self.minimum_hours_between_txs + assert ( + self.strategy.minimum_minutes_since_last_attempt + == self.minimum_minutes_since_last_attempt + ) + + def test_get_acceptable_counterparties(self): + """Test the get_acceptable_counterparties method of the Strategy class.""" + # setup + couterparties = ("couterparty_1", "couterparty_2", "couterparty_3") + is_valid_counterparty = [True, False, True] + + # operation + with patch.object( + self.strategy, "is_valid_counterparty", side_effect=is_valid_counterparty + ): + actual_acceptable_counterparties = self.strategy.get_acceptable_counterparties( + couterparties + ) + + # after + assert actual_acceptable_counterparties == ("couterparty_1", "couterparty_3") + + def test_is_enough_time_since_last_attempt_i(self): + """Test the is_enough_time_since_last_attempt method of the Strategy class where now IS greater than last attempt + min minutes.""" + # setup + counterparty_last_attempt_time_str = "2020-12-22 20:30:00.000000" + counterparty_last_attempt_time = datetime.datetime.strptime( + counterparty_last_attempt_time_str, "%Y-%m-%d %H:%M:%S.%f" + ) + + mocked_now_greater_than_last_plus_minimum = "2020-12-22 20:33:00.000000" + datetime_mock = Mock(wraps=datetime.datetime) + datetime_mock.now.return_value = datetime.datetime.strptime( + mocked_now_greater_than_last_plus_minimum, "%Y-%m-%d %H:%M:%S.%f" + ) + self.strategy.last_attempt = {self.counterparty: counterparty_last_attempt_time} + + # operation + with patch("datetime.datetime", new=datetime_mock): + is_enough_time = self.strategy.is_enough_time_since_last_attempt( + self.counterparty + ) + + # after + assert is_enough_time is True + + def test_is_enough_time_since_last_attempt_ii(self): + """Test the is_enough_time_since_last_attempt method of the Strategy class where now is NOT greater than last attempt + min minutes.""" + # setup + counterparty_last_attempt_time_str = "2020-12-22 20:30:00.000000" + counterparty_last_attempt_time = datetime.datetime.strptime( + counterparty_last_attempt_time_str, "%Y-%m-%d %H:%M:%S.%f" + ) + + mocked_now_less_than_last_plus_minimum = "2020-12-22 20:31:00.000000" + datetime_mock = Mock(wraps=datetime.datetime) + datetime_mock.now.return_value = datetime.datetime.strptime( + mocked_now_less_than_last_plus_minimum, "%Y-%m-%d %H:%M:%S.%f" + ) + self.strategy.last_attempt = {self.counterparty: counterparty_last_attempt_time} + + # operation + with patch("datetime.datetime", new=datetime_mock): + is_enough_time = self.strategy.is_enough_time_since_last_attempt( + self.counterparty + ) + + # after + assert is_enough_time is False + + def test_is_enough_time_since_last_attempt_iii(self): + """Test the is_enough_time_since_last_attempt method of the Strategy class where now counterparty is NOT in last_attempt.""" + # setup + self.strategy.last_attempt = {} + + # operation + is_enough_time = self.strategy.is_enough_time_since_last_attempt( + self.counterparty + ) + + # after + assert is_enough_time is True + + def test_is_valid_counterparty_i(self): + """Test the is_valid_counterparty method of the Strategy class where is_registered is False.""" + # operation + with patch.object(self.db, "is_registered", return_value=False): + with patch.object(self.logger, "log") as mock_logger: + is_valid = self.strategy.is_valid_counterparty(self.counterparty) + + # after + mock_logger.assert_any_call( + logging.INFO, f"Invalid counterparty={self.counterparty}, not registered!", + ) + assert is_valid is False + + def test_is_valid_counterparty_ii(self): + """Test the is_valid_counterparty method of the Strategy class where is_enough_time_since_last_attempt is False.""" + # operation + with patch.object(self.db, "is_registered", return_value=True): + with patch.object( + self.strategy, "is_enough_time_since_last_attempt", return_value=False + ): + with patch.object(self.logger, "log") as mock_logger: + is_valid = self.strategy.is_valid_counterparty(self.counterparty) + + # after + mock_logger.assert_any_call( + logging.DEBUG, + f"Not enough time since last attempt for counterparty={self.counterparty}!", + ) + assert is_valid is False + + def test_is_valid_counterparty_iii(self): + """Test the is_valid_counterparty method of the Strategy class where is_allowed_to_trade is False.""" + # operation + with patch.object(self.db, "is_registered", return_value=True): + with patch.object( + self.strategy, "is_enough_time_since_last_attempt", return_value=True + ): + with patch.object(self.db, "is_allowed_to_trade", return_value=False): + is_valid = self.strategy.is_valid_counterparty(self.counterparty) + + # after + assert is_valid is False + + def test_is_valid_counterparty_iv(self): + """Test the is_valid_counterparty method of the Strategy class where it succeeds.""" + # operation + with patch.object(self.db, "is_registered", return_value=True): + with patch.object( + self.strategy, "is_enough_time_since_last_attempt", return_value=True + ): + with patch.object(self.db, "is_allowed_to_trade", return_value=True): + is_valid = self.strategy.is_valid_counterparty(self.counterparty) + + # after + assert is_valid is True + + def test_successful_trade_with_counterparty(self): + """Test the successful_trade_with_counterparty method of the Strategy class.""" + # setup + data = {"some_key_1": "some_value_1", "some_key_2": "some_value_2"} + + mocked_now_str = "2020-12-22 20:33:00.000000" + mock_now = datetime.datetime.strptime(mocked_now_str, "%Y-%m-%d %H:%M:%S.%f") + datetime_mock = Mock(wraps=datetime.datetime) + datetime_mock.now.return_value = mock_now + + # operation + with patch.object(self.db, "set_trade") as mock_set_trade: + with patch("datetime.datetime", new=datetime_mock): + with patch.object(self.logger, "log") as mock_logger: + self.strategy.successful_trade_with_counterparty( + self.counterparty, data + ) + + # after + mock_set_trade.assert_any_call(self.counterparty, mock_now, data) + + mock_logger.assert_any_call( + logging.INFO, + f"Successful trade with={self.counterparty}. Data acquired={data}!", + ) + + def test_register_counterparty(self): + """Test the register_counterparty method of the Strategy class.""" + # setup + developer_handle = "some_developer_handle" + + # operation + with patch.object(self.db, "set_registered") as mock_set_registered: + self.strategy.register_counterparty(self.counterparty, developer_handle) + + # after + mock_set_registered.assert_any_call(self.counterparty, developer_handle) From abc9713cde4cdf6bb30f416cc180b4d445b76c74 Mon Sep 17 00:00:00 2001 From: ali Date: Wed, 23 Dec 2020 14:26:46 +0000 Subject: [PATCH 043/204] hashes --- packages/fetchai/skills/confirmation_aw2/skill.yaml | 2 +- packages/hashes.csv | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/fetchai/skills/confirmation_aw2/skill.yaml b/packages/fetchai/skills/confirmation_aw2/skill.yaml index b8862de77a..a3a4709ec8 100644 --- a/packages/fetchai/skills/confirmation_aw2/skill.yaml +++ b/packages/fetchai/skills/confirmation_aw2/skill.yaml @@ -12,7 +12,7 @@ fingerprint: behaviours.py: QmWRRAXwJf7mEcV4L2DdG43zkyg55PBViiLnpy1Chj28xn dialogues.py: QmcUgBjxeytE5aAx3VvPyna5EcBuqck9KazG3HygCWjawv handlers.py: QmVeJuN78rMn3kWFbntfBUk6v88DYBW88ZQZ129KZo84wp - registration_db.py: QmWJkpgWRnyxYqfG5BqheU2UiuZVk59z76QX5YfzSZTASF + registration_db.py: QmbdtzWvWvWizrzLcvNWphVF7LVkDq3VSWC3MkaK4RU6H5 strategy.py: QmafVgQyrcetTPmURWWzqqFgkU2yY2DJp2G9FWwvxcZTmW fingerprint_ignore_patterns: [] connections: diff --git a/packages/hashes.csv b/packages/hashes.csv index df4be7dfa3..d06f2cb96c 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -69,7 +69,7 @@ fetchai/skills/carpark_client,QmWabiH48yVv4hNaufU3jtCqe3GSEsmhMe5DEPeciTYTP3 fetchai/skills/carpark_detection,QmYKRWjC3VEU8QatkptXoGyyL3P9uR8reoDdmgLW2r318x fetchai/skills/coin_price,QmXRHQim1XGbij8QnJ4CPBksdGxmBmpbmPmHdrPAw3z3pE fetchai/skills/confirmation_aw1,QmQbqko8t8utxmfywvDqRhpE1gACVdLxrZNnv98U1aAYyR -fetchai/skills/confirmation_aw2,QmbzgZnGsRMP5JLvQ52iTkwztoG3a6Ya1iUSQ7K5MZs4Dn +fetchai/skills/confirmation_aw2,QmSEK1j5AhjKDKkCWgn7xrc2ySWE3HCp4NQ5zcyuEDt2dX fetchai/skills/confirmation_aw3,QmPkTAzgHivkAs3fViiW3z8q7zkqAN8DS3DcKne992sS2u fetchai/skills/echo,QmTir9ngs1i9dLweJs5gB8Q5NLqDufuJ5A3A2bznibKPMk fetchai/skills/erc1155_client,QmTTuStPpw2zygu9dd9cNkJnzjSVihiNHu1qACSYBT5fvL From ff3f9b00799482fc2d13d41db3e37af628fb9a5d Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Wed, 23 Dec 2020 18:14:21 +0300 Subject: [PATCH 044/204] fixes --- aea/cli/config.py | 8 ++++---- aea/configurations/base.py | 4 ++-- aea/configurations/validation.py | 31 ++++++++++++++++++++----------- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/aea/cli/config.py b/aea/cli/config.py index fb9a46f4dc..88fc4fd75f 100644 --- a/aea/cli/config.py +++ b/aea/cli/config.py @@ -51,8 +51,8 @@ from aea.helpers.storage.backends.base import JSON_TYPES -class VariableDoesNotExists(ValueError): - """Variable does not exiss in a config exception.""" +class VariableDoesNotExist(ValueError): + """Variable does not exist in a config exception.""" JsonPath = List[str] @@ -105,7 +105,7 @@ def set_command( agent_config_manager = AgentConfigManager(ctx.agent_config, ctx.cwd) current_value = None - with contextlib.suppress(VariableDoesNotExists): + with contextlib.suppress(VariableDoesNotExist): current_value = agent_config_manager.get_variable(json_path) # type was not specified, tried to auto determine @@ -244,7 +244,7 @@ def get_variable(self, path: VariablePath) -> JSON_TYPES: if value is not NotExists: return cast(JSON_TYPES, value) - raise VariableDoesNotExists( + raise VariableDoesNotExist( f"Attribute `{'.'.join(json_path)}` for {'{}({}) config'.format(component_id.component_type,component_id.public_id) if component_id else 'AgentConfig'} does not exist" ) diff --git a/aea/configurations/base.py b/aea/configurations/base.py index a902c6da5e..b127c2dec8 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -426,7 +426,7 @@ def _apply_params_to_instance( ) if instance is None: - instance = cls(**params) # type: ignore + instance = cls(**params) else: instance.__init__(**params) # type: ignore @@ -861,7 +861,7 @@ def _apply_params_to_instance( ) -> "SkillComponentConfiguration": """Constructs or update instance with params provided.""" if instance is None: - instance = cls(**params) # type: ignore + instance = cls(**params) else: instance.__init__(**params) # type: ignore return instance diff --git a/aea/configurations/validation.py b/aea/configurations/validation.py index 298867408e..8cbfa0143c 100644 --- a/aea/configurations/validation.py +++ b/aea/configurations/validation.py @@ -181,24 +181,33 @@ def validate(self, json_data: Dict) -> None: json_data_copy = deepcopy(json_data) # validate component_configurations - component_configurations = json_data_copy.pop( - "component_configurations", {} + self.validate_agent_components_configuration( + json_data_copy.pop("component_configurations", []) ) - for idx, component_configuration_json in enumerate( - component_configurations - ): - component_id = self.split_component_id_and_config( - idx, component_configuration_json - ) - self.validate_component_configuration( - component_id, component_configuration_json - ) # validate agent config self._validator.validate(instance=json_data_copy) else: self._validator.validate(instance=json_data) + def validate_agent_components_configuration( + self, component_configurations: Dict + ) -> None: + """ + Validate agent component configurations overrides. + + :param component_configurations: + + :return: None + """ + for idx, component_configuration_json in enumerate(component_configurations): + component_id = self.split_component_id_and_config( + idx, component_configuration_json + ) + self.validate_component_configuration( + component_id, component_configuration_json + ) + @property def required_fields(self) -> List[str]: """ From 822451ab0758abbce10dada184d2913fc84ff7ce Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Wed, 23 Dec 2020 19:26:20 +0300 Subject: [PATCH 045/204] fixes and coverage --- aea/cli/config.py | 6 ++--- aea/configurations/base.py | 18 ++++++++------ aea/configurations/validation.py | 4 +-- tests/test_cli/test_config.py | 2 +- tests/test_cli/test_fetch.py | 6 ++--- tests/test_cli/test_issue_certificates.py | 3 +-- tests/test_configurations/test_base.py | 30 +++++++++++++++++++++++ tests/test_helpers/test_base.py | 28 +++++++++++++++++++++ 8 files changed, 78 insertions(+), 19 deletions(-) diff --git a/aea/cli/config.py b/aea/cli/config.py index 88fc4fd75f..952092d740 100644 --- a/aea/cli/config.py +++ b/aea/cli/config.py @@ -126,7 +126,7 @@ def set_command( agent_config_manager.set_variable(json_path, converted_value) agent_config_manager.dump_config() - except ExtraPropertiesError as e: + except ExtraPropertiesError as e: # pragma: nocover raise ClickException(f"Attribute `{e.args[0][0]}` is not allowed to change!") except (ValueError, AEAException) as e: raise ClickException(*e.args) @@ -178,7 +178,7 @@ def agent_config_file_path(self) -> Path: @classmethod def load(cls, aea_project_path: str) -> "AgentConfigManager": """Create AgentConfigManager instance from agent project path.""" - raise NotImplementedError + raise NotImplementedError # pragma: nocover def set_variable(self, path: VariablePath, value: JSON_TYPES) -> None: """ @@ -284,7 +284,7 @@ def _parse_path(self, path: VariablePath) -> Tuple[Optional[ComponentId], JsonPa json_path, *_, component_id = handle_dotted_path( path, self.agent_config.author ) - else: + else: # pragma: nocover if isinstance(path[0], ComponentId): json_path = path[1:] component_id = path[0] diff --git a/aea/configurations/base.py b/aea/configurations/base.py index 43ab66db6d..4c6a8e692a 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -342,7 +342,7 @@ def _create_or_update_from_json( cls, obj: Dict, instance=None ) -> "PackageConfiguration": """Create new config object or updates existing one from json data.""" - raise NotImplementedError + raise NotImplementedError # pragma: nocover @staticmethod def _compare_data_to_pattern( @@ -359,7 +359,7 @@ def _compare_data_to_pattern( def check_excludes(path): for exclude in excludes: - if len(exclude) > len(path): + if len(exclude) > len(path): # pragma: nocover continue if path[: len(exclude)] == exclude: @@ -369,22 +369,23 @@ def check_excludes(path): for path, new_value in data_path_value.items(): if check_excludes(path): continue + if path not in pattern_path_value: errors.append( f"Attribute `{'.'.join(path)}` is not allowed to be updated!" ) continue - current_value = data_path_value[path] + pattern_value = pattern_path_value[path] - if current_value is None: + if pattern_value is None: # not possible to determine data type for optional value not set # it will be checked with jsonschema later - continue + continue # pragma: nocover - if not issubclass(type(new_value), type(current_value)): + if not issubclass(type(new_value), type(pattern_value)): errors.append( - f"For {'.'.join(path)} `{type(current_value)}` data type is expected, but `{type(new_value)}` was provided!" + f"For attribute `{'.'.join(path)}` `{type(pattern_value).__name__}` data type is expected, but `{type(new_value).__name__}` was provided!" ) return errors @@ -879,7 +880,7 @@ def _apply_params_to_instance( """Constructs or update instance with params provided.""" if instance is None: instance = cls(**params) - else: + else: # pragma: nocover instance.__init__(**params) # type: ignore return instance @@ -1115,6 +1116,7 @@ class AgentConfig(PackageConfiguration): CHECK_EXCLUDES = [ ("private_key_paths",), ("default_routing",), + ("logging_config",), ] def __init__( diff --git a/aea/configurations/validation.py b/aea/configurations/validation.py index 8cbfa0143c..767325ed59 100644 --- a/aea/configurations/validation.py +++ b/aea/configurations/validation.py @@ -75,13 +75,13 @@ def _get_path_to_custom_config_schema_from_type(component_type: ComponentType) - class ExtraPropertiesError(ValueError): """Extra properties exception.""" - def __str__(self) -> str: + def __str__(self) -> str: # pragma: nocover """Get string representation of the object.""" return ( f"ExtraPropertiesError: properties not expected: {', '.join(self.args[0])}" ) - def __repr__(self) -> str: + def __repr__(self) -> str: # pragma: nocover """Get string representation of the object.""" return str(self) diff --git a/tests/test_cli/test_config.py b/tests/test_cli/test_config.py index 14ad03da04..e738ec952a 100644 --- a/tests/test_cli/test_config.py +++ b/tests/test_cli/test_config.py @@ -347,7 +347,7 @@ def test_set_type_none(self): *CLI_LOG_OPTION, "config", "set", - "agent.logging_config.disable_existing_loggers", + "agent.logging_config.some_value", "", "--type=none", ], diff --git a/tests/test_cli/test_fetch.py b/tests/test_cli/test_fetch.py index 48f8541527..72a7cf0ab8 100644 --- a/tests/test_cli/test_fetch.py +++ b/tests/test_cli/test_fetch.py @@ -271,19 +271,19 @@ def test_fetch_negative(self, *_mocks) -> None: class TestFetchAgentNonMixedErrorLocal(BaseTestFetchAgentError): """Test 'aea fetch' in local mode when it fails.""" - EXPECTED_ERROR_MESSAGE = f".*Exception: {BaseTestFetchAgentError.ERROR_MESSAGE}" + EXPECTED_ERROR_MESSAGE = f".*{BaseTestFetchAgentError.ERROR_MESSAGE}" MODE = "--local" class TestFetchAgentMixedModeError(BaseTestFetchAgentError): """Test 'aea fetch' in mixed mode when it fails.""" - EXPECTED_ERROR_MESSAGE = f".*Exception: {BaseTestFetchAgentError.ERROR_MESSAGE}" + EXPECTED_ERROR_MESSAGE = f".*{BaseTestFetchAgentError.ERROR_MESSAGE}" MODE = "" class TestFetchAgentRemoteModeError(BaseTestFetchAgentError): """Test 'aea fetch' in remote mode when it fails.""" - EXPECTED_ERROR_MESSAGE = rf".*Exception: {BaseTestFetchAgentError.ERROR_MESSAGE}" + EXPECTED_ERROR_MESSAGE = rf".*{BaseTestFetchAgentError.ERROR_MESSAGE}" MODE = "--remote" diff --git a/tests/test_cli/test_issue_certificates.py b/tests/test_cli/test_issue_certificates.py index 28a1e8b498..cb8a0418fc 100644 --- a/tests/test_cli/test_issue_certificates.py +++ b/tests/test_cli/test_issue_certificates.py @@ -184,7 +184,6 @@ def setup_class(cls): def test_run(self): """Run the test.""" with pytest.raises( - Exception, - match="Cannot find private key with id 'bad_ledger_id'", + Exception, match="Cannot find private key with id 'bad_ledger_id'", ): self.run_cli_command("issue-certificates", cwd=self._get_cwd()) diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index a7ee851b45..02d1c59cc7 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -35,6 +35,7 @@ ConnectionConfig, ContractConfig, Dependency, + PackageConfiguration, PackageId, PackageType, PackageVersion, @@ -232,8 +233,12 @@ def test_update_method(self): "handlers": {"dummy": {"args": dict(handler_arg_1=42)}}, "models": {"dummy": {"args": dict(model_arg_1=42)}}, } + directory = "test_directory" + skill_config.directory = directory skill_config.update(new_configurations) + assert skill_config.directory == directory + assert ( expected_dummy_behaviour_args == skill_config.behaviours.read("dummy").args ) @@ -330,6 +335,10 @@ def setup(self): "models": {"dummy": {"args": dict(model_arg_1=42)}}, } + def test_all_components_id(self): + """Test all components id listing.""" + assert self.dummy_skill_component_id in self.aea_config.all_components_id + def test_component_configurations_setter(self): """Test component configuration setter.""" assert self.aea_config.component_configurations == {} @@ -973,3 +982,24 @@ def test_check_public_id_consistency_negative(): with pytest.raises(ValueError, match=f"Directory {random_dir_name} is not valid."): component_configuration = ProtocolConfig("name", "author") component_configuration.check_public_id_consistency(Path(random_dir_name)) + + +def test_compare_data_pattern(): + """Test PackageConfiguration._compare_data_to_pattern.""" + errors = PackageConfiguration._compare_data_to_pattern({"a": 12}, {"a": 13}) + assert not errors + + errors = PackageConfiguration._compare_data_to_pattern({"a": 12}, {"a": "string"}) + assert errors + assert ( + errors[0] + == "For attribute `a` `str` data type is expected, but `int` was provided!" + ) + + errors = PackageConfiguration._compare_data_to_pattern({"a": 12}, {"b": 12}) + assert errors + assert errors[0] == "Attribute `a` is not allowed to be updated!" + + errors = PackageConfiguration._compare_data_to_pattern({"a": {}}, {"a": {"b": 12}}) + assert errors + assert errors[0] == "Attribute `a` is not allowed to be updated!" diff --git a/tests/test_helpers/test_base.py b/tests/test_helpers/test_base.py index 4f926fbff9..5b3d719771 100644 --- a/tests/test_helpers/test_base.py +++ b/tests/test_helpers/test_base.py @@ -39,6 +39,7 @@ MaxRetriesError, RegexConstrainedString, compute_specifier_from_version, + dict_to_path_value, ensure_dir, exception_log_and_reraise, find_topological_order, @@ -263,6 +264,23 @@ def test_recursive_update_negative_different_type(): recursive_update(to_update, new_values) +def test_recursive_update_new_fields(): + """Test the 'recursive update' utility, with new fields.""" + # here we try to update an integer with a boolean - it raises error. + to_update = dict(subdict=dict(to_update=1)) + new_values = dict(subdict=dict(to_update2=False)) + + with pytest.raises( + ValueError, + match="Key 'to_update2' is not contained in the dictionary to update.", + ): + recursive_update(to_update, new_values) + assert "to_update2" not in to_update["subdict"] + + recursive_update(to_update, new_values, allow_new_values=True) + assert "to_update2" in to_update["subdict"] + + def test_recursive_update_negative_unknown_field(): """Test the 'recursive update' utility, when there are unknown fields.""" # here we try to update an integer with a boolean - it raises error. @@ -552,3 +570,13 @@ def test_compute_specifier_from_version(): version = "1.1.5" expected_range = ">=1.1.0, <1.2.0" assert expected_range == compute_specifier_from_version(Version(version)) + + +def test_dict_to_path_value(): + """Test dict_to_path_value.""" + path_values = { + tuple(path): value + for path, value in dict_to_path_value({"a": 12, "b": {"c": 1}}) + } + assert path_values.get(("a",)) == 12 + assert path_values.get(("b", "c")) == 1 From 27446d08ac065cc3527bafa4f68fc1c81c115887 Mon Sep 17 00:00:00 2001 From: ali Date: Wed, 23 Dec 2020 18:02:08 +0000 Subject: [PATCH 046/204] added tests for confirmation aw3 skill --- .../skills/confirmation_aw3/handlers.py | 2 +- .../test_confirmation_aw3/__init__.py | 20 + .../intermediate_class.py | 46 ++ .../test_confirmation_aw3/test_behaviours.py | 87 ++++ .../test_confirmation_aw3/test_dialogues.py | 61 +++ .../test_confirmation_aw3/test_handlers.py | 396 ++++++++++++++++++ .../test_registration_db.py | 264 ++++++++++++ .../test_confirmation_aw3/test_strategy.py | 222 ++++++++++ 8 files changed, 1097 insertions(+), 1 deletion(-) create mode 100644 tests/test_packages/test_skills/test_confirmation_aw3/__init__.py create mode 100644 tests/test_packages/test_skills/test_confirmation_aw3/intermediate_class.py create mode 100644 tests/test_packages/test_skills/test_confirmation_aw3/test_behaviours.py create mode 100644 tests/test_packages/test_skills/test_confirmation_aw3/test_dialogues.py create mode 100644 tests/test_packages/test_skills/test_confirmation_aw3/test_handlers.py create mode 100644 tests/test_packages/test_skills/test_confirmation_aw3/test_registration_db.py create mode 100644 tests/test_packages/test_skills/test_confirmation_aw3/test_strategy.py diff --git a/packages/fetchai/skills/confirmation_aw3/handlers.py b/packages/fetchai/skills/confirmation_aw3/handlers.py index a36aac6336..0960effa22 100644 --- a/packages/fetchai/skills/confirmation_aw3/handlers.py +++ b/packages/fetchai/skills/confirmation_aw3/handlers.py @@ -73,7 +73,7 @@ def handle(self, message: Message) -> None: default_dialogues = cast(DefaultDialogues, self.context.default_dialogues) default_dialogue = cast(DefaultDialogue, default_dialogues.update(default_msg)) if default_dialogue is None: - self._handle_unidentified_dialogue(default_dialogue) + self._handle_unidentified_dialogue(default_msg) return # handle message diff --git a/tests/test_packages/test_skills/test_confirmation_aw3/__init__.py b/tests/test_packages/test_skills/test_confirmation_aw3/__init__.py new file mode 100644 index 0000000000..71f1d7313c --- /dev/null +++ b/tests/test_packages/test_skills/test_confirmation_aw3/__init__.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""The tests module contains the tests of the packages/skills/confirmation_aw3 dir.""" diff --git a/tests/test_packages/test_skills/test_confirmation_aw3/intermediate_class.py b/tests/test_packages/test_skills/test_confirmation_aw3/intermediate_class.py new file mode 100644 index 0000000000..1c5119fe87 --- /dev/null +++ b/tests/test_packages/test_skills/test_confirmation_aw3/intermediate_class.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the tests of the behaviour classes of the simple_data_request skill.""" +from pathlib import Path + +from aea.test_tools.test_skill import BaseSkillTestCase + +from tests.conftest import ROOT_DIR + + +class ConfirmationAW3TestCase(BaseSkillTestCase): + """Sets the confirmation_aw3 class up for testing (overrides the necessary config values so tests can be done on confirmation_aw3 skill).""" + + path_to_skill = Path( + ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw3" + ) + + @classmethod + def setup(cls): + """Setup the test class.""" + cls.aw1_aea = "some_aw1_aea" + cls.leaderboard_url = "some_leaderboard_url" + cls.leaderboard_token = "some_leaderboard_token" + config_overrides = {"models": {"strategy": {"args": { + "aw1_aea": cls.aw1_aea, + "leaderboard_url": cls.leaderboard_url, + "leaderboard_token": cls.leaderboard_token, + }}}} + + super().setup(config_overrides=config_overrides) diff --git a/tests/test_packages/test_skills/test_confirmation_aw3/test_behaviours.py b/tests/test_packages/test_skills/test_confirmation_aw3/test_behaviours.py new file mode 100644 index 0000000000..62d0589ed6 --- /dev/null +++ b/tests/test_packages/test_skills/test_confirmation_aw3/test_behaviours.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the tests of the behaviour classes of the confirmation aw3 skill.""" + +from pathlib import Path +from typing import cast +from unittest.mock import patch + +from aea.helpers.search.models import Query, Constraint, ConstraintType, Attribute, DataModel + +from packages.fetchai.protocols.oef_search.message import OefSearchMessage +from packages.fetchai.skills.confirmation_aw3.behaviours import SearchBehaviour +from packages.fetchai.skills.confirmation_aw3.strategy import Strategy + +from tests.conftest import ROOT_DIR +from tests.test_packages.test_skills.test_confirmation_aw3.intermediate_class import ( + ConfirmationAW3TestCase, +) + + +class TestSearchBehaviour(ConfirmationAW3TestCase): + """Test search behaviour of confirmation aw3.""" + + path_to_skill = Path(ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw3") + + @classmethod + def setup(cls): + """Setup the test class.""" + super().setup() + + cls.search_behaviour = cast( + SearchBehaviour, cls._skill.skill_context.behaviours.search + ) + cls.strategy = cast(Strategy, cls._skill.skill_context.strategy) + + def test_act(self): + """Test the act method of the transaction behaviour.""" + # setup + self.strategy.is_searching = True + + mock_query = Query( + [Constraint("some_attribute", ConstraintType("==", "some_service"))], + DataModel( + "some_name", + [ + Attribute( + "some_attribute", str, False, "Some attribute descriptions." + ) + ], + ), + ) + + # operation + with patch.object(self.strategy, "update_search_query_params") as mock_update: + with patch.object(self.strategy, "get_location_and_service_query", return_value=mock_query): + self.search_behaviour.act() + + # after + self.assert_quantity_in_outbox(1) + + mock_update.assert_called_once() + + has_attributes, error_str = self.message_has_attributes( + actual_message=self.get_message_from_outbox(), + message_type=OefSearchMessage, + performative=OefSearchMessage.Performative.SEARCH_SERVICES, + to=self.skill.skill_context.search_service_address, + sender=self.skill.skill_context.agent_address, + query=mock_query, + ) + assert has_attributes, error_str diff --git a/tests/test_packages/test_skills/test_confirmation_aw3/test_dialogues.py b/tests/test_packages/test_skills/test_confirmation_aw3/test_dialogues.py new file mode 100644 index 0000000000..80a6f138ec --- /dev/null +++ b/tests/test_packages/test_skills/test_confirmation_aw3/test_dialogues.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the tests of the dialogue classes of the generic buyer skill.""" + +from pathlib import Path +from typing import cast + +from aea.test_tools.test_skill import COUNTERPARTY_ADDRESS + +from packages.fetchai.protocols.http.message import HttpMessage +from packages.fetchai.skills.confirmation_aw3.dialogues import HttpDialogue, HttpDialogues + +from tests.conftest import ROOT_DIR +from tests.test_packages.test_skills.test_confirmation_aw3.intermediate_class import ( + ConfirmationAW3TestCase, +) + + +class TestDialogues(ConfirmationAW3TestCase): + """Test dialogue classes of confirmation aw3.""" + + path_to_skill = Path(ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw3") + + @classmethod + def setup(cls): + """Setup the test class.""" + super().setup() + + cls.http_dialogues = cast( + HttpDialogues, cls._skill.skill_context.http_dialogues + ) + + def test_http_dialogues(self): + """Test the HttpDialogues class.""" + _, dialogue = self.http_dialogues.create( + counterparty=COUNTERPARTY_ADDRESS, + performative=HttpMessage.Performative.REQUEST, + method="some_method", + url="some_url", + version="some_version", + headers="some_headers", + body=b"some_body", + ) + assert dialogue.role == HttpDialogue.Role.CLIENT + assert dialogue.self_address == self.skill.skill_context.agent_address diff --git a/tests/test_packages/test_skills/test_confirmation_aw3/test_handlers.py b/tests/test_packages/test_skills/test_confirmation_aw3/test_handlers.py new file mode 100644 index 0000000000..2ccdc0a6b3 --- /dev/null +++ b/tests/test_packages/test_skills/test_confirmation_aw3/test_handlers.py @@ -0,0 +1,396 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the tests of the handler classes of the confirmation aw3 skill.""" + +import logging +from pathlib import Path +from typing import cast +from unittest.mock import patch + +from aea.crypto.ledger_apis import LedgerApis +from aea.protocols.dialogue.base import DialogueMessage + +from packages.fetchai.protocols.default.dialogues import DefaultDialogue +from packages.fetchai.protocols.default.message import DefaultMessage +from packages.fetchai.protocols.http.message import HttpMessage +from packages.fetchai.skills.confirmation_aw3.dialogues import HttpDialogue, HttpDialogues, DefaultDialogues +from packages.fetchai.skills.confirmation_aw3.handlers import DefaultHandler, HttpHandler +from packages.fetchai.skills.confirmation_aw3.strategy import Strategy + +from tests.conftest import ROOT_DIR +from tests.test_packages.test_skills.test_confirmation_aw3.intermediate_class import ( + ConfirmationAW3TestCase, +) + + +class TestDefaultHandler(ConfirmationAW3TestCase): + """Test default handler of confirmation aw3.""" + + path_to_skill = Path(ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw3") + + @classmethod + def setup(cls): + """Setup the test class.""" + super().setup() + cls.default_handler = cast( + DefaultHandler, cls._skill.skill_context.handlers.default_handler + ) + cls.logger = cls._skill.skill_context.logger + cls.strategy = cast(Strategy, cls._skill.skill_context.strategy) + + cls.default_dialogues = cast( + DefaultDialogues, cls._skill.skill_context.default_dialogues + ) + + cls.list_of_default_messages = ( + DialogueMessage( + DefaultMessage.Performative.BYTES, {"content": b"some_content"} + ), + ) + + cls.confirmed_aea = b"ConfirmedAEA" + cls.developer_handle = b"DeveloperHandle" + + def test_setup(self): + """Test the setup method of the default handler.""" + assert self.default_handler.setup() is None + self.assert_quantity_in_outbox(0) + + def test_handle_unidentified_dialogue(self): + """Test the _handle_unidentified_dialogue method of the register handler.""" + # setup + incorrect_dialogue_reference = ("", "") + incoming_message = self.build_incoming_message( + message_type=DefaultMessage, + dialogue_reference=incorrect_dialogue_reference, + performative=DefaultMessage.Performative.BYTES, + content=b"some_content", + ) + + # operation + with patch.object(self.logger, "log") as mock_logger: + self.default_handler.handle(incoming_message) + + # after + mock_logger.assert_any_call( + logging.INFO, + f"received invalid default message={incoming_message}, unidentified dialogue.", + ) + + def test_handle_bytes_i(self): + """Test the _handle_bytes method of the default handler where the sender IS aw1_aea.""" + # setup + incoming_message = cast( + DefaultMessage, + self.build_incoming_message( + message_type=DefaultMessage, + performative=DefaultMessage.Performative.BYTES, + content=self.confirmed_aea + b"_" + self.developer_handle, + sender=self.aw1_aea, + ), + ) + + # operation + with patch.object(LedgerApis, "is_valid_address", return_value=True): + with patch.object(self.logger, "log") as mock_logger: + with patch.object( + self.strategy, "register_counterparty" + ) as mock_register: + self.default_handler.handle(incoming_message) + + # after + mock_register.called_once() + + mock_logger.assert_any_call( + logging.INFO, + f"adding confirmed_aea={self.confirmed_aea.decode('utf-8')} with developer_handle={self.developer_handle.decode('utf-8')} to db.", + ) + + def test_handle_bytes_ii(self): + """Test the _handle_bytes method of the default handler where the content is undecodable.""" + # setup + incorrect_content = "some_incorrect_content" + + incoming_message = cast( + DefaultMessage, + self.build_incoming_message( + message_type=DefaultMessage, + performative=DefaultMessage.Performative.BYTES, + content=incorrect_content, + sender=self.aw1_aea, + ), + ) + + # operation + with patch.object(LedgerApis, "is_valid_address", return_value=True): + with patch.object(self.logger, "log") as mock_logger: + with patch.object( + self.strategy, "register_counterparty" + ) as mock_register: + self.default_handler.handle(incoming_message) + + # after + mock_register.called_once() + + mock_logger.assert_any_call( + logging.WARNING, "received invalid developer_handle=." + ) + + def test_handle_bytes_iii(self): + """Test the _handle_bytes method of the default handler where is_valid_address is False.""" + # setup + incoming_message = cast( + DefaultMessage, + self.build_incoming_message( + message_type=DefaultMessage, + performative=DefaultMessage.Performative.BYTES, + content=self.confirmed_aea + b"_" + self.developer_handle, + sender=self.aw1_aea, + ), + ) + + # operation + with patch.object(LedgerApis, "is_valid_address", return_value=False): + with patch.object(self.logger, "log") as mock_logger: + with patch.object( + self.strategy, "register_counterparty" + ) as mock_register: + self.default_handler.handle(incoming_message) + + # after + mock_register.called_once() + + default_dialogue = cast( + DefaultDialogue, self.default_dialogues.get_dialogue(incoming_message) + ) + + mock_logger.assert_any_call( + logging.WARNING, + f"received invalid address={self.confirmed_aea.decode('utf-8')} in dialogue={default_dialogue}.", + ) + + def test_handle_bytes_iv(self): + """Test the _handle_bytes method of the default handler where the sender is NOT aw1_aea.""" + # setup + incoming_message = cast( + DefaultMessage, + self.build_incoming_message( + message_type=DefaultMessage, + performative=DefaultMessage.Performative.BYTES, + content=self.confirmed_aea + b"_" + self.developer_handle, + sender="some_other_aea", + ), + ) + + # operation + with patch.object(self.logger, "log") as mock_logger: + with patch.object(self.strategy, "register_counterparty") as mock_register: + self.default_handler.handle(incoming_message) + + # after + mock_register.called_once() + + default_dialogue = cast( + DefaultDialogue, self.default_dialogues.get_dialogue(incoming_message) + ) + + mock_logger.assert_any_call( + logging.WARNING, + f"cannot handle default message of performative={incoming_message.performative} in dialogue={default_dialogue}. Invalid sender={incoming_message.sender}", + ) + + def test_handle_invalid(self): + """Test the _handle_invalid method of the default handler.""" + # setup + default_dialogue = cast( + DefaultDialogue, + self.prepare_skill_dialogue( + dialogues=self.default_dialogues, + messages=self.list_of_default_messages[:1], + ), + ) + incoming_message = cast( + DefaultMessage, + self.build_incoming_message_for_skill_dialogue( + dialogue=default_dialogue, + performative=DefaultMessage.Performative.ERROR, + error_code=DefaultMessage.ErrorCode.DECODING_ERROR, + error_msg="some_error_message", + error_data={"some_key": b"some_value"}, + ), + ) + + # operation + with patch.object(self.logger, "log") as mock_logger: + self.default_handler.handle(incoming_message) + + # after + mock_logger.assert_any_call( + logging.WARNING, + f"cannot handle default message of performative={incoming_message.performative} in dialogue={default_dialogue}.", + ) + + def test_teardown(self): + """Test the teardown method of the default handler.""" + assert self.default_handler.teardown() is None + self.assert_quantity_in_outbox(0) + + +class TestHttpHandler(ConfirmationAW3TestCase): + """Test http handler of confirmation aw3.""" + + path_to_skill = Path(ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw3") + + @classmethod + def setup(cls): + """Setup the test class.""" + super().setup() + cls.http_handler = cast( + HttpHandler, cls._skill.skill_context.handlers.http_handler + ) + cls.logger = cls._skill.skill_context.logger + cls.strategy = cast(Strategy, cls._skill.skill_context.strategy) + + cls.http_dialogues = cast( + HttpDialogues, cls._skill.skill_context.http_dialogues + ) + + cls.method= "some_method", + cls.url= "some_url", + cls.version= "some_version", + cls.headers= "some_headers", + cls.body= b'some_body', + + cls.list_of_http_messages = ( + DialogueMessage( + HttpMessage.Performative.REQUEST, + { + "method": cls.method, + "url": cls.url, + "version": cls.version, + "headers": cls.headers, + "body": cls.body, + + } + ), + ) + + cls.confirmed_aea = b"ConfirmedAEA" + cls.developer_handle = b"DeveloperHandle" + + def test_setup(self): + """Test the setup method of the http handler.""" + assert self.http_handler.setup() is None + self.assert_quantity_in_outbox(0) + + def test_handle_unidentified_dialogue(self): + """Test the _handle_unidentified_dialogue method of the register handler.""" + # setup + incorrect_dialogue_reference = ("", "") + incoming_message = self.build_incoming_message( + message_type=HttpMessage, + dialogue_reference=incorrect_dialogue_reference, + performative=HttpMessage.Performative.RESPONSE, + version=self.version, + status_code=200, + status_text="some_status_text", + headers=self.headers, + body=self.body, + ) + + # operation + with patch.object(self.logger, "log") as mock_logger: + self.http_handler.handle(incoming_message) + + # after + mock_logger.assert_any_call( + logging.INFO, + f"received invalid http message={incoming_message}, unidentified dialogue.", + ) + + def test__handle_response(self): + """Test the _handle_bytes method of the http handler where the sender IS aw1_aea.""" + # setup + status_code = 200 + status_text = "some_status_text" + + http_dialogue = cast( + HttpDialogue, + self.prepare_skill_dialogue( + dialogues=self.http_dialogues, + messages=self.list_of_http_messages[:1], + ), + ) + + incoming_message = cast( + HttpMessage, + self.build_incoming_message_for_skill_dialogue( + dialogue=http_dialogue, + performative=HttpMessage.Performative.RESPONSE, + version=self.version, + status_code=status_code, + status_text=status_text, + headers=self.headers, + body=self.body, + ), + ) + + # operation + with patch.object(self.logger, "log") as mock_logger: + self.http_handler.handle(incoming_message) + + # after + mock_logger.assert_any_call( + logging.INFO, + "received http response with status_code={}, status_text={} and body={!r} in dialogue={}".format( + status_code, status_text, self.body, http_dialogue + ), + ) + + def test_handle_invalid(self): + """Test the _handle_invalid method of the http handler.""" + # setup + incoming_message = cast( + HttpMessage, + self.build_incoming_message( + message_type=HttpMessage, + performative=HttpMessage.Performative.REQUEST, + method=self.method, + url=self.url, + version=self.version, + headers=self.headers, + body=self.body, + ), + ) + + # operation + with patch.object(self.logger, "log") as mock_logger: + self.http_handler.handle(incoming_message) + + # after + http_dialogue = cast(HttpDialogue, self.http_dialogues.get_dialogue(incoming_message)) + mock_logger.assert_any_call( + logging.WARNING, + f"cannot handle http message of performative={incoming_message.performative} in dialogue={http_dialogue}.", + ) + + def test_teardown(self): + """Test the teardown method of the http handler.""" + assert self.http_handler.teardown() is None + self.assert_quantity_in_outbox(0) diff --git a/tests/test_packages/test_skills/test_confirmation_aw3/test_registration_db.py b/tests/test_packages/test_skills/test_confirmation_aw3/test_registration_db.py new file mode 100644 index 0000000000..fdc7a42d0a --- /dev/null +++ b/tests/test_packages/test_skills/test_confirmation_aw3/test_registration_db.py @@ -0,0 +1,264 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the tests of the RegistrationDB class of the confirmation aw3 skill.""" +import datetime +import json +from pathlib import Path +from unittest.mock import patch + +import pytest + +from packages.fetchai.skills.confirmation_aw3.registration_db import RegistrationDB + +from tests.conftest import ROOT_DIR +from tests.test_packages.test_skills.test_confirmation_aw3.intermediate_class import ( + ConfirmationAW3TestCase, +) + + +class TestStrategy(ConfirmationAW3TestCase): + """Test RegistrationDB of confirmation aw3.""" + + path_to_skill = Path(ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw3") + + @classmethod + def setup(cls): + """Setup the test class.""" + super().setup() + cls.custom_path = None + cls.db = RegistrationDB( + custom_path=cls.custom_path, + name="strategy", + skill_context=cls._skill.skill_context, + ) + + cls.address = "some_address" + cls.timestamp = datetime.datetime.now() + cls.data = {"some_key_1": "some_value_1", "some_key_2": "some_value_2"} + cls.developer_handle = "developer_handle" + + def test_set_trade(self): + """Test the set_trade method of the RegistrationDB class.""" + # operation + with patch.object(self.db, "_execute_single_sql") as mock_exe: + self.db.set_trade( + self.address, self.timestamp, self.data, + ) + + # after + mock_exe.assert_any_call( + "INSERT INTO trades_table(address, created_at, data) values(?, ?, ?)", + (self.address, self.timestamp, json.dumps(self.data)), + ) + + def test_get_trade_count(self): + """Test the get_trade_count method of the RegistrationDB class.""" + # setup + result = [["1", "2"], ["3", "4"]] + + # operation + with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: + actual_result = self.db.get_trade_count( + self.address, + ) + + # after + mock_exe.assert_any_call( + "SELECT COUNT(*) FROM trades_table where address=?", + (self.address,), + ) + assert actual_result == 1 + + def test_get_developer_handle_i(self): + """Test the get_developer_handle method of the RegistrationDB class which succeeds.""" + # setup + result = [["developer_handle"], ["something_1", "something_2"]] + + # operation + with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: + actual_result = self.db.get_developer_handle( + self.address, + ) + + # after + mock_exe.assert_any_call( + "SELECT developer_handle FROM registered_table where address=?", + (self.address,), + ) + assert actual_result == "developer_handle" + + def test_get_developer_handle_ii(self): + """Test the get_developer_handle method of the RegistrationDB class where the length of result[0] != 1.""" + # setup + result = [["developer_handle", "something_1"], ["something_2", "something_3"]] + + # operation + with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: + with pytest.raises(ValueError, match=f"More than one developer_handle found for address={self.address}."): + self.db.get_developer_handle(self.address) + + # after + mock_exe.assert_any_call( + "SELECT developer_handle FROM registered_table where address=?", + (self.address,), + ) + + def test_get_addresses_i(self): + """Test the get_addresses method of the RegistrationDB class where there are more than 0 addresses.""" + # setup + result = [["address_1", "something_1"], ["address_2", "something_2"]] + + # operation + with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: + actual_addresses = self.db.get_addresses( + self.developer_handle, + ) + + # after + mock_exe.assert_any_call( + "SELECT address FROM registered_table where developer_handle=?", + (self.developer_handle,), + ) + assert actual_addresses == ["address_1", "address_2"] + + def test_get_addresses_ii(self): + """Test the get_addresses method of the RegistrationDB class where there are 0 addresses.""" + # setup + result = [] + + # operation + with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: + with pytest.raises(ValueError, match=f"Should find at least one address for developer_handle={self.developer_handle}."): + self.db.get_addresses(self.developer_handle,) + + # after + mock_exe.assert_any_call( + "SELECT address FROM registered_table where developer_handle=?", + (self.developer_handle,), + ) + + def test_get_handle_and_trades(self): + """Test the get_handle_and_trades method of the RegistrationDB class.""" + # setup + result = ["address_1", "address_2"] + trade_counts = [2, 5] + + # operation + with patch.object(self.db, "get_developer_handle", return_value=self.developer_handle) as mock_developer_handle: + with patch.object(self.db, "get_addresses", return_value=result) as mock_addresses: + with patch.object(self.db, "get_trade_count", side_effect=trade_counts) as mock_trade_counts: + actual_addresses = self.db.get_handle_and_trades( + self.address, + ) + + # after + mock_developer_handle.assert_any_call(self.address) + mock_addresses.assert_any_call(self.developer_handle) + mock_trade_counts.assert_called() + + assert actual_addresses == (self.developer_handle, sum(trade_counts)) + + def test_get_all_addresses_and_handles(self): + """Test the get_all_addresses_and_handles method of the RegistrationDB class.""" + # setup + result = [("address_1", "something_1"), ("address_2", "something_2")] + + # operation + with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: + actual_result = self.db.get_all_addresses_and_handles() + + # after + mock_exe.assert_any_call("SELECT address, developer_handle FROM registered_table", ()) + assert actual_result == result + + def test_get_leaderboard(self): + """Test the get_leaderboard method of the RegistrationDB class.""" + # setup + addresses_and_handlers = [("address_1", "handler_1"), ("address_2", "handler_2")] + trade_counts = [2, 5] + + expected_result = [("address_2", "handler_2", 5), ("address_1", "handler_1", 2)] + + # operation + with patch.object(self.db, "get_all_addresses_and_handles", return_value=addresses_and_handlers) as mock_get_addresses: + with patch.object(self.db, "get_trade_count", side_effect=trade_counts) as mock_trade_counts: + actual_result = self.db.get_leaderboard() + + # after + mock_get_addresses.assert_called_once() + mock_trade_counts.assert_called() + + assert actual_result == expected_result + + def test_set_registered_i(self): + """Test the set_registered method of the RegistrationDB class where is_registered is False.""" + # operation + with patch.object(self.db, "is_registered", return_value=False) as mock_registered: + with patch.object(self.db, "_execute_single_sql") as mock_exe: + self.db.set_registered(self.address, self.developer_handle) + + # after + mock_registered.assert_any_call(self.address) + mock_exe.assert_any_call( + "INSERT OR REPLACE INTO registered_table(address, ethereum_address, ethereum_signature, fetchai_signature, developer_handle, tweet) values(?, ?, ?, ?, ?, ?)", + (self.address, "", "", "", self.developer_handle, ""), + ) + + def test_set_registered_ii(self): + """Test the set_registered method of the RegistrationDB class where is_registered is True.""" + # operation + with patch.object(self.db, "is_registered", return_value=True) as mock_registered: + with patch.object(self.db, "_execute_single_sql") as mock_exe: + self.db.set_registered(self.address, self.developer_handle) + + # after + mock_registered.assert_any_call(self.address) + mock_exe.assert_not_called() + + def test_is_registered_i(self): + """Test the is_registered method of the RegistrationDB class where result's length is more than 0.""" + # setup + result = [("address_1", "something_1"), ("address_2", "something_2")] + + # operation + with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: + is_registered = self.db.is_registered(self.address) + + # after + mock_exe.assert_any_call( + "SELECT * FROM registered_table WHERE address=?", + (self.address,), + ) + assert is_registered is True + + def test_is_registered_ii(self): + """Test the is_registered method of the RegistrationDB class where result's length is 0.""" + # setup + result = [] + + # operation + with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: + is_registered = self.db.is_registered(self.address) + + # after + mock_exe.assert_any_call( + "SELECT * FROM registered_table WHERE address=?", + (self.address,), + ) + assert is_registered is False diff --git a/tests/test_packages/test_skills/test_confirmation_aw3/test_strategy.py b/tests/test_packages/test_skills/test_confirmation_aw3/test_strategy.py new file mode 100644 index 0000000000..d51270f24d --- /dev/null +++ b/tests/test_packages/test_skills/test_confirmation_aw3/test_strategy.py @@ -0,0 +1,222 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the tests of the strategy class of the confirmation aw3 skill.""" + +import datetime +import json +import logging +from pathlib import Path +from typing import cast +from unittest.mock import Mock, patch + +from packages.fetchai.protocols.http.message import HttpMessage +from packages.fetchai.skills.confirmation_aw3.registration_db import RegistrationDB +from packages.fetchai.skills.confirmation_aw3.strategy import Strategy, HTTP_CLIENT_PUBLIC_ID + +from tests.conftest import ROOT_DIR +from tests.test_packages.test_skills.test_confirmation_aw3.intermediate_class import ( + ConfirmationAW3TestCase, +) + + +class TestStrategy(ConfirmationAW3TestCase): + """Test Strategy of confirmation aw3.""" + + path_to_skill = Path(ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw3") + + @classmethod + def setup(cls): + """Setup the test class.""" + super().setup() + + cls.location_name = "berlin" + cls.locations = { + cls.location_name: { + "latitude": 52.52, + "longitude": 13.405 + }, + } + cls.search_query_type = "weather" + cls.search_queries = { + cls.search_query_type: { + "constraint_type": "==", + "search_key": "seller_service", + "search_value": "weather_data", + }, + } + cls.leaderboard_url = "some_url" + cls.leaderboard_token = "some_token" + + cls.strategy = Strategy( + aw1_aea="some_aw1_aea", + locations=cls.locations, + search_queries=cls.search_queries, + leaderboard_url=cls.leaderboard_url, + leaderboard_token=cls.leaderboard_token, + name="strategy", + skill_context=cls._skill.skill_context, + ) + + cls.address = "some_address" + cls.info = { + "ethereum_address": "some_value", + "signature_of_ethereum_address": "some_signature_of_ethereum_address", + "signature_of_fetchai_address": "some_signature_of_fetchai_address", + "developer_handle": "some_developer_handle", + "tweet": "some_tweet", + } + cls.logger = cls._skill.skill_context.logger + cls.db = cast(RegistrationDB, cls._skill.skill_context.registration_db) + + cls.counterparty = "couterparty_1" + + def test__init__(self): + """Test the __init__ of Strategy class.""" + assert self.strategy.aw1_aea == self.aw1_aea + assert self.strategy._locations == self.locations + assert self.strategy._search_queries == self.search_queries + assert self.strategy.leaderboard_url == f"{self.leaderboard_url}/insert" + assert self.strategy.leaderboard_token == self.leaderboard_token + + def test_get_acceptable_counterparties(self): + """Test the get_acceptable_counterparties method of the Strategy class.""" + # setup + couterparties = ("couterparty_1", "couterparty_2", "couterparty_3") + is_valid_counterparty = [True, False, True] + + # operation + with patch.object( + self.strategy, "is_valid_counterparty", side_effect=is_valid_counterparty + ): + actual_acceptable_counterparties = self.strategy.get_acceptable_counterparties( + couterparties + ) + + # after + assert actual_acceptable_counterparties == ("couterparty_1", "couterparty_3") + + def test_is_valid_counterparty_i(self): + """Test the is_valid_counterparty method of the Strategy class where is_registered is False.""" + # operation + with patch.object(self.db, "is_registered", return_value=False) as mock_is_regostered: + with patch.object(self.logger, "log") as mock_logger: + is_valid = self.strategy.is_valid_counterparty(self.counterparty) + + # after + mock_is_regostered.assert_any_call(self.counterparty) + mock_logger.assert_any_call( + logging.INFO, f"Invalid counterparty={self.counterparty}, not registered!", + ) + assert is_valid is False + + def test_is_valid_counterparty_ii(self): + """Test the is_valid_counterparty method of the Strategy class where is_registered is True.""" + # operation + with patch.object(self.db, "is_registered", return_value=True) as mock_is_regostered: + is_valid = self.strategy.is_valid_counterparty(self.counterparty) + + # after + mock_is_regostered.assert_any_call(self.counterparty) + assert is_valid is True + + def test_successful_trade_with_counterparty(self): + """Test the successful_trade_with_counterparty method of the Strategy class.""" + # setup + data = {"some_key_1": "some_value_1", "some_key_2": "some_value_2"} + developer_handle = "some_developer_handle" + nb_trades = 5 + + mocked_now_str = "2020-12-22 20:33:00.000000" + mock_now = datetime.datetime.strptime(mocked_now_str, "%Y-%m-%d %H:%M:%S.%f") + datetime_mock = Mock(wraps=datetime.datetime) + datetime_mock.now.return_value = mock_now + + # operation + with patch.object(self.db, "set_trade") as mock_set_trade: + with patch("datetime.datetime", new=datetime_mock): + with patch.object(self.logger, "log") as mock_logger: + with patch.object(self.db, "get_handle_and_trades", return_value=(developer_handle, nb_trades)) as mock_handle: + self.strategy.successful_trade_with_counterparty( + self.counterparty, data + ) + + # after + mock_set_trade.assert_any_call(self.counterparty, mock_now, data) + mock_handle.assert_any_call(self.counterparty) + + mock_logger.assert_any_call( + logging.INFO, + f"Successful trade with={self.counterparty}. Data acquired={data}!", + ) + + self.assert_quantity_in_outbox(1) + + has_attributes, error_str = self.message_has_attributes( + actual_message=self.get_message_from_outbox(), + message_type=HttpMessage, + performative=HttpMessage.Performative.REQUEST, + to=str(HTTP_CLIENT_PUBLIC_ID), + sender=self.skill.skill_context.agent_address, + method="POST", + url=self.strategy.leaderboard_url, + headers="Content-Type: application/json; charset=utf-8", + version="", + body=json.dumps( + { + "name": developer_handle, + "points": nb_trades, + "token": self.leaderboard_token, + } + ).encode("utf-8") + ) + assert has_attributes, error_str + + mock_logger.assert_any_call( + logging.INFO, + f"Notifying leaderboard: developer_handle={developer_handle}, nb_trades={nb_trades}.", + ) + + def test_register_counterparty(self): + """Test the register_counterparty method of the Strategy class.""" + # setup + developer_handle = "some_developer_handle" + + # operation + with patch.object(self.db, "set_registered") as mock_set_registered: + self.strategy.register_counterparty(self.counterparty, developer_handle) + + # after + mock_set_registered.assert_any_call(self.counterparty, developer_handle) + + def test_update_search_query_params(self): + """Test the update_search_query_params method of the Strategy class.""" + # operation + with patch.object(self.logger, "log") as mock_logger: + self.strategy.update_search_query_params() + + # after + mock_logger.assert_any_call( + logging.INFO, + f"New search_type={self.search_query_type} and location={self.location_name}.", + ) + + + + + From b73dab5bc4904a36706707c4457744567a86977a Mon Sep 17 00:00:00 2001 From: ali Date: Wed, 23 Dec 2020 18:09:56 +0000 Subject: [PATCH 047/204] linter, type checker, hashes, tox.ini update, makefile update --- Makefile | 6 +- .../skills/confirmation_aw3/skill.yaml | 2 +- packages/hashes.csv | 2 +- path_to_db | Bin 0 -> 16384 bytes .../intermediate_class.py | 20 ++-- .../test_confirmation_aw3/test_behaviours.py | 12 +- .../test_confirmation_aw3/test_dialogues.py | 5 +- .../test_confirmation_aw3/test_handlers.py | 31 +++-- .../test_registration_db.py | 106 ++++++++++++------ .../test_confirmation_aw3/test_strategy.py | 31 ++--- tox.ini | 2 +- 11 files changed, 137 insertions(+), 80 deletions(-) create mode 100644 path_to_db diff --git a/Makefile b/Makefile index 85b147da70..49feadb09c 100644 --- a/Makefile +++ b/Makefile @@ -78,17 +78,17 @@ common_checks: security misc_checks lint static docs .PHONY: test test: - pytest -rfE --doctest-modules aea packages/fetchai/protocols packages/fetchai/connections packages/fetchai/skills/confirmation_aw1 packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation packages/fetchai/skills/simple_buyer packages/fetchai/skills/simple_data_request packages/fetchai/skills/simple_seller packages/fetchai/skills/simple_service_registration packages/fetchai/skills/simple_service_search tests/ --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term --cov=aea --cov=packages/fetchai/protocols --cov=packages/fetchai/connections --cov=packages/fetchai/skills/confirmation_aw1 --cov=packages/fetchai/skills/generic_buyer --cov=packages/fetchai/skills/generic_seller --cov=packages/fetchai/skills/tac_control --cov=packages/fetchai/skills/tac_control_contract --cov=packages/fetchai/skills/tac_participation --cov=packages/fetchai/skills/tac_negotiation --cov=packages/fetchai/skills/simple_buyer --cov=packages/fetchai/skills/simple_data_request --cov=packages/fetchai/skills/simple_seller --cov=packages/fetchai/skills/simple_service_registration --cov=packages/fetchai/skills/simple_service_search --cov-config=.coveragerc + pytest -rfE --doctest-modules aea packages/fetchai/protocols packages/fetchai/connections packages/fetchai/skills/confirmation_aw1 packages/fetchai/skills/confirmation_aw2 packages/fetchai/skills/confirmation_aw3 packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation packages/fetchai/skills/simple_buyer packages/fetchai/skills/simple_data_request packages/fetchai/skills/simple_seller packages/fetchai/skills/simple_service_registration packages/fetchai/skills/simple_service_search tests/ --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term --cov=aea --cov=packages/fetchai/protocols --cov=packages/fetchai/connections --cov=packages/fetchai/skills/confirmation_aw1 --cov=packages/fetchai/skills/confirmation_aw2 --cov=packages/fetchai/skills/confirmation_aw3 --cov=packages/fetchai/skills/generic_buyer --cov=packages/fetchai/skills/generic_seller --cov=packages/fetchai/skills/tac_control --cov=packages/fetchai/skills/tac_control_contract --cov=packages/fetchai/skills/tac_participation --cov=packages/fetchai/skills/tac_negotiation --cov=packages/fetchai/skills/simple_buyer --cov=packages/fetchai/skills/simple_data_request --cov=packages/fetchai/skills/simple_seller --cov=packages/fetchai/skills/simple_service_registration --cov=packages/fetchai/skills/simple_service_search --cov-config=.coveragerc find . -name ".coverage*" -not -name ".coveragerc" -exec rm -fr "{}" \; .PHONY: test-sub test-sub: - pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/confirmation_aw1 packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation tests/test_$(tdir) --cov=aea.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term --cov-config=.coveragerc + pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/confirmation_aw1 packages/fetchai/skills/confirmation_aw2 packages/fetchai/skills/confirmation_aw3 packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation tests/test_$(tdir) --cov=aea.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term --cov-config=.coveragerc find . -name ".coverage*" -not -name ".coveragerc" -exec rm -fr "{}" \; .PHONY: test-sub-p test-sub-p: - pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/confirmation_aw1 packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation packages/fetchai/skills/simple_buyer packages/fetchai/skills/simple_data_request packages/fetchai/skills/simple_seller packages/fetchai/skills/simple_service_registration packages/fetchai/skills/simple_service_search tests/test_packages/test_$(tdir) --cov=packages.fetchai.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term --cov-config=.coveragerc + pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/confirmation_aw1 packages/fetchai/skills/confirmation_aw2 packages/fetchai/skills/confirmation_aw3 packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation packages/fetchai/skills/simple_buyer packages/fetchai/skills/simple_data_request packages/fetchai/skills/simple_seller packages/fetchai/skills/simple_service_registration packages/fetchai/skills/simple_service_search tests/test_packages/test_$(tdir) --cov=packages.fetchai.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term --cov-config=.coveragerc find . -name ".coverage*" -not -name ".coveragerc" -exec rm -fr "{}" \; diff --git a/packages/fetchai/skills/confirmation_aw3/skill.yaml b/packages/fetchai/skills/confirmation_aw3/skill.yaml index b649f89bdf..0277a7106f 100644 --- a/packages/fetchai/skills/confirmation_aw3/skill.yaml +++ b/packages/fetchai/skills/confirmation_aw3/skill.yaml @@ -11,7 +11,7 @@ fingerprint: __init__.py: QmNzTwNA8LVXpmYdLyrg4vZ6GYoAsJYLUAuafwfwZdtiYz behaviours.py: QmeTbjNHgFJddBcxxz9LoiT8tTaod5hZvptp2etgnP9JRe dialogues.py: QmeuRrNPpbvyTAcPwAtEkTfJTDunqyfzPoDhMZWTvCyrJA - handlers.py: QmfS3N2JhkC29iqHLiK273HVA4Qf5rziJKXHpYTg3C1Pdc + handlers.py: QmXU5wYz7mkDYmtPbsunGhseGxRUBbp5N8TxAbvdeuvjDF registration_db.py: QmZcK4tLLyRmb3QaKnYfFdSyChEQpDdEL8NeQ23f3dqYtp strategy.py: QmcE2Zu8XxnJaGcttWewavHzpCHH5B2spHETexXYydmYeV fingerprint_ignore_patterns: [] diff --git a/packages/hashes.csv b/packages/hashes.csv index d06f2cb96c..b21af2a847 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -70,7 +70,7 @@ fetchai/skills/carpark_detection,QmYKRWjC3VEU8QatkptXoGyyL3P9uR8reoDdmgLW2r318x fetchai/skills/coin_price,QmXRHQim1XGbij8QnJ4CPBksdGxmBmpbmPmHdrPAw3z3pE fetchai/skills/confirmation_aw1,QmQbqko8t8utxmfywvDqRhpE1gACVdLxrZNnv98U1aAYyR fetchai/skills/confirmation_aw2,QmSEK1j5AhjKDKkCWgn7xrc2ySWE3HCp4NQ5zcyuEDt2dX -fetchai/skills/confirmation_aw3,QmPkTAzgHivkAs3fViiW3z8q7zkqAN8DS3DcKne992sS2u +fetchai/skills/confirmation_aw3,QmeGfq7mHH6AjybtW85tm7wtW8PmhYu72egmtcSkeuDDiH fetchai/skills/echo,QmTir9ngs1i9dLweJs5gB8Q5NLqDufuJ5A3A2bznibKPMk fetchai/skills/erc1155_client,QmTTuStPpw2zygu9dd9cNkJnzjSVihiNHu1qACSYBT5fvL fetchai/skills/erc1155_deploy,QmX6H5MHsDKNyeKzYzhfnBENQjqLnfHaA3LvVBkinywZrc diff --git a/path_to_db b/path_to_db new file mode 100644 index 0000000000000000000000000000000000000000..37db5688daddd75df7dc0423bc89cc72522ad0e5 GIT binary patch literal 16384 zcmeI%O;5rw7zglf5HTja+&J0`i6me`^sZ=jFiupMiJX>7A7IHga4YJ~5I>Xgt9i7W zKwuj0l7Ew~Yxk@#zxC3rcWZl!_;3;hLh&P3VTQp@IcJQSI_Gs>rfl6R#;Z8tcnvF>vgxSNw(HOJm46lxaHV%9@agR$`3 zY`df<@+VV@+_4Cy{~f+Ij(R3ii#~3;7gz0WpI=)2)x7k|GdUDzKL< z4CnuC{k&Kg1Rwwb2tWV=5P$##AOHafKp+*s|9@-*2tWV=5P$##AOHafKmY;|fWY<( Fd;y5_ptt}4 literal 0 HcmV?d00001 diff --git a/tests/test_packages/test_skills/test_confirmation_aw3/intermediate_class.py b/tests/test_packages/test_skills/test_confirmation_aw3/intermediate_class.py index 1c5119fe87..6aa5b2269a 100644 --- a/tests/test_packages/test_skills/test_confirmation_aw3/intermediate_class.py +++ b/tests/test_packages/test_skills/test_confirmation_aw3/intermediate_class.py @@ -27,9 +27,7 @@ class ConfirmationAW3TestCase(BaseSkillTestCase): """Sets the confirmation_aw3 class up for testing (overrides the necessary config values so tests can be done on confirmation_aw3 skill).""" - path_to_skill = Path( - ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw3" - ) + path_to_skill = Path(ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw3") @classmethod def setup(cls): @@ -37,10 +35,16 @@ def setup(cls): cls.aw1_aea = "some_aw1_aea" cls.leaderboard_url = "some_leaderboard_url" cls.leaderboard_token = "some_leaderboard_token" - config_overrides = {"models": {"strategy": {"args": { - "aw1_aea": cls.aw1_aea, - "leaderboard_url": cls.leaderboard_url, - "leaderboard_token": cls.leaderboard_token, - }}}} + config_overrides = { + "models": { + "strategy": { + "args": { + "aw1_aea": cls.aw1_aea, + "leaderboard_url": cls.leaderboard_url, + "leaderboard_token": cls.leaderboard_token, + } + } + } + } super().setup(config_overrides=config_overrides) diff --git a/tests/test_packages/test_skills/test_confirmation_aw3/test_behaviours.py b/tests/test_packages/test_skills/test_confirmation_aw3/test_behaviours.py index 62d0589ed6..7e6259e343 100644 --- a/tests/test_packages/test_skills/test_confirmation_aw3/test_behaviours.py +++ b/tests/test_packages/test_skills/test_confirmation_aw3/test_behaviours.py @@ -22,7 +22,13 @@ from typing import cast from unittest.mock import patch -from aea.helpers.search.models import Query, Constraint, ConstraintType, Attribute, DataModel +from aea.helpers.search.models import ( + Attribute, + Constraint, + ConstraintType, + DataModel, + Query, +) from packages.fetchai.protocols.oef_search.message import OefSearchMessage from packages.fetchai.skills.confirmation_aw3.behaviours import SearchBehaviour @@ -68,7 +74,9 @@ def test_act(self): # operation with patch.object(self.strategy, "update_search_query_params") as mock_update: - with patch.object(self.strategy, "get_location_and_service_query", return_value=mock_query): + with patch.object( + self.strategy, "get_location_and_service_query", return_value=mock_query + ): self.search_behaviour.act() # after diff --git a/tests/test_packages/test_skills/test_confirmation_aw3/test_dialogues.py b/tests/test_packages/test_skills/test_confirmation_aw3/test_dialogues.py index 80a6f138ec..2ce828d1cf 100644 --- a/tests/test_packages/test_skills/test_confirmation_aw3/test_dialogues.py +++ b/tests/test_packages/test_skills/test_confirmation_aw3/test_dialogues.py @@ -24,7 +24,10 @@ from aea.test_tools.test_skill import COUNTERPARTY_ADDRESS from packages.fetchai.protocols.http.message import HttpMessage -from packages.fetchai.skills.confirmation_aw3.dialogues import HttpDialogue, HttpDialogues +from packages.fetchai.skills.confirmation_aw3.dialogues import ( + HttpDialogue, + HttpDialogues, +) from tests.conftest import ROOT_DIR from tests.test_packages.test_skills.test_confirmation_aw3.intermediate_class import ( diff --git a/tests/test_packages/test_skills/test_confirmation_aw3/test_handlers.py b/tests/test_packages/test_skills/test_confirmation_aw3/test_handlers.py index 2ccdc0a6b3..5653d10c10 100644 --- a/tests/test_packages/test_skills/test_confirmation_aw3/test_handlers.py +++ b/tests/test_packages/test_skills/test_confirmation_aw3/test_handlers.py @@ -29,8 +29,15 @@ from packages.fetchai.protocols.default.dialogues import DefaultDialogue from packages.fetchai.protocols.default.message import DefaultMessage from packages.fetchai.protocols.http.message import HttpMessage -from packages.fetchai.skills.confirmation_aw3.dialogues import HttpDialogue, HttpDialogues, DefaultDialogues -from packages.fetchai.skills.confirmation_aw3.handlers import DefaultHandler, HttpHandler +from packages.fetchai.skills.confirmation_aw3.dialogues import ( + DefaultDialogues, + HttpDialogue, + HttpDialogues, +) +from packages.fetchai.skills.confirmation_aw3.handlers import ( + DefaultHandler, + HttpHandler, +) from packages.fetchai.skills.confirmation_aw3.strategy import Strategy from tests.conftest import ROOT_DIR @@ -271,11 +278,11 @@ def setup(cls): HttpDialogues, cls._skill.skill_context.http_dialogues ) - cls.method= "some_method", - cls.url= "some_url", - cls.version= "some_version", - cls.headers= "some_headers", - cls.body= b'some_body', + cls.method = "some_method" + cls.url = "some_url" + cls.version = "some_version" + cls.headers = "some_headers" + cls.body = b"some_body" cls.list_of_http_messages = ( DialogueMessage( @@ -286,8 +293,7 @@ def setup(cls): "version": cls.version, "headers": cls.headers, "body": cls.body, - - } + }, ), ) @@ -333,8 +339,7 @@ def test__handle_response(self): http_dialogue = cast( HttpDialogue, self.prepare_skill_dialogue( - dialogues=self.http_dialogues, - messages=self.list_of_http_messages[:1], + dialogues=self.http_dialogues, messages=self.list_of_http_messages[:1], ), ) @@ -384,7 +389,9 @@ def test_handle_invalid(self): self.http_handler.handle(incoming_message) # after - http_dialogue = cast(HttpDialogue, self.http_dialogues.get_dialogue(incoming_message)) + http_dialogue = cast( + HttpDialogue, self.http_dialogues.get_dialogue(incoming_message) + ) mock_logger.assert_any_call( logging.WARNING, f"cannot handle http message of performative={incoming_message.performative} in dialogue={http_dialogue}.", diff --git a/tests/test_packages/test_skills/test_confirmation_aw3/test_registration_db.py b/tests/test_packages/test_skills/test_confirmation_aw3/test_registration_db.py index fdc7a42d0a..011ce39b49 100644 --- a/tests/test_packages/test_skills/test_confirmation_aw3/test_registration_db.py +++ b/tests/test_packages/test_skills/test_confirmation_aw3/test_registration_db.py @@ -73,15 +73,14 @@ def test_get_trade_count(self): result = [["1", "2"], ["3", "4"]] # operation - with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: - actual_result = self.db.get_trade_count( - self.address, - ) + with patch.object( + self.db, "_execute_single_sql", return_value=result + ) as mock_exe: + actual_result = self.db.get_trade_count(self.address,) # after mock_exe.assert_any_call( - "SELECT COUNT(*) FROM trades_table where address=?", - (self.address,), + "SELECT COUNT(*) FROM trades_table where address=?", (self.address,), ) assert actual_result == 1 @@ -91,10 +90,10 @@ def test_get_developer_handle_i(self): result = [["developer_handle"], ["something_1", "something_2"]] # operation - with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: - actual_result = self.db.get_developer_handle( - self.address, - ) + with patch.object( + self.db, "_execute_single_sql", return_value=result + ) as mock_exe: + actual_result = self.db.get_developer_handle(self.address,) # after mock_exe.assert_any_call( @@ -109,8 +108,13 @@ def test_get_developer_handle_ii(self): result = [["developer_handle", "something_1"], ["something_2", "something_3"]] # operation - with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: - with pytest.raises(ValueError, match=f"More than one developer_handle found for address={self.address}."): + with patch.object( + self.db, "_execute_single_sql", return_value=result + ) as mock_exe: + with pytest.raises( + ValueError, + match=f"More than one developer_handle found for address={self.address}.", + ): self.db.get_developer_handle(self.address) # after @@ -125,10 +129,10 @@ def test_get_addresses_i(self): result = [["address_1", "something_1"], ["address_2", "something_2"]] # operation - with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: - actual_addresses = self.db.get_addresses( - self.developer_handle, - ) + with patch.object( + self.db, "_execute_single_sql", return_value=result + ) as mock_exe: + actual_addresses = self.db.get_addresses(self.developer_handle,) # after mock_exe.assert_any_call( @@ -143,8 +147,13 @@ def test_get_addresses_ii(self): result = [] # operation - with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: - with pytest.raises(ValueError, match=f"Should find at least one address for developer_handle={self.developer_handle}."): + with patch.object( + self.db, "_execute_single_sql", return_value=result + ) as mock_exe: + with pytest.raises( + ValueError, + match=f"Should find at least one address for developer_handle={self.developer_handle}.", + ): self.db.get_addresses(self.developer_handle,) # after @@ -160,12 +169,16 @@ def test_get_handle_and_trades(self): trade_counts = [2, 5] # operation - with patch.object(self.db, "get_developer_handle", return_value=self.developer_handle) as mock_developer_handle: - with patch.object(self.db, "get_addresses", return_value=result) as mock_addresses: - with patch.object(self.db, "get_trade_count", side_effect=trade_counts) as mock_trade_counts: - actual_addresses = self.db.get_handle_and_trades( - self.address, - ) + with patch.object( + self.db, "get_developer_handle", return_value=self.developer_handle + ) as mock_developer_handle: + with patch.object( + self.db, "get_addresses", return_value=result + ) as mock_addresses: + with patch.object( + self.db, "get_trade_count", side_effect=trade_counts + ) as mock_trade_counts: + actual_addresses = self.db.get_handle_and_trades(self.address,) # after mock_developer_handle.assert_any_call(self.address) @@ -180,24 +193,37 @@ def test_get_all_addresses_and_handles(self): result = [("address_1", "something_1"), ("address_2", "something_2")] # operation - with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: + with patch.object( + self.db, "_execute_single_sql", return_value=result + ) as mock_exe: actual_result = self.db.get_all_addresses_and_handles() # after - mock_exe.assert_any_call("SELECT address, developer_handle FROM registered_table", ()) + mock_exe.assert_any_call( + "SELECT address, developer_handle FROM registered_table", () + ) assert actual_result == result def test_get_leaderboard(self): """Test the get_leaderboard method of the RegistrationDB class.""" # setup - addresses_and_handlers = [("address_1", "handler_1"), ("address_2", "handler_2")] + addresses_and_handlers = [ + ("address_1", "handler_1"), + ("address_2", "handler_2"), + ] trade_counts = [2, 5] expected_result = [("address_2", "handler_2", 5), ("address_1", "handler_1", 2)] # operation - with patch.object(self.db, "get_all_addresses_and_handles", return_value=addresses_and_handlers) as mock_get_addresses: - with patch.object(self.db, "get_trade_count", side_effect=trade_counts) as mock_trade_counts: + with patch.object( + self.db, + "get_all_addresses_and_handles", + return_value=addresses_and_handlers, + ) as mock_get_addresses: + with patch.object( + self.db, "get_trade_count", side_effect=trade_counts + ) as mock_trade_counts: actual_result = self.db.get_leaderboard() # after @@ -209,7 +235,9 @@ def test_get_leaderboard(self): def test_set_registered_i(self): """Test the set_registered method of the RegistrationDB class where is_registered is False.""" # operation - with patch.object(self.db, "is_registered", return_value=False) as mock_registered: + with patch.object( + self.db, "is_registered", return_value=False + ) as mock_registered: with patch.object(self.db, "_execute_single_sql") as mock_exe: self.db.set_registered(self.address, self.developer_handle) @@ -223,7 +251,9 @@ def test_set_registered_i(self): def test_set_registered_ii(self): """Test the set_registered method of the RegistrationDB class where is_registered is True.""" # operation - with patch.object(self.db, "is_registered", return_value=True) as mock_registered: + with patch.object( + self.db, "is_registered", return_value=True + ) as mock_registered: with patch.object(self.db, "_execute_single_sql") as mock_exe: self.db.set_registered(self.address, self.developer_handle) @@ -237,13 +267,14 @@ def test_is_registered_i(self): result = [("address_1", "something_1"), ("address_2", "something_2")] # operation - with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: + with patch.object( + self.db, "_execute_single_sql", return_value=result + ) as mock_exe: is_registered = self.db.is_registered(self.address) # after mock_exe.assert_any_call( - "SELECT * FROM registered_table WHERE address=?", - (self.address,), + "SELECT * FROM registered_table WHERE address=?", (self.address,), ) assert is_registered is True @@ -253,12 +284,13 @@ def test_is_registered_ii(self): result = [] # operation - with patch.object(self.db, "_execute_single_sql", return_value=result) as mock_exe: + with patch.object( + self.db, "_execute_single_sql", return_value=result + ) as mock_exe: is_registered = self.db.is_registered(self.address) # after mock_exe.assert_any_call( - "SELECT * FROM registered_table WHERE address=?", - (self.address,), + "SELECT * FROM registered_table WHERE address=?", (self.address,), ) assert is_registered is False diff --git a/tests/test_packages/test_skills/test_confirmation_aw3/test_strategy.py b/tests/test_packages/test_skills/test_confirmation_aw3/test_strategy.py index d51270f24d..5d02ce175f 100644 --- a/tests/test_packages/test_skills/test_confirmation_aw3/test_strategy.py +++ b/tests/test_packages/test_skills/test_confirmation_aw3/test_strategy.py @@ -27,7 +27,10 @@ from packages.fetchai.protocols.http.message import HttpMessage from packages.fetchai.skills.confirmation_aw3.registration_db import RegistrationDB -from packages.fetchai.skills.confirmation_aw3.strategy import Strategy, HTTP_CLIENT_PUBLIC_ID +from packages.fetchai.skills.confirmation_aw3.strategy import ( + HTTP_CLIENT_PUBLIC_ID, + Strategy, +) from tests.conftest import ROOT_DIR from tests.test_packages.test_skills.test_confirmation_aw3.intermediate_class import ( @@ -47,10 +50,7 @@ def setup(cls): cls.location_name = "berlin" cls.locations = { - cls.location_name: { - "latitude": 52.52, - "longitude": 13.405 - }, + cls.location_name: {"latitude": 52.52, "longitude": 13.405}, } cls.search_query_type = "weather" cls.search_queries = { @@ -114,7 +114,9 @@ def test_get_acceptable_counterparties(self): def test_is_valid_counterparty_i(self): """Test the is_valid_counterparty method of the Strategy class where is_registered is False.""" # operation - with patch.object(self.db, "is_registered", return_value=False) as mock_is_regostered: + with patch.object( + self.db, "is_registered", return_value=False + ) as mock_is_regostered: with patch.object(self.logger, "log") as mock_logger: is_valid = self.strategy.is_valid_counterparty(self.counterparty) @@ -128,7 +130,9 @@ def test_is_valid_counterparty_i(self): def test_is_valid_counterparty_ii(self): """Test the is_valid_counterparty method of the Strategy class where is_registered is True.""" # operation - with patch.object(self.db, "is_registered", return_value=True) as mock_is_regostered: + with patch.object( + self.db, "is_registered", return_value=True + ) as mock_is_regostered: is_valid = self.strategy.is_valid_counterparty(self.counterparty) # after @@ -151,7 +155,11 @@ def test_successful_trade_with_counterparty(self): with patch.object(self.db, "set_trade") as mock_set_trade: with patch("datetime.datetime", new=datetime_mock): with patch.object(self.logger, "log") as mock_logger: - with patch.object(self.db, "get_handle_and_trades", return_value=(developer_handle, nb_trades)) as mock_handle: + with patch.object( + self.db, + "get_handle_and_trades", + return_value=(developer_handle, nb_trades), + ) as mock_handle: self.strategy.successful_trade_with_counterparty( self.counterparty, data ) @@ -183,7 +191,7 @@ def test_successful_trade_with_counterparty(self): "points": nb_trades, "token": self.leaderboard_token, } - ).encode("utf-8") + ).encode("utf-8"), ) assert has_attributes, error_str @@ -215,8 +223,3 @@ def test_update_search_query_params(self): logging.INFO, f"New search_type={self.search_query_type} and location={self.location_name}.", ) - - - - - diff --git a/tox.ini b/tox.ini index 3223ceccfa..c645622f78 100644 --- a/tox.ini +++ b/tox.ini @@ -37,7 +37,7 @@ deps = pytest-rerunfailures==9.0 aioprometheus==20.0.1 -commands = pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/confirmation_aw1 packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation packages/fetchai/skills/simple_buyer packages/fetchai/skills/simple_data_request packages/fetchai/skills/simple_seller packages/fetchai/skills/simple_service_registration packages/fetchai/skills/simple_service_search tests/ --cov-report=html --cov-report=xml --cov-report=term --cov-report=term-missing --cov=aea --cov=packages/fetchai/connections --cov=packages/fetchai/protocols --cov=packages/fetchai/skills/confirmation_aw1 --cov=packages/fetchai/skills/generic_buyer --cov=packages/fetchai/skills/generic_seller --cov=packages/fetchai/skills/tac_control --cov=packages/fetchai/skills/tac_control_contract --cov=packages/fetchai/skills/tac_participation --cov=packages/fetchai/skills/tac_negotiation --cov=packages/fetchai/skills/simple_buyer --cov=packages/fetchai/skills/simple_data_request --cov=packages/fetchai/skills/simple_seller --cov=packages/fetchai/skills/simple_service_registration --cov=packages/fetchai/skills/simple_service_search --cov-config=.coveragerc {posargs} +commands = pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/confirmation_aw1 packages/fetchai/skills/confirmation_aw3 packages/fetchai/skills/confirmation_aw3 packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation packages/fetchai/skills/tac_negotiation packages/fetchai/skills/simple_buyer packages/fetchai/skills/simple_data_request packages/fetchai/skills/simple_seller packages/fetchai/skills/simple_service_registration packages/fetchai/skills/simple_service_search tests/ --cov-report=html --cov-report=xml --cov-report=term --cov-report=term-missing --cov=aea --cov=packages/fetchai/connections --cov=packages/fetchai/protocols --cov=packages/fetchai/skills/confirmation_aw1 --cov=packages/fetchai/skills/confirmation_aw2 --cov=packages/fetchai/skills/confirmation_aw3 --cov=packages/fetchai/skills/generic_buyer --cov=packages/fetchai/skills/generic_seller --cov=packages/fetchai/skills/tac_control --cov=packages/fetchai/skills/tac_control_contract --cov=packages/fetchai/skills/tac_participation --cov=packages/fetchai/skills/tac_negotiation --cov=packages/fetchai/skills/simple_buyer --cov=packages/fetchai/skills/simple_data_request --cov=packages/fetchai/skills/simple_seller --cov=packages/fetchai/skills/simple_service_registration --cov=packages/fetchai/skills/simple_service_search --cov-config=.coveragerc {posargs} [testenv:py3.6] basepython = python3.6 From 87665b5969669473c686b269404eb4ab747caea9 Mon Sep 17 00:00:00 2001 From: ali Date: Thu, 24 Dec 2020 10:05:50 +0000 Subject: [PATCH 048/204] filling coverage gaps; cleanups --- .../confirmation_aw2/registration_db.py | 2 +- .../confirmation_aw3/registration_db.py | 2 +- path_to_db | Bin 16384 -> 0 bytes .../test_registration_db.py | 23 ++++-- .../intermediate_class.py | 38 ++++++++++ .../test_confirmation_aw2/test_handlers.py | 11 ++- .../test_registration_db.py | 53 +++++++++++-- .../test_confirmation_aw2/test_strategy.py | 24 ++++-- .../test_registration_db.py | 32 +++++++- .../test_confirmation_aw3/test_strategy.py | 71 +++++++++++++++++- 10 files changed, 226 insertions(+), 30 deletions(-) delete mode 100644 path_to_db create mode 100644 tests/test_packages/test_skills/test_confirmation_aw2/intermediate_class.py diff --git a/packages/fetchai/skills/confirmation_aw2/registration_db.py b/packages/fetchai/skills/confirmation_aw2/registration_db.py index f13f22cd70..0d1ce2442e 100644 --- a/packages/fetchai/skills/confirmation_aw2/registration_db.py +++ b/packages/fetchai/skills/confirmation_aw2/registration_db.py @@ -48,7 +48,7 @@ def __init__(self, **kwargs): else custom_path ) if not os.path.exists(os.path.dirname(os.path.abspath(self.db_path))): - raise ValueError(f"Path={self.db_path} not valid!") + raise ValueError(f"Path={self.db_path} not valid!") # pragma: nocover self._initialise_backend() def _initialise_backend(self) -> None: diff --git a/packages/fetchai/skills/confirmation_aw3/registration_db.py b/packages/fetchai/skills/confirmation_aw3/registration_db.py index 0ec5f6239e..92a03fab2b 100644 --- a/packages/fetchai/skills/confirmation_aw3/registration_db.py +++ b/packages/fetchai/skills/confirmation_aw3/registration_db.py @@ -49,7 +49,7 @@ def __init__(self, **kwargs): else custom_path ) if not os.path.exists(os.path.dirname(os.path.abspath(self.db_path))): - raise ValueError(f"Path={self.db_path} not valid!") + raise ValueError(f"Path={self.db_path} not valid!") # pragma: nocover self._initialise_backend() def _initialise_backend(self) -> None: diff --git a/path_to_db b/path_to_db deleted file mode 100644 index 37db5688daddd75df7dc0423bc89cc72522ad0e5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI%O;5rw7zglf5HTja+&J0`i6me`^sZ=jFiupMiJX>7A7IHga4YJ~5I>Xgt9i7W zKwuj0l7Ew~Yxk@#zxC3rcWZl!_;3;hLh&P3VTQp@IcJQSI_Gs>rfl6R#;Z8tcnvF>vgxSNw(HOJm46lxaHV%9@agR$`3 zY`df<@+VV@+_4Cy{~f+Ij(R3ii#~3;7gz0WpI=)2)x7k|GdUDzKL< z4CnuC{k&Kg1Rwwb2tWV=5P$##AOHafKp+*s|9@-*2tWV=5P$##AOHafKmY;|fWY<( Fd;y5_ptt}4 diff --git a/tests/test_packages/test_skills/test_confirmation_aw1/test_registration_db.py b/tests/test_packages/test_skills/test_confirmation_aw1/test_registration_db.py index 205fb842bb..82e2fe0dde 100644 --- a/tests/test_packages/test_skills/test_confirmation_aw1/test_registration_db.py +++ b/tests/test_packages/test_skills/test_confirmation_aw1/test_registration_db.py @@ -25,8 +25,6 @@ from aea.test_tools.test_skill import BaseSkillTestCase -from packages.fetchai.skills.confirmation_aw1.registration_db import RegistrationDB - from tests.conftest import ROOT_DIR @@ -40,13 +38,24 @@ def setup(cls): """Setup the test class.""" super().setup() cls.custom_path = None - cls.db = RegistrationDB( - custom_path=cls.custom_path, - name="strategy", - skill_context=cls._skill.skill_context, - ) + cls.db = cls._skill.skill_context.registration_db cls.address = "some_address" + def test__initialise_backend(self): + """Test the _initialise_backend method of the RegistrationDB class.""" + # operation + with patch("os.path.isfile", return_value=False) as mock_is_file: + with patch.object(self.db, "_execute_single_sql") as mock_exe: + self.db._initialise_backend() + + # after + mock_is_file.assert_called_once() + mock_exe.assert_any_call( + "CREATE TABLE IF NOT EXISTS registered_table (address TEXT, ethereum_address TEXT, " + "ethereum_signature TEXT, fetchai_signature TEXT, " + "developer_handle TEXT, tweet TEXT)" + ) + def test_set_registered(self): """Test the set_registered method of the RegistrationDB class.""" # setup diff --git a/tests/test_packages/test_skills/test_confirmation_aw2/intermediate_class.py b/tests/test_packages/test_skills/test_confirmation_aw2/intermediate_class.py new file mode 100644 index 0000000000..20cda1757f --- /dev/null +++ b/tests/test_packages/test_skills/test_confirmation_aw2/intermediate_class.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the tests of the behaviour classes of the simple_data_request skill.""" +from pathlib import Path + +from aea.test_tools.test_skill import BaseSkillTestCase + +from tests.conftest import ROOT_DIR + + +class ConfirmationAW2TestCase(BaseSkillTestCase): + """Sets the confirmation_aw2 class up for testing (overrides the necessary config values so tests can be done on confirmation_aw2 skill).""" + + path_to_skill = Path(ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw2") + + @classmethod + def setup(cls): + """Setup the test class.""" + cls.aw1_aea = "some_aw1_aea" + config_overrides = {"models": {"strategy": {"args": {"aw1_aea": cls.aw1_aea}}}} + + super().setup(config_overrides=config_overrides) diff --git a/tests/test_packages/test_skills/test_confirmation_aw2/test_handlers.py b/tests/test_packages/test_skills/test_confirmation_aw2/test_handlers.py index 15f8f80463..3d1b165e86 100644 --- a/tests/test_packages/test_skills/test_confirmation_aw2/test_handlers.py +++ b/tests/test_packages/test_skills/test_confirmation_aw2/test_handlers.py @@ -25,7 +25,6 @@ from aea.crypto.ledger_apis import LedgerApis from aea.protocols.dialogue.base import DialogueMessage -from aea.test_tools.test_skill import BaseSkillTestCase from packages.fetchai.protocols.default.dialogues import DefaultDialogue from packages.fetchai.protocols.default.message import DefaultMessage @@ -35,9 +34,12 @@ from packages.fetchai.skills.confirmation_aw2.strategy import Strategy from tests.conftest import ROOT_DIR +from tests.test_packages.test_skills.test_confirmation_aw2.intermediate_class import ( + ConfirmationAW2TestCase, +) -class TestDefaultHandler(BaseSkillTestCase): +class TestDefaultHandler(ConfirmationAW2TestCase): """Test default handler of confirmation aw2.""" path_to_skill = Path(ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw2") @@ -45,10 +47,7 @@ class TestDefaultHandler(BaseSkillTestCase): @classmethod def setup(cls): """Setup the test class.""" - cls.aw1_aea = "some_aw1_aea" - config_overrides = {"models": {"strategy": {"args": {"aw1_aea": cls.aw1_aea}}}} - - super().setup(config_overrides=config_overrides) + super().setup() cls.default_handler = cast( DefaultHandler, cls._skill.skill_context.handlers.default_handler ) diff --git a/tests/test_packages/test_skills/test_confirmation_aw2/test_registration_db.py b/tests/test_packages/test_skills/test_confirmation_aw2/test_registration_db.py index 0ecd7ce794..7cd34b4b86 100644 --- a/tests/test_packages/test_skills/test_confirmation_aw2/test_registration_db.py +++ b/tests/test_packages/test_skills/test_confirmation_aw2/test_registration_db.py @@ -23,14 +23,15 @@ from pathlib import Path from unittest.mock import Mock, patch -from aea.test_tools.test_skill import BaseSkillTestCase - from packages.fetchai.skills.confirmation_aw2.registration_db import RegistrationDB from tests.conftest import ROOT_DIR +from tests.test_packages.test_skills.test_confirmation_aw2.intermediate_class import ( + ConfirmationAW2TestCase, +) -class TestStrategy(BaseSkillTestCase): +class TestStrategy(ConfirmationAW2TestCase): """Test RegistrationDB of confirmation aw2.""" path_to_skill = Path(ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw2") @@ -38,9 +39,7 @@ class TestStrategy(BaseSkillTestCase): @classmethod def setup(cls): """Setup the test class.""" - cls.aw1_aea = "some_aw1_aea" - config_overrides = {"models": {"strategy": {"args": {"aw1_aea": cls.aw1_aea}}}} - super().setup(config_overrides=config_overrides) + super().setup() cls.custom_path = None cls.db = RegistrationDB( custom_path=cls.custom_path, @@ -60,6 +59,23 @@ def setup(cls): cls.first_info = "first_info" cls.second_info = "second_info" + def test__initialise_backend(self): + """Test the _initialise_backend method of the RegistrationDB class.""" + # operation + with patch.object(self.db, "_execute_single_sql") as mock_exe: + self.db._initialise_backend() + + # after + mock_exe.assert_any_call( + "CREATE TABLE IF NOT EXISTS registered_table (address TEXT, ethereum_address TEXT, " + "ethereum_signature TEXT, fetchai_signature TEXT, " + "developer_handle TEXT, tweet TEXT)" + ) + mock_exe.assert_any_call( + "CREATE TABLE IF NOT EXISTS trade_table (address TEXT PRIMARY KEY, first_trade timestamp, " + "second_trade timestamp, first_info TEXT, second_info TEXT)" + ) + def test_set_trade_i(self): """Test the set_trade method of the RegistrationDB class where record IS None.""" # operation @@ -112,6 +128,31 @@ def test_set_trade_ii(self): ), ) + def test_set_trade_iii(self): + """Test the set_trade method of the RegistrationDB class where record is NOT None and first_trade is None.""" + # setup + record = ( + self.address, + None, + None, + self.first_info, + self.second_info, + ) + + # operation + with patch.object( + self.db, "get_trade_table", return_value=record + ) as mock_get_trade_table: + with patch.object(self.db, "_execute_single_sql") as mock_exe: + self.db.set_trade( + self.address, self.timestamp, self.data, + ) + + # after + mock_get_trade_table.assert_called_once() + + mock_exe.assert_not_called() + def test_get_trade_table(self): """Test the get_trade_table method of the RegistrationDB class.""" # setup diff --git a/tests/test_packages/test_skills/test_confirmation_aw2/test_strategy.py b/tests/test_packages/test_skills/test_confirmation_aw2/test_strategy.py index 13bd901e6e..ad9efb2e6f 100644 --- a/tests/test_packages/test_skills/test_confirmation_aw2/test_strategy.py +++ b/tests/test_packages/test_skills/test_confirmation_aw2/test_strategy.py @@ -24,15 +24,18 @@ from typing import cast from unittest.mock import Mock, patch -from aea.test_tools.test_skill import BaseSkillTestCase +import pytest from packages.fetchai.skills.confirmation_aw2.registration_db import RegistrationDB from packages.fetchai.skills.confirmation_aw2.strategy import Strategy from tests.conftest import ROOT_DIR +from tests.test_packages.test_skills.test_confirmation_aw2.intermediate_class import ( + ConfirmationAW2TestCase, +) -class TestStrategy(BaseSkillTestCase): +class TestStrategy(ConfirmationAW2TestCase): """Test Strategy of confirmation aw2.""" path_to_skill = Path(ROOT_DIR, "packages", "fetchai", "skills", "confirmation_aw2") @@ -40,9 +43,7 @@ class TestStrategy(BaseSkillTestCase): @classmethod def setup(cls): """Setup the test class.""" - cls.aw1_aea = "some_aw1_aea" - config_overrides = {"models": {"strategy": {"args": {"aw1_aea": cls.aw1_aea}}}} - super().setup(config_overrides=config_overrides) + super().setup() cls.minimum_hours_between_txs = 4 cls.minimum_minutes_since_last_attempt = 2 @@ -67,7 +68,7 @@ def setup(cls): cls.counterparty = "couterparty_1" - def test__init__(self): + def test__init__i(self): """Test the __init__ of Strategy class.""" assert self.strategy.aw1_aea == self.aw1_aea assert self.strategy.minimum_hours_between_txs == self.minimum_hours_between_txs @@ -76,6 +77,17 @@ def test__init__(self): == self.minimum_minutes_since_last_attempt ) + def test__init__ii(self): + """Test the __init__ of Strategy class where aw1_aea is None.""" + with pytest.raises(ValueError, match="aw1_aea must be provided!"): + Strategy( + aw1_aea=None, + mininum_hours_between_txs=self.minimum_hours_between_txs, + minimum_minutes_since_last_attempt=self.minimum_minutes_since_last_attempt, + name="strategy", + skill_context=self.skill.skill_context, + ) + def test_get_acceptable_counterparties(self): """Test the get_acceptable_counterparties method of the Strategy class.""" # setup diff --git a/tests/test_packages/test_skills/test_confirmation_aw3/test_registration_db.py b/tests/test_packages/test_skills/test_confirmation_aw3/test_registration_db.py index 011ce39b49..f29c727c09 100644 --- a/tests/test_packages/test_skills/test_confirmation_aw3/test_registration_db.py +++ b/tests/test_packages/test_skills/test_confirmation_aw3/test_registration_db.py @@ -204,8 +204,8 @@ def test_get_all_addresses_and_handles(self): ) assert actual_result == result - def test_get_leaderboard(self): - """Test the get_leaderboard method of the RegistrationDB class.""" + def test_get_leaderboard_i(self): + """Test the get_leaderboard method of the RegistrationDB class where none of the number of trades are 0.""" # setup addresses_and_handlers = [ ("address_1", "handler_1"), @@ -232,6 +232,34 @@ def test_get_leaderboard(self): assert actual_result == expected_result + def test_get_leaderboard_ii(self): + """Test the get_leaderboard method of the RegistrationDB class where some number of trades are 0.""" + # setup + addresses_and_handlers = [ + ("address_1", "handler_1"), + ("address_2", "handler_2"), + ] + trade_counts = [2, 0] + + expected_result = [("address_1", "handler_1", 2)] + + # operation + with patch.object( + self.db, + "get_all_addresses_and_handles", + return_value=addresses_and_handlers, + ) as mock_get_addresses: + with patch.object( + self.db, "get_trade_count", side_effect=trade_counts + ) as mock_trade_counts: + actual_result = self.db.get_leaderboard() + + # after + mock_get_addresses.assert_called_once() + mock_trade_counts.assert_called() + + assert actual_result == expected_result + def test_set_registered_i(self): """Test the set_registered method of the RegistrationDB class where is_registered is False.""" # operation diff --git a/tests/test_packages/test_skills/test_confirmation_aw3/test_strategy.py b/tests/test_packages/test_skills/test_confirmation_aw3/test_strategy.py index 5d02ce175f..8e7bf8f695 100644 --- a/tests/test_packages/test_skills/test_confirmation_aw3/test_strategy.py +++ b/tests/test_packages/test_skills/test_confirmation_aw3/test_strategy.py @@ -25,6 +25,8 @@ from typing import cast from unittest.mock import Mock, patch +import pytest + from packages.fetchai.protocols.http.message import HttpMessage from packages.fetchai.skills.confirmation_aw3.registration_db import RegistrationDB from packages.fetchai.skills.confirmation_aw3.strategy import ( @@ -86,7 +88,7 @@ def setup(cls): cls.counterparty = "couterparty_1" - def test__init__(self): + def test__init__i(self): """Test the __init__ of Strategy class.""" assert self.strategy.aw1_aea == self.aw1_aea assert self.strategy._locations == self.locations @@ -94,6 +96,73 @@ def test__init__(self): assert self.strategy.leaderboard_url == f"{self.leaderboard_url}/insert" assert self.strategy.leaderboard_token == self.leaderboard_token + def test__init__ii(self): + """Test the __init__ of Strategy class where aw1_aea is None.""" + with pytest.raises(ValueError, match="aw1_aea must be provided!"): + Strategy( + aw1_aea=None, + locations=self.locations, + search_queries=self.search_queries, + leaderboard_url=self.leaderboard_url, + leaderboard_token=self.leaderboard_token, + name="strategy", + skill_context=self.skill.skill_context, + ) + + def test__init__iii(self): + """Test the __init__ of Strategy class where length of locations is 0.""" + with pytest.raises(ValueError, match="locations must have at least one entry"): + Strategy( + aw1_aea="some_aw1_aea", + locations={}, + search_queries=self.search_queries, + leaderboard_url=self.leaderboard_url, + leaderboard_token=self.leaderboard_token, + name="strategy", + skill_context=self.skill.skill_context, + ) + + def test__init__iv(self): + """Test the __init__ of Strategy class where length of search_queries is 0.""" + with pytest.raises( + ValueError, match="search_queries must have at least one entry" + ): + Strategy( + aw1_aea="some_aw1_aea", + locations=self.locations, + search_queries={}, + leaderboard_url=self.leaderboard_url, + leaderboard_token=self.leaderboard_token, + name="strategy", + skill_context=self.skill.skill_context, + ) + + def test__init__v(self): + """Test the __init__ of Strategy class where leaderboard_url is None.""" + with pytest.raises(ValueError, match="No leader board url provided!"): + Strategy( + aw1_aea="some_aw1_aea", + locations=self.locations, + search_queries=self.search_queries, + leaderboard_url=None, + leaderboard_token=self.leaderboard_token, + name="strategy", + skill_context=self.skill.skill_context, + ) + + def test__init__vi(self): + """Test the __init__ of Strategy class where leaderboard_token is None.""" + with pytest.raises(ValueError, match="No leader board token provided!"): + Strategy( + aw1_aea="some_aw1_aea", + locations=self.locations, + search_queries=self.search_queries, + leaderboard_url=self.leaderboard_url, + leaderboard_token=None, + name="strategy", + skill_context=self.skill.skill_context, + ) + def test_get_acceptable_counterparties(self): """Test the get_acceptable_counterparties method of the Strategy class.""" # setup From 86816d5ab5ba9643d9c20edf712fed428180856a Mon Sep 17 00:00:00 2001 From: ali Date: Thu, 24 Dec 2020 10:08:26 +0000 Subject: [PATCH 049/204] hashes --- packages/fetchai/skills/confirmation_aw2/skill.yaml | 2 +- packages/fetchai/skills/confirmation_aw3/skill.yaml | 2 +- packages/hashes.csv | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/fetchai/skills/confirmation_aw2/skill.yaml b/packages/fetchai/skills/confirmation_aw2/skill.yaml index a3a4709ec8..d41ffc914a 100644 --- a/packages/fetchai/skills/confirmation_aw2/skill.yaml +++ b/packages/fetchai/skills/confirmation_aw2/skill.yaml @@ -12,7 +12,7 @@ fingerprint: behaviours.py: QmWRRAXwJf7mEcV4L2DdG43zkyg55PBViiLnpy1Chj28xn dialogues.py: QmcUgBjxeytE5aAx3VvPyna5EcBuqck9KazG3HygCWjawv handlers.py: QmVeJuN78rMn3kWFbntfBUk6v88DYBW88ZQZ129KZo84wp - registration_db.py: QmbdtzWvWvWizrzLcvNWphVF7LVkDq3VSWC3MkaK4RU6H5 + registration_db.py: QmdA9dBcZpdM5SbNhdMVBEVT1ameCm4TdnZmbNwjiQE1ZW strategy.py: QmafVgQyrcetTPmURWWzqqFgkU2yY2DJp2G9FWwvxcZTmW fingerprint_ignore_patterns: [] connections: diff --git a/packages/fetchai/skills/confirmation_aw3/skill.yaml b/packages/fetchai/skills/confirmation_aw3/skill.yaml index 0277a7106f..3a644ce98d 100644 --- a/packages/fetchai/skills/confirmation_aw3/skill.yaml +++ b/packages/fetchai/skills/confirmation_aw3/skill.yaml @@ -12,7 +12,7 @@ fingerprint: behaviours.py: QmeTbjNHgFJddBcxxz9LoiT8tTaod5hZvptp2etgnP9JRe dialogues.py: QmeuRrNPpbvyTAcPwAtEkTfJTDunqyfzPoDhMZWTvCyrJA handlers.py: QmXU5wYz7mkDYmtPbsunGhseGxRUBbp5N8TxAbvdeuvjDF - registration_db.py: QmZcK4tLLyRmb3QaKnYfFdSyChEQpDdEL8NeQ23f3dqYtp + registration_db.py: QmQRXDjGgo1pa5Y86Jia2FtfE2nzFtNmAXaKGhSBxM6NLM strategy.py: QmcE2Zu8XxnJaGcttWewavHzpCHH5B2spHETexXYydmYeV fingerprint_ignore_patterns: [] connections: diff --git a/packages/hashes.csv b/packages/hashes.csv index b21af2a847..6103d51d27 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -69,8 +69,8 @@ fetchai/skills/carpark_client,QmWabiH48yVv4hNaufU3jtCqe3GSEsmhMe5DEPeciTYTP3 fetchai/skills/carpark_detection,QmYKRWjC3VEU8QatkptXoGyyL3P9uR8reoDdmgLW2r318x fetchai/skills/coin_price,QmXRHQim1XGbij8QnJ4CPBksdGxmBmpbmPmHdrPAw3z3pE fetchai/skills/confirmation_aw1,QmQbqko8t8utxmfywvDqRhpE1gACVdLxrZNnv98U1aAYyR -fetchai/skills/confirmation_aw2,QmSEK1j5AhjKDKkCWgn7xrc2ySWE3HCp4NQ5zcyuEDt2dX -fetchai/skills/confirmation_aw3,QmeGfq7mHH6AjybtW85tm7wtW8PmhYu72egmtcSkeuDDiH +fetchai/skills/confirmation_aw2,Qmdmyr7DyhEtHVCuikcVzVNQZn5pV5UeAre7CvWVC1cv6J +fetchai/skills/confirmation_aw3,Qmc58zKmhcwHm6kD9U2yXoXnKx8dVRjbBWSL4Epk1Lvdoq fetchai/skills/echo,QmTir9ngs1i9dLweJs5gB8Q5NLqDufuJ5A3A2bznibKPMk fetchai/skills/erc1155_client,QmTTuStPpw2zygu9dd9cNkJnzjSVihiNHu1qACSYBT5fvL fetchai/skills/erc1155_deploy,QmX6H5MHsDKNyeKzYzhfnBENQjqLnfHaA3LvVBkinywZrc From 3fcc17b44b68607b059deaef87537cf8a5532953 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Thu, 24 Dec 2020 20:49:18 +0300 Subject: [PATCH 050/204] MAM overrides support --- aea/aea_builder.py | 58 ++--- aea/cli/config.py | 21 ++ aea/configurations/data_types.py | 7 + aea/configurations/project.py | 111 ---------- aea/manager/__init__.py | 23 ++ aea/{ => manager}/manager.py | 215 ++++++++---------- aea/manager/project.py | 251 ++++++++++++++++++++++ aea/test_tools/test_cases.py | 2 +- scripts/whitelist.py | 2 + tests/test_aea_builder.py | 12 +- tests/test_cli/test_config.py | 13 +- tests/test_configurations/test_project.py | 98 --------- tests/test_manager.py | 190 +++++++++------- 13 files changed, 532 insertions(+), 471 deletions(-) delete mode 100644 aea/configurations/project.py create mode 100644 aea/manager/__init__.py rename aea/{ => manager}/manager.py (78%) create mode 100644 aea/manager/project.py delete mode 100644 tests/test_configurations/test_project.py diff --git a/aea/aea_builder.py b/aea/aea_builder.py index 8bac447ade..3e761232f6 100644 --- a/aea/aea_builder.py +++ b/aea/aea_builder.py @@ -1005,13 +1005,13 @@ def _build_identity_from_wallet(self, wallet: Wallet) -> Identity: identity = Identity( self._name, addresses=wallet.addresses, - default_address_key=self._get_default_ledger(), + default_address_key=self.get_default_ledger(), ) else: identity = Identity( self._name, - address=wallet.addresses[self._get_default_ledger()], - default_address_key=self._get_default_ledger(), + address=wallet.addresses[self.get_default_ledger()], + default_address_key=self.get_default_ledger(), ) return identity @@ -1133,7 +1133,7 @@ def build(self, connection_ids: Optional[Collection[PublicId]] = None,) -> AEA: self._build_called = True return aea - def _get_default_ledger(self) -> str: + def get_default_ledger(self) -> str: """ Return default ledger. @@ -1373,15 +1373,18 @@ def find_component_directory_from_component_id( raise ValueError("Package {} not found.".format(component_id)) - @staticmethod - def _try_to_load_agent_configuration_file(aea_project_path: Path) -> None: + @classmethod + def try_to_load_agent_configuration_file( + cls, aea_project_path: Union[Path, str] + ) -> AgentConfig: """Try to load the agent configuration file..""" try: - configuration_file_path = Path(aea_project_path, DEFAULT_AEA_CONFIG_FILE) + aea_project_path = Path(aea_project_path) + configuration_file_path = cls.get_configuration_file_path(aea_project_path) with configuration_file_path.open(mode="r", encoding="utf-8") as fp: loader = ConfigLoader.from_configuration_type(PackageType.AGENT) agent_configuration = loader.load(fp) - logging.config.dictConfig(agent_configuration.logging_config) # type: ignore + return agent_configuration except FileNotFoundError: # pragma: nocover raise ValueError( "Agent configuration file '{}' not found in the current directory.".format( @@ -1570,45 +1573,20 @@ def from_aea_project( :return: an AEABuilder. """ aea_project_path = Path(aea_project_path) - cls._try_to_load_agent_configuration_file(aea_project_path) - verify_or_create_private_keys( - aea_project_path=aea_project_path, exit_on_error=False - ) - builder = AEABuilder(with_default_packages=False) - load_env_file(str(aea_project_path / DEFAULT_ENV_DOTFILE)) - # load agent configuration file - configuration_file = cls.get_configuration_file_path(aea_project_path) - agent_configuration = cls.loader.load(configuration_file.open()) + # load to verify + agent_configuration = cls.try_to_load_agent_configuration_file(aea_project_path) + logging.config.dictConfig(agent_configuration.logging_config) # type: ignore - builder.set_from_configuration( - agent_configuration, aea_project_path, skip_consistency_check + verify_or_create_private_keys( + aea_project_path=aea_project_path, exit_on_error=False ) - return builder - - @classmethod - def from_config_json( - cls, - json_data: List[Dict], - aea_project_path: PathLike, - skip_consistency_check: bool = False, - ) -> "AEABuilder": - """ - Load agent configuration for alreaady provided json data. - - :param json_data: list of dicts with agent configuration - :param aea_project_path: path to project root - :param skip_consistency_check: skip consistency check on configs load. - - :return: AEABuilder instance - """ - aea_project_path = Path(aea_project_path) - builder = AEABuilder(with_default_packages=False) # load agent configuration file - agent_configuration = cls.loader.load_agent_config_from_json(json_data) + agent_configuration = cls.try_to_load_agent_configuration_file(aea_project_path) + builder = AEABuilder(with_default_packages=False) builder.set_from_configuration( agent_configuration, aea_project_path, skip_consistency_check ) diff --git a/aea/cli/config.py b/aea/cli/config.py index 952092d740..409264b716 100644 --- a/aea/cli/config.py +++ b/aea/cli/config.py @@ -19,6 +19,7 @@ """Implementation of the 'aea config' subcommand.""" import contextlib import json +from copy import deepcopy from pathlib import Path from typing import Dict, List, NewType, Optional, Tuple, Union, cast @@ -342,3 +343,23 @@ def convert_value_str_to_type(value: str, type_str: str) -> JSON_TYPES: return type_(value) except (ValueError, json.decoder.JSONDecodeError): # pragma: no cover raise ValueError("Cannot convert {} to type {}".format(value, type_)) + + def get_overridables(self) -> Tuple[Dict, Dict[ComponentId, Dict]]: + """Get config overridables.""" + agent_overridable = self.agent_config.get_overridable() + + components_overridables: Dict[ComponentId, Dict] = {} + for component_id in self.agent_config.all_components_id: + obj = {} + component_config = self.load_component_configuration( + component_id, skip_consistency_check=True + ) + obj.update(component_config.get_overridable()) + obj.update( + deepcopy( + self.agent_config.component_configurations.get(component_id, {}) + ) + ) + if obj: + components_overridables[component_id] = obj + return agent_overridable, components_overridables diff --git a/aea/configurations/data_types.py b/aea/configurations/data_types.py index 56a548f7c6..7ac75957f3 100644 --- a/aea/configurations/data_types.py +++ b/aea/configurations/data_types.py @@ -598,6 +598,13 @@ def json(self) -> Dict: """Get the JSON representation.""" return dict(**self.public_id.json, type=str(self.component_type)) + @classmethod + def from_json(cls, json_data: Dict) -> "ComponentId": + """Create component id from json data.""" + return cls( + component_type=json_data["type"], public_id=PublicId.from_json(json_data) + ) + class PyPIPackageName(RegexConstrainedString): """A PyPI Package name.""" diff --git a/aea/configurations/project.py b/aea/configurations/project.py deleted file mode 100644 index 87e5cfc713..0000000000 --- a/aea/configurations/project.py +++ /dev/null @@ -1,111 +0,0 @@ -# -*- coding: utf-8 -*- -# ------------------------------------------------------------------------------ -# -# Copyright 2018-2020 Fetch.AI Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------------ -"""This module contains the implementation of AEA agents project configuiration.""" -import os -from shutil import rmtree -from typing import Any, Dict, List, Set - -from aea.aea import AEA -from aea.aea_builder import AEABuilder -from aea.cli.fetch import fetch_agent_locally -from aea.cli.registry.fetch import fetch_agent -from aea.cli.utils.context import Context -from aea.configurations.base import PublicId -from aea.configurations.constants import DEFAULT_REGISTRY_NAME - - -class Project: - """Agent project representation.""" - - def __init__(self, public_id: PublicId, path: str): - """Init project with public_id and project's path.""" - self.public_id: PublicId = public_id - self.path: str = path - self.agents: Set[str] = set() - - @classmethod - def load( - cls, - working_dir: str, - public_id: PublicId, - is_local: bool = False, - is_restore: bool = False, - registry_path: str = DEFAULT_REGISTRY_NAME, - skip_consistency_check: bool = False, - ) -> "Project": - """ - Load project with given public_id to working_dir. - - :param working_dir: the working directory - :param public_id: the public id - :param is_local: whether to fetch from local or remote - :param registry_path: the path to the registry locally - :param skip_consistency_check: consistency checks flag - """ - ctx = Context(cwd=working_dir, registry_path=registry_path) - ctx.set_config("skip_consistency_check", skip_consistency_check) - - path = os.path.join(working_dir, public_id.author, public_id.name) - target_dir = os.path.join(public_id.author, public_id.name) - - if not is_restore and not os.path.exists(target_dir): - if is_local: - ctx.set_config("is_local", True) - fetch_agent_locally(ctx, public_id, target_dir=target_dir) - else: - fetch_agent(ctx, public_id, target_dir=target_dir) - - return cls(public_id, path) - - def remove(self) -> None: - """Remove project, do cleanup.""" - rmtree(self.path) - - -class AgentAlias: - """Agent alias representation.""" - - def __init__( - self, - project: Project, - agent_name: str, - config: List[Dict], - agent: AEA, - builder: AEABuilder, - ): - """Init agent alias with project, config, name, agent, builder.""" - self.project = project - self.config = config - self.agent_name = agent_name - self.agent = agent - self.builder = builder - self.project.agents.add(self.agent_name) - - def remove_from_project(self): - """Remove agent alias from project.""" - self.project.agents.remove(self.agent_name) - - @property - def dict(self) -> Dict[str, Any]: - """Convert AgentAlias to dict.""" - return { - "public_id": str(self.project.public_id), - "agent_name": self.agent_name, - "config": self.config, - } diff --git a/aea/manager/__init__.py b/aea/manager/__init__.py new file mode 100644 index 0000000000..a49ba9a54d --- /dev/null +++ b/aea/manager/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the manager modules.""" +from aea.manager.manager import MultiAgentManager + + +__all__ = ["MultiAgentManager"] diff --git a/aea/manager.py b/aea/manager/manager.py similarity index 78% rename from aea/manager.py rename to aea/manager/manager.py index cb76e5e983..14b7ed602c 100644 --- a/aea/manager.py +++ b/aea/manager/manager.py @@ -16,33 +16,20 @@ # limitations under the License. # # ------------------------------------------------------------------------------ - """This module contains the implementation of AEA agents manager.""" import asyncio -import copy import json import os import threading from asyncio.tasks import FIRST_COMPLETED -from pathlib import Path from shutil import rmtree from threading import Thread -from typing import Any, Callable, Dict, List, Optional, Set +from typing import Any, Callable, Dict, List, Optional, Set, Tuple from aea.aea import AEA -from aea.aea_builder import AEABuilder -from aea.configurations.base import AgentConfig, ComponentId, PackageType, PublicId -from aea.configurations.constants import ( - CONNECTIONS, - CONTRACTS, - DEFAULT_LEDGER, - DEFAULT_REGISTRY_NAME, - PROTOCOLS, - SKILLS, -) -from aea.configurations.loader import ConfigLoaders -from aea.configurations.project import AgentAlias, Project -from aea.crypto.helpers import create_private_key +from aea.configurations.constants import DEFAULT_REGISTRY_NAME +from aea.configurations.data_types import PublicId +from aea.manager.project import AgentAlias, Project class AgentRunAsyncTask: @@ -135,7 +122,6 @@ def start(self) -> None: class MultiAgentManager: """Multi agents manager.""" - AGENT_DO_NOT_OVERRIDE_VALUES = [CONNECTIONS, CONTRACTS, PROTOCOLS, SKILLS] MODES = ["async", "threaded"] DEFAULT_TIMEOUT_FOR_BLOCKING_OPERATIONS = 60 SAVE_FILENAME = "save.json" @@ -197,7 +183,7 @@ def _run_thread(self) -> None: async def _manager_loop(self) -> None: """Await and control running manager.""" - if not self._event: + if not self._event: # pragma: nocover raise ValueError("Do not use this method directly, use start_manager.") self._started_event.set() @@ -318,13 +304,19 @@ def add_project( self._versionless_projects_set.add(public_id.to_any()) - self._projects[public_id] = Project.load( + project = Project.load( self.working_dir, public_id, local, registry_path=self.registry_path, is_restore=restore, ) + + if not restore: + project.install_pypi_dependencies() + project.build() + + self._projects[public_id] = project return self def remove_project( @@ -360,7 +352,6 @@ def add_agent( agent_name: Optional[str] = None, agent_overrides: Optional[dict] = None, component_overrides: Optional[List[dict]] = None, - config: Optional[List[dict]] = None, ) -> "MultiAgentManager": """ Create new agent configuration based on project with config overrides applied. @@ -375,12 +366,6 @@ def add_agent( :return: manager """ - if any((agent_overrides, component_overrides)) and config is not None: - raise ValueError( # pragma: nocover - "Can not add agent with overrides and full config." - "One of those must be used." - ) - agent_name = agent_name or public_id.name if agent_name in self._agents: @@ -391,16 +376,82 @@ def add_agent( project = self._projects[public_id] - agent_alias = self._build_agent_alias( - project=project, - agent_name=agent_name, - agent_overrides=agent_overrides, - component_overrides=component_overrides, - config=config, + agent_alias = AgentAlias( + project=project, agent_name=agent_name, keys_dir=self._keys_dir, ) + agent_alias.set_overrides(agent_overrides, component_overrides) + project.agents.add(agent_name) self._agents[agent_name] = agent_alias return self + def add_agent_with_config( + self, public_id: PublicId, config: List[dict], agent_name: Optional[str] = None, + ) -> "MultiAgentManager": + """ + Create new agent configuration based on project with config provided. + + Alias is stored in memory only! + + :param public_id: base agent project public id + :param agent_name: unique name for the agent + :param config: agent config (used for agent re-creation). + + :return: manager + """ + agent_name = agent_name or public_id.name + + if agent_name in self._agents: # pragma: nocover + raise ValueError(f"Agent with name {agent_name} already exists!") + + if public_id not in self._projects: # pragma: nocover + raise ValueError(f"{public_id} project is not added!") + + project = self._projects[public_id] + + agent_alias = AgentAlias( + project=project, agent_name=agent_name, keys_dir=self._keys_dir, + ) + agent_alias.set_agent_config_from_data(config) + project.agents.add(agent_name) + self._agents[agent_name] = agent_alias + return self + + def get_agent_overridables(self, agent_name: str) -> Tuple[Dict, List[Dict]]: + """ + Get agent config overridables. + + :param agent_name: str + + :return: Tuple of agent overridables dict and and list of component overridables dict. + """ + if agent_name not in self._agents: # pragma: nocover + raise ValueError(f"Agent with name {agent_name} does not exist!") + + return self._agents[agent_name].get_overridables() + + def set_agent_overrides( + self, + agent_name: str, + agent_overides: Optional[Dict], + components_overrides: Optional[List[Dict]], + ) -> None: + """ + Set agent overrides. + + :param agent_name: str + :param agent_overides: optional dict of agent config overrides + :param components_overrides: optional list of dict of components overrides + + :return: None + """ + if agent_name not in self._agents: # pragma: nocover + raise ValueError(f"Agent with name {agent_name} does not exist!") + + if self._is_agent_running(agent_name): # pragma: nocover + raise ValueError("Agent is running. stop it first!") + + self._agents[agent_name].set_overrides(agent_overides, components_overrides) + def list_agents_info(self) -> List[Dict[str, Any]]: """ List agents detailed info. @@ -466,9 +517,9 @@ def start_agent(self, agent_name: str) -> "MultiAgentManager": raise ValueError(f"{agent_name} is already started!") if self._mode == "async": - task = AgentRunAsyncTask(agent_alias.agent, self._loop) + task = AgentRunAsyncTask(agent_alias.get_aea_instance(), self._loop) elif self._mode == "threaded": - task = AgentRunThreadTask(agent_alias.agent, self._loop) + task = AgentRunThreadTask(agent_alias.get_aea_instance(), self._loop) task.start() self._agents_tasks[agent_name] = task @@ -594,98 +645,6 @@ def _ensure_working_dir(self) -> None: if not os.path.exists(self._keys_dir): os.makedirs(self._keys_dir) - def _build_agent_alias( - self, - project: Project, - agent_name: str, - agent_overrides: Optional[dict] = None, - component_overrides: Optional[List[dict]] = None, - config: Optional[List[dict]] = None, - ) -> AgentAlias: - """Create agent alias for project, with given name and overrided values.""" - if not config: - config = self._make_config( - project.path, agent_overrides, component_overrides - ) - - builder = AEABuilder.from_config_json(config, project.path) - builder.set_name(agent_name) - builder.set_runtime_mode("threaded") - - if not builder.private_key_paths: - default_ledger = config[0].get("default_ledger", DEFAULT_LEDGER) - builder.add_private_key( - default_ledger, self._create_private_key(agent_name, default_ledger) - ) - agent = builder.build() - return AgentAlias(project, agent_name, config, agent, builder) - - def install_pypi_dependencies(self) -> None: - """Install dependencies for every project has at least one agent alias.""" - for project in self._projects.values(): - self._install_pypi_dependencies_for_project(project) - - def _install_pypi_dependencies_for_project(self, project: Project) -> None: - """Install dependencies for project specified if has at least one agent alias.""" - if not project.agents: - return - self._install_pypi_dependencies_for_agent(list(project.agents)[0]) - - def _install_pypi_dependencies_for_agent(self, agent_name: str) -> None: - """Install dependencies for the agent registered.""" - self._agents[agent_name].builder.install_pypi_dependencies() - - def _make_config( - self, - project_path: str, - agent_overrides: Optional[dict] = None, - component_overrides: Optional[List[dict]] = None, - ) -> List[dict]: - """Make new config based on project's config with overrides applied.""" - agent_overrides = agent_overrides or {} - component_overrides = component_overrides or [] - - if any([key in agent_overrides for key in self.AGENT_DO_NOT_OVERRIDE_VALUES]): - raise ValueError( - 'Do not override any of {" ".join(self.AGENT_DO_NOT_OVERRIDE_VALUES)}' - ) - - agent_configuration_file_path: Path = AEABuilder.get_configuration_file_path( - project_path - ) - loader = ConfigLoaders.from_package_type(PackageType.AGENT) - with agent_configuration_file_path.open() as fp: - agent_config: AgentConfig = loader.load(fp) - - # prepare configuration overrides - # - agent part - agent_update_dictionary: Dict = dict(**agent_overrides) - # - components part - components_configs: Dict[ComponentId, Dict] = {} - for obj in component_overrides: - obj = copy.copy(obj) - author, name, version = ( - obj.pop("author"), - obj.pop("name"), - obj.pop("version"), - ) - component_id = ComponentId(obj.pop("type"), PublicId(author, name, version)) - components_configs[component_id] = obj - agent_update_dictionary["component_configurations"] = components_configs - # do the override (and valiation) - agent_config.update(agent_update_dictionary) - - # return the multi-paged JSON object. - json_data = agent_config.ordered_json - result: List[Dict] = [json_data] + json_data.pop("component_configurations") - return result - - def _create_private_key(self, name, ledger) -> str: - """Create new key for agent alias in working dir keys dir.""" - path = os.path.join(self._keys_dir, f"{name}_{ledger}_private.key") - create_private_key(ledger, path) - return path - def _load_state(self, local: bool) -> None: """ Load saved state from file. @@ -712,7 +671,7 @@ def _load_state(self, local: bool) -> None: ) for agent_settings in save_json["agents"]: - self.add_agent( + self.add_agent_with_config( public_id=PublicId.from_str(agent_settings["public_id"]), agent_name=agent_settings["agent_name"], config=agent_settings["config"], diff --git a/aea/manager/project.py b/aea/manager/project.py new file mode 100644 index 0000000000..cf1cec72c8 --- /dev/null +++ b/aea/manager/project.py @@ -0,0 +1,251 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2020 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the implementation of AEA agents project configuiration.""" +import os +from copy import deepcopy +from pathlib import Path +from shutil import rmtree +from typing import Any, Dict, List, Optional, Set, Tuple, Union + +from aea.aea import AEA +from aea.aea_builder import AEABuilder +from aea.cli.config import AgentConfigManager +from aea.cli.fetch import fetch_agent_locally +from aea.cli.registry.fetch import fetch_agent +from aea.cli.utils.context import Context +from aea.configurations.base import AgentConfig, PublicId +from aea.configurations.constants import DEFAULT_REGISTRY_NAME +from aea.configurations.data_types import ComponentId +from aea.crypto.helpers import create_private_key + + +class _Base: + """Base class to share some methods.""" + + @classmethod + def _get_agent_config(cls, path: Union[Path, str]) -> AgentConfig: + """Get agent config instance.""" + return AEABuilder.try_to_load_agent_configuration_file(path) + + @classmethod + def _get_builder( + cls, + agent_config: AgentConfig, + aea_project_path: Union[Path, str], + skip_consistency_check: bool = False, + ) -> AEABuilder: + """Get AEABuilder instance.""" + builder = AEABuilder(with_default_packages=False) + builder.set_from_configuration( + agent_config, Path(aea_project_path), skip_consistency_check + ) + return builder + + @property + def builder(self) -> AEABuilder: + """Get AEABuilder instance.""" + raise NotImplementedError # pragma: nocover + + def install_pypi_dependencies(self) -> None: + """Install python dependencies for the project.""" + self.builder.install_pypi_dependencies() + + def build(self) -> None: + """Call all build entry points.""" + self.builder.call_all_build_entrypoints() + + +class Project(_Base): + """Agent project representation.""" + + def __init__(self, public_id: PublicId, path: str): + """Init project with public_id and project's path.""" + self.public_id: PublicId = public_id + self.path: str = path + self.agents: Set[str] = set() + + @classmethod + def load( + cls, + working_dir: str, + public_id: PublicId, + is_local: bool = False, + is_restore: bool = False, + registry_path: str = DEFAULT_REGISTRY_NAME, + skip_consistency_check: bool = False, + ) -> "Project": + """ + Load project with given public_id to working_dir. + + :param working_dir: the working directory + :param public_id: the public id + :param is_local: whether to fetch from local or remote + :param registry_path: the path to the registry locally + :param skip_consistency_check: consistency checks flag + """ + ctx = Context(cwd=working_dir, registry_path=registry_path) + ctx.set_config("skip_consistency_check", skip_consistency_check) + + path = os.path.join(working_dir, public_id.author, public_id.name) + target_dir = os.path.join(public_id.author, public_id.name) + + if not is_restore and not os.path.exists(target_dir): + if is_local: + ctx.set_config("is_local", True) + fetch_agent_locally(ctx, public_id, target_dir=target_dir) + else: + fetch_agent(ctx, public_id, target_dir=target_dir) # pragma: nocover + + return cls(public_id, path) + + def remove(self) -> None: + """Remove project, do cleanup.""" + rmtree(self.path) + + @property + def builder(self) -> AEABuilder: + """Get builder instance.""" + return self._get_builder(self._get_agent_config(self.path), self.path) + + +class AgentAlias(_Base): + """Agent alias representation.""" + + def __init__( + self, project: Project, agent_name: str, keys_dir: Optional[str], + ): + """Init agent alias with project, config, name, agent, builder.""" + self.project = project + self.agent_name = agent_name + self._keys_dir = keys_dir or project.path + self._agent_config: AgentConfig = self._get_agent_config(project.path) + + def set_agent_config_from_data(self, json_data: List[Dict]) -> None: + """ + Set agent config instance constructed from json data. + + :param json_data: agent config json data + + :return: None + """ + self._agent_config = AEABuilder.loader.load_agent_config_from_json(json_data) + + @property + def builder(self) -> AEABuilder: + """Get builder instance.""" + builder = self._get_builder(self.agent_config, self.project.path) + if not builder.private_key_paths: + # no keys, but create one for builder only + default_ledger = builder.get_default_ledger() + builder.add_private_key( + default_ledger, self._create_private_key(default_ledger) + ) + builder.set_name(self.agent_name) + builder.set_runtime_mode("threaded") + return builder + + @property + def agent_config(self) -> AgentConfig: + """Get agent config.""" + return self._agent_config + + def _create_private_key(self, ledger: str, replace: bool = False) -> str: + """ + Create new key for agent alias in working dir keys dir. + + If file exists, check `replace` option. + """ + filepath = os.path.join( + self._keys_dir, f"{self.agent_name}_{ledger}_private.key" + ) + if os.path.exists(filepath) and not replace: + return filepath + create_private_key(ledger, filepath) + return filepath + + def remove_from_project(self): + """Remove agent alias from project.""" + self.project.agents.remove(self.agent_name) + + @property + def dict(self) -> Dict[str, Any]: + """Convert AgentAlias to dict.""" + return { + "public_id": str(self.project.public_id), + "agent_name": self.agent_name, + "config": self.config_json, + } + + @property + def config_json(self) -> List[Dict]: + """Get agent config json data.""" + json_data = self.agent_config.ordered_json + result: List[Dict] = [json_data] + json_data.pop("component_configurations", {}) + return result + + def get_aea_instance(self) -> AEA: + """Build new aea instance.""" + return self.builder.build() + + def set_overrides( + self, + agent_overrides: Optional[Dict] = None, + component_overrides: Optional[List[Dict]] = None, + ) -> None: + """Set override for this agent alias's config.""" + overrides = deepcopy(agent_overrides or {}) + component_configurations: Dict[ComponentId, Dict] = {} + + for component_override in deepcopy(component_overrides or []): + try: + component_id = ComponentId.from_json( + {"version": "any", **component_override} + ) + component_override.pop("author") + component_override.pop("name") + component_override.pop("type") + component_override.pop("version") + component_configurations[component_id] = component_override + except (ValueError, KeyError) as e: # pragma: nocover + raise ValueError( + f"Component overrides are incorrect: {e} during process: {component_override}" + ) + + overrides["component_configurations"] = component_configurations + return self.agent_config_manager.update_config(overrides) + + @property + def agent_config_manager(self) -> AgentConfigManager: + """Get agent configuration manager instance for the config.""" + return AgentConfigManager(self.agent_config, self.project.path) + + def get_overridables(self) -> Tuple[Dict, List[Dict]]: + """Get all overridables for this agent alias's config.""" + ( + agent_overridables, + components_overridables, + ) = self.agent_config_manager.get_overridables() + components_configurations = [] + for component_id, obj in components_overridables.items(): + if not obj: # pragma: nocover + continue + obj.update(component_id.json) + components_configurations.append(obj) + + return agent_overridables, components_configurations diff --git a/aea/test_tools/test_cases.py b/aea/test_tools/test_cases.py index f8903577ff..d836900d6d 100644 --- a/aea/test_tools/test_cases.py +++ b/aea/test_tools/test_cases.py @@ -175,7 +175,7 @@ def run_cli_command(cls, *args: str, cwd: str = ".") -> Result: catch_exceptions=False, ) cls.last_cli_runner_result = result - if result.exit_code != 0: + if result.exit_code != 0: # pragma: nocover raise AEATestingException( "Failed to execute AEA CLI command with args {}.\n" "Exit code: {}\nException: {}".format( diff --git a/scripts/whitelist.py b/scripts/whitelist.py index a82f4c269e..11e2ade7f0 100644 --- a/scripts/whitelist.py +++ b/scripts/whitelist.py @@ -237,3 +237,5 @@ _.decoding_error_count _.ErrorHandler # unused class (aea/error_handler/scaffold.py:27) ensure_dir # unused function (aea/helpers/base.py:561) +_.get_agent_overridables # unused method (aea/manager/manager.py:419) +_.set_agent_overrides # unused method (aea/manager/manager.py:432) diff --git a/tests/test_aea_builder.py b/tests/test_aea_builder.py index 9a2de55891..bc1d7b3992 100644 --- a/tests/test_aea_builder.py +++ b/tests/test_aea_builder.py @@ -54,7 +54,6 @@ from aea.helpers.base import cd from aea.helpers.exception_policy import ExceptionPolicyEnum from aea.helpers.install_dependency import run_install_subprocess -from aea.helpers.yaml_utils import yaml_load_all from aea.protocols.base import Protocol from aea.registries.resources import Resources from aea.skills.base import Skill @@ -558,10 +557,10 @@ def test__build_identity_from_wallet(): with pytest.raises(ValueError): builder._build_identity_from_wallet(wallet) - wallet.addresses = {builder._get_default_ledger(): "addr1"} + wallet.addresses = {builder.get_default_ledger(): "addr1"} builder._build_identity_from_wallet(wallet) - wallet.addresses = {builder._get_default_ledger(): "addr1", "fetchai": "addr2"} + wallet.addresses = {builder.get_default_ledger(): "addr1", "fetchai": "addr2"} builder._build_identity_from_wallet(wallet) @@ -668,13 +667,6 @@ def test_from_project(self): dummy_model = dummy_skill.models["dummy"] assert dummy_model.config == {"model_arg_1": 42, "model_arg_2": "2"} - def test_from_json(self): - """Test load project from json file with path specified.""" - with open(Path(self._get_cwd(), DEFAULT_AEA_CONFIG_FILE), "r") as fp: - json_config = yaml_load_all(fp) - - AEABuilder.from_config_json(json_config, Path(self._get_cwd())) - class TestFromAEAProjectMakeSkillAbstract(AEATestCase): """Test builder set from project dir, to make a skill 'abstract'.""" diff --git a/tests/test_cli/test_config.py b/tests/test_cli/test_config.py index e738ec952a..27fdf2d656 100644 --- a/tests/test_cli/test_config.py +++ b/tests/test_cli/test_config.py @@ -17,7 +17,6 @@ # # ------------------------------------------------------------------------------ """This test module contains the tests for the `aea config` sub-command.""" - import os import shutil import tempfile @@ -26,7 +25,9 @@ import pytest from click.exceptions import ClickException +from aea.aea_builder import AEABuilder from aea.cli import cli +from aea.cli.config import AgentConfigManager from aea.cli.utils.constants import ALLOWED_PATH_ROOTS from aea.configurations.base import AgentConfig, DEFAULT_AEA_CONFIG_FILE, PackageType from aea.configurations.loader import ConfigLoader @@ -697,3 +698,13 @@ def test_set_get_correct_path(self): agent_config = self.load_agent_config() assert agent_config.component_configurations + + +def test_AgentConfigManager_get_overridables(): + """Test agent config manager get_overridables.""" + path = Path(CUR_PATH, "data", "dummy_aea") + agent_config = AEABuilder.try_to_load_agent_configuration_file(path) + config_manager = AgentConfigManager(agent_config, path) + agent_overridables, component_overridables = config_manager.get_overridables() + assert "description" in agent_overridables + assert "is_abstract" in list(component_overridables.values())[0] diff --git a/tests/test_configurations/test_project.py b/tests/test_configurations/test_project.py deleted file mode 100644 index 26a6d6aa83..0000000000 --- a/tests/test_configurations/test_project.py +++ /dev/null @@ -1,98 +0,0 @@ -# -*- coding: utf-8 -*- -# ------------------------------------------------------------------------------ -# -# Copyright 2018-2019 Fetch.AI Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------------ -"""This module contains the tests for the aea configurations project.""" -import os -import shutil -import tempfile -from unittest.mock import Mock - -import pytest - -from aea.cli import cli -from aea.configurations.project import AgentAlias, Project -from aea.helpers.base import cd -from aea.test_tools.click_testing import CliRunner - -from tests.conftest import MAX_FLAKY_RERUNS, MY_FIRST_AEA_PUBLIC_ID, ROOT_DIR - - -class TestProjectAndAgentAlias: - """Check project and agent alias.""" - - def setup(self): - """Set the test up.""" - self.cwd = os.getcwd() - self.t = tempfile.mkdtemp() - os.chdir(self.t) - self.runner = CliRunner() - self.project_public_id = MY_FIRST_AEA_PUBLIC_ID - self.project_path = os.path.join( - self.t, self.project_public_id.author, self.project_public_id.name - ) - - def _test_project(self, is_local: bool, skip_consistency_check: bool): - """Test method to handle both local and remote registry.""" - registry_path = os.path.join(ROOT_DIR, "packages") - project = Project.load( - self.t, - self.project_public_id, - is_local=is_local, - registry_path=registry_path, - skip_consistency_check=skip_consistency_check, - ) - assert os.path.exists(self.project_path) - - with cd(self.project_path): - result = self.runner.invoke( - cli, - ["--skip-consistency-check", "config", "get", "agent.agent_name"], - catch_exceptions=False, - standalone_mode=False, - ) - assert self.project_public_id.name in result.output - project.remove() - assert not os.path.exists(self.project_path) - - def test_project_local(self): - """Test project loaded and removed, from local registry.""" - self._test_project(True, False) - - @pytest.mark.integration - @pytest.mark.flaky(reruns=MAX_FLAKY_RERUNS) - def test_project_remote(self): - """Test project loaded and removed, from remove registry.""" - self.project_public_id = MY_FIRST_AEA_PUBLIC_ID.to_latest() - self._test_project(False, True) - - def test_agents(self): - """Test agent added to project and rmeoved.""" - project = Project(self.project_public_id, self.project_path) - alias = AgentAlias(project, "test", [], Mock(), Mock()) - assert project.agents - alias.remove_from_project() - assert not project.agents - assert all(key in alias.dict for key in ["agent_name", "config", "public_id"]) - - def teardown(self): - """Tear dowm the test.""" - os.chdir(self.cwd) - try: - shutil.rmtree(self.t) - except (OSError, IOError): - pass diff --git a/tests/test_manager.py b/tests/test_manager.py index 6b747d8c6e..d495897b4f 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -20,6 +20,7 @@ import os from contextlib import suppress from shutil import rmtree +from unittest.case import TestCase from unittest.mock import Mock, patch import pytest @@ -33,14 +34,17 @@ from tests.conftest import MY_FIRST_AEA_PUBLIC_ID -class TestMultiAgentManagerAsyncMode: # pylint: disable=unused-argument,protected-access,attribute-defined-outside-init +@patch("aea.aea_builder.AEABuilder.install_pypi_dependencies") +class TestMultiAgentManagerAsyncMode( + TestCase +): # pylint: disable=unused-argument,protected-access,attribute-defined-outside-init """Tests for MultiAgentManager in async mode.""" MODE = "async" echo_skill_id = ECHO_SKILL_PUBLIC_ID - def setup(self): + def setUp(self): """Set test case.""" self.agent_name = "test_what_ever12" self.working_dir = "MultiAgentManager_dir" @@ -51,11 +55,13 @@ def setup(self): assert not os.path.exists(self.working_dir) self.manager = MultiAgentManager(self.working_dir, mode=self.MODE) - def teardown(self): + def tearDown(self): """Tear down test case.""" self.manager.stop_manager() + if os.path.exists(self.working_dir): + rmtree(self.working_dir) - def test_workdir_created_removed(self): + def test_workdir_created_removed(self, *args): """Check work dit created removed on MultiAgentManager start and stop.""" assert not os.path.exists(self.working_dir) self.manager.start_manager() @@ -64,7 +70,7 @@ def test_workdir_created_removed(self): assert not os.path.exists(self.working_dir) assert not os.path.exists(self.working_dir) - def test_keys_dir_presents(self): + def test_keys_dir_presents(self, *args): """Check not fails on exists key dir.""" try: os.makedirs(self.working_dir) @@ -75,7 +81,7 @@ def test_keys_dir_presents(self): with suppress(Exception): rmtree(self.working_dir) - def test_MultiAgentManager_is_running(self): + def test_MultiAgentManager_is_running(self, *args): """Check MultiAgentManager is running property reflects state.""" assert not self.manager.is_running self.manager.start_manager() @@ -83,7 +89,7 @@ def test_MultiAgentManager_is_running(self): self.manager.stop_manager() assert not self.manager.is_running - def test_add_remove_project(self): + def test_add_remove_project(self, *args): """Test add and remove project.""" self.manager.start_manager() @@ -105,7 +111,7 @@ def test_add_remove_project(self): assert self.project_public_id in self.manager.list_projects() assert os.path.exists(self.project_path) - def test_add_agent(self): + def test_add_agent(self, *args): """Test add agent alias.""" self.manager.start_manager() @@ -113,33 +119,58 @@ def test_add_agent(self): new_tick_interval = 0.2111 + component_overrides = [ + { + **self.echo_skill_id.json, + "type": "skill", + "behaviours": {"echo": {"args": {"tick_interval": new_tick_interval}}}, + } + ] self.manager.add_agent( self.project_public_id, self.agent_name, - component_overrides=[ - { - **self.echo_skill_id.json, - "type": "skill", - "behaviours": { - "echo": {"args": {"tick_interval": new_tick_interval}} - }, - } - ], + component_overrides=component_overrides, ) agent_alias = self.manager.get_agent_alias(self.agent_name) assert agent_alias.agent_name == self.agent_name assert ( - agent_alias.agent.resources.get_behaviour( - self.echo_skill_id, "echo" - ).tick_interval + agent_alias.get_aea_instance() + .resources.get_behaviour(self.echo_skill_id, "echo") + .tick_interval == new_tick_interval ) + with pytest.raises(ValueError, match="already exists"): self.manager.add_agent( self.project_public_id, self.agent_name, ) - def test_remove_agent(self): + def test_set_overrides(self, *args): + """Test agent set overrides.""" + self.test_add_agent() + new_tick_interval = 1000.0 + component_overrides = [ + { + **self.echo_skill_id.json, + "type": "skill", + "behaviours": {"echo": {"args": {"tick_interval": new_tick_interval}}}, + } + ] + self.manager.set_agent_overrides( + self.agent_name, + agent_overides=None, + components_overrides=component_overrides, + ) + agent_alias = self.manager.get_agent_alias(self.agent_name) + assert agent_alias.agent_name == self.agent_name + assert ( + agent_alias.get_aea_instance() + .resources.get_behaviour(self.echo_skill_id, "echo") + .tick_interval + == new_tick_interval + ) + + def test_remove_agent(self, *args): """Test remove agent alias.""" self.test_add_agent() assert self.agent_name in self.manager.list_agents() @@ -149,7 +180,7 @@ def test_remove_agent(self): with pytest.raises(ValueError, match="does not exist!"): self.manager.remove_agent(self.agent_name) - def test_remove_project_with_alias(self): + def test_remove_project_with_alias(self, *args): """Test remove project with alias presents.""" self.test_add_agent() @@ -158,46 +189,45 @@ def test_remove_project_with_alias(self): ): self.manager.remove_project(self.project_public_id) - def test_add_agent_for_non_exist_project(self): + def test_add_agent_for_non_exist_project(self, *args): """Test add agent when no project added.""" with pytest.raises(ValueError, match=" project is not added"): self.manager.add_agent(PublicId("test", "test", "0.1.0"), "another_agent") - def test_agent_actually_running(self): + def test_agent_actually_running(self, *args): """Test MultiAgentManager starts agent correctly and agent perform acts.""" self.test_add_agent() - agent_alias = self.manager.get_agent_alias(self.agent_name) - behaviour = agent_alias.agent.resources.get_behaviour( - self.echo_skill_id, "echo" - ) + + self.manager.start_all_agents() + agent = self.manager._agents_tasks[self.agent_name].agent + behaviour = agent.resources.get_behaviour(self.echo_skill_id, "echo") assert behaviour + with patch.object(behaviour, "act") as act_mock: - self.manager.start_all_agents() wait_for_condition(lambda: act_mock.call_count > 0, timeout=10) - def test_exception_handling(self): + def test_exception_handling(self, *args): """Test erro callback works.""" self.test_add_agent() - agent_alias = self.manager.get_agent_alias(self.agent_name) - behaviour = agent_alias.agent.resources.get_behaviour( - self.echo_skill_id, "echo" - ) + self.manager.start_all_agents() + agent = self.manager._agents_tasks[self.agent_name].agent + behaviour = agent.resources.get_behaviour(self.echo_skill_id, "echo") + assert behaviour + callback_mock = Mock() self.manager.add_error_callback(callback_mock) - assert behaviour with patch.object(behaviour, "act", side_effect=ValueError("expected")): self.manager.start_all_agents() wait_for_condition(lambda: callback_mock.call_count > 0, timeout=10) - def test_stop_from_exception_handling(self): - """Test stop MultiAgentManager from erro callback.""" + def test_stop_from_exception_handling(self, *args): + """Test stop MultiAgentManager from error callback.""" self.test_add_agent() - agent_alias = self.manager.get_agent_alias(self.agent_name) - behaviour = agent_alias.agent.resources.get_behaviour( - self.echo_skill_id, "echo" - ) + self.manager.start_all_agents() + agent = self.manager._agents_tasks[self.agent_name].agent + behaviour = agent.resources.get_behaviour(self.echo_skill_id, "echo") def handler(*args, **kwargs): self.manager.stop_manager() @@ -210,7 +240,7 @@ def handler(*args, **kwargs): self.manager.start_all_agents() wait_for_condition(lambda: not self.manager.is_running, timeout=10) - def test_start_all(self): + def test_start_all(self, *args): """Test MultiAgentManager start all agents.""" self.test_add_agent() assert self.agent_name in self.manager.list_agents() @@ -229,7 +259,7 @@ def test_start_all(self): with pytest.raises(ValueError, match="is not registered!"): self.manager.start_agent("non_exists_agent") - def test_stop_agent(self): + def test_stop_agent(self, *args): """Test stop agent.""" self.test_start_all() wait_for_condition( @@ -245,16 +275,24 @@ def test_stop_agent(self): with pytest.raises(ValueError, match=" is not running!"): self.manager.stop_agents([self.agent_name]) - def test_do_no_allow_override_some_fields(self): + def test_do_no_allow_override_some_fields(self, *args): """Do not allo to override some values in agent config.""" self.manager.start_manager() self.manager.add_project(self.project_public_id, local=True) - BAD_OVERRIDES = ["skills", "connections", "contracts", "protocols"] + BAD_OVERRIDES = [ + "skills", + "connections", + "contracts", + "protocols", + "some_field?", + ] for bad_override in BAD_OVERRIDES: - with pytest.raises(ValueError, match="Do not override any"): + with pytest.raises( + ValueError, match=r"Attribute `.*` is not allowed to be updated!" + ): self.manager.add_agent( self.project_public_id, self.agent_name, @@ -262,19 +300,19 @@ def test_do_no_allow_override_some_fields(self): ) @staticmethod - def test_invalid_mode(): + def test_invalid_mode(*args): """Test MultiAgentManager fails on invalid mode.""" with pytest.raises(ValueError, match="Invalid mode"): MultiAgentManager("test_dir", mode="invalid_mode") - def test_double_start(self): + def test_double_start(self, *args): """Test double MultiAgentManager start.""" self.manager.start_manager() assert self.manager.is_running self.manager.start_manager() assert self.manager.is_running - def test_double_stop(self): + def test_double_stop(self, *args): """Test double MultiAgentManager stop.""" self.manager.start_manager() assert self.manager.is_running @@ -284,14 +322,14 @@ def test_double_stop(self): assert not self.manager.is_running @pytest.mark.asyncio - async def test_run_loop_direct_call(self): + async def test_run_loop_direct_call(self, *args): """Test do not allow to run MultiAgentManager_loop directly.""" with pytest.raises( ValueError, match="Do not use this method directly, use start_manager" ): await self.manager._manager_loop() - def test_remove_running_agent(self): + def test_remove_running_agent(self, *args): """Test fail on remove running agent.""" self.test_start_all() with pytest.raises(ValueError, match="Agent is running. stop it first!"): @@ -305,35 +343,7 @@ def test_remove_running_agent(self): self.manager.remove_agent(self.agent_name) assert self.agent_name not in self.manager.list_agents() - def test_install_pypi_dependencies(self): - """Test add agent alias.""" - self.manager.start_manager() - - self.manager.add_project(self.project_public_id, local=True) - - # check empty project, nothing should be installed - with patch( - "aea.aea_builder.AEABuilder.install_pypi_dependencies", return_value=None - ) as install_mock: - self.manager.install_pypi_dependencies() - install_mock.assert_not_called() - - self.manager.add_agent( - self.project_public_id, self.agent_name, - ) - - self.manager.add_agent( - self.project_public_id, self.agent_name + "2222", - ) - - # check project with two agents, install once! - with patch( - "aea.aea_builder.AEABuilder.install_pypi_dependencies", return_value=None - ) as install_mock: - self.manager.install_pypi_dependencies() - install_mock.assert_called_once() - - def test_save_load_positive(self): + def test_save_load_positive(self, *args): """Test save-load func of MultiAgentManager for positive result.""" self.manager.start_manager() self.manager.add_project(self.project_public_id, local=True) @@ -346,7 +356,7 @@ def test_save_load_positive(self): assert self.project_public_id in self.manager._projects.keys() assert self.agent_name in self.manager._agents.keys() - def test_list_agents_info_positive(self): + def test_list_agents_info_positive(self, *args): """Test list_agents_info method for positive result.""" self.manager.start_manager() self.manager.add_project(self.project_public_id, local=True) @@ -362,7 +372,7 @@ def test_list_agents_info_positive(self): ] assert result == expected_result - def test_add_same_project_versions(self): + def test_add_same_project_versions(self, *args): """Test add the same project twice.""" self.manager.start_manager() @@ -374,6 +384,22 @@ def test_add_same_project_versions(self): PublicId.from_str("fetchai/my_first_aea:0.15.0"), local=False ) + def test_get_overridables(self, *args): + """Test get overridables.""" + self.manager.start_manager() + self.manager.add_project(self.project_public_id, local=True) + self.manager.add_agent(self.project_public_id, self.agent_name) + + ( + agent_overridables, + components_overridables, + ) = self.manager.get_agent_overridables(self.agent_name) + assert "default_ledger" in agent_overridables + assert "timeout" in agent_overridables + assert "description" in agent_overridables + assert len(components_overridables) == 3 + assert "is_abstract" in components_overridables[0] + class TestMultiAgentManagerThreadedMode(TestMultiAgentManagerAsyncMode): """Tests for MultiAgentManager in threaded mode.""" From 89e8e46fbb1439de6a97b0ee8653c7be67448dad Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Mon, 28 Dec 2020 13:36:58 +0300 Subject: [PATCH 051/204] build dir fix for multi agent manager --- aea/aea_builder.py | 7 +++++-- aea/manager/manager.py | 9 +++++++-- aea/manager/project.py | 8 ++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/aea/aea_builder.py b/aea/aea_builder.py index 3e761232f6..b1f9e52063 100644 --- a/aea/aea_builder.py +++ b/aea/aea_builder.py @@ -897,15 +897,18 @@ def remove_contract(self, public_id: PublicId) -> "AEABuilder": self.remove_component(ComponentId(ComponentType.CONTRACT, public_id)) return self - def call_all_build_entrypoints(self): + def call_all_build_entrypoints(self, root_dir: str = "."): """Call all the build entrypoints.""" for config in self._package_dependency_manager._dependencies.values(): # type: ignore # pylint: disable=protected-access self.run_build_for_component_configuration(config, logger=self.logger) + target_directory = os.path.abspath( + os.path.join(root_dir, self.AEA_CLASS.get_build_dir()) + ) + if self._build_entrypoint: self.logger.info("Building AEA package...") source_directory = "." - target_directory = os.path.abspath(self.AEA_CLASS.get_build_dir()) build_entrypoint = cast(str, self._build_entrypoint) self._run_build_entrypoint( build_entrypoint, source_directory, target_directory, logger=self.logger diff --git a/aea/manager/manager.py b/aea/manager/manager.py index 14b7ed602c..d99c0c4c46 100644 --- a/aea/manager/manager.py +++ b/aea/manager/manager.py @@ -516,10 +516,15 @@ def start_agent(self, agent_name: str) -> "MultiAgentManager": if self._is_agent_running(agent_name): raise ValueError(f"{agent_name} is already started!") + aea = agent_alias.get_aea_instance() + # override build dir to project's one + aea.DEFAULT_BUILD_DIR_NAME = os.path.join( + agent_alias.project.path, aea.DEFAULT_BUILD_DIR_NAME + ) if self._mode == "async": - task = AgentRunAsyncTask(agent_alias.get_aea_instance(), self._loop) + task = AgentRunAsyncTask(aea, self._loop) elif self._mode == "threaded": - task = AgentRunThreadTask(agent_alias.get_aea_instance(), self._loop) + task = AgentRunThreadTask(aea, self._loop) task.start() self._agents_tasks[agent_name] = task diff --git a/aea/manager/project.py b/aea/manager/project.py index cf1cec72c8..386539ea2c 100644 --- a/aea/manager/project.py +++ b/aea/manager/project.py @@ -66,10 +66,6 @@ def install_pypi_dependencies(self) -> None: """Install python dependencies for the project.""" self.builder.install_pypi_dependencies() - def build(self) -> None: - """Call all build entry points.""" - self.builder.call_all_build_entrypoints() - class Project(_Base): """Agent project representation.""" @@ -80,6 +76,10 @@ def __init__(self, public_id: PublicId, path: str): self.path: str = path self.agents: Set[str] = set() + def build(self) -> None: + """Call all build entry points.""" + self.builder.call_all_build_entrypoints(self.path) + @classmethod def load( cls, From a32a068ef28dc873561ca80bff24dc5c29056103 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Mon, 28 Dec 2020 10:54:39 +0000 Subject: [PATCH 052/204] fix weather skill demo guide, fix aea build and remove-key commands --- aea/aea_builder.py | 13 +++++++---- aea/cli/build.py | 4 +++- aea/cli/remove_key.py | 1 + docs/weather-skills.md | 49 ++++++++++++++++++++++++++---------------- 4 files changed, 44 insertions(+), 23 deletions(-) diff --git a/aea/aea_builder.py b/aea/aea_builder.py index 340fdabb03..1ee898d60b 100644 --- a/aea/aea_builder.py +++ b/aea/aea_builder.py @@ -1553,7 +1553,10 @@ def _find_import_order( @classmethod def from_aea_project( - cls, aea_project_path: PathLike, skip_consistency_check: bool = False + cls, + aea_project_path: PathLike, + skip_consistency_check: bool = False, + verify_or_create_keys: bool = True, ) -> "AEABuilder": """ Construct the builder from an AEA project. @@ -1567,13 +1570,15 @@ def from_aea_project( :param aea_project_path: path to the AEA project. :param skip_consistency_check: if True, the consistency check are skipped. + :param verify_or_create_keys: if True, verify_or_create_keys :return: an AEABuilder. """ aea_project_path = Path(aea_project_path) cls._try_to_load_agent_configuration_file(aea_project_path) - verify_or_create_private_keys( - aea_project_path=aea_project_path, exit_on_error=False - ) + if verify_or_create_keys: + verify_or_create_private_keys( + aea_project_path=aea_project_path, exit_on_error=False + ) builder = AEABuilder(with_default_packages=False) load_env_file(str(aea_project_path / DEFAULT_ENV_DOTFILE)) diff --git a/aea/cli/build.py b/aea/cli/build.py index bb06c0154c..0ad0cbb55d 100644 --- a/aea/cli/build.py +++ b/aea/cli/build.py @@ -48,7 +48,9 @@ def build_aea(skip_consistency_check: bool) -> None: """ try: builder = AEABuilder.from_aea_project( - Path("."), skip_consistency_check=skip_consistency_check + Path("."), + skip_consistency_check=skip_consistency_check, + verify_or_create_keys=False, ) builder.call_all_build_entrypoints() except Exception as e: diff --git a/aea/cli/remove_key.py b/aea/cli/remove_key.py index d2c6ffbd43..d203d0d485 100644 --- a/aea/cli/remove_key.py +++ b/aea/cli/remove_key.py @@ -74,6 +74,7 @@ def _try_remove_key(ctx: Context, type_: str, connection: bool = False): raise click.ClickException( f"There is no {'connection ' if connection else ''}key registered with id {type_}." ) + private_keys.delete(type_) ctx.agent_loader.dump( ctx.agent_config, open(os.path.join(ctx.cwd, DEFAULT_AEA_CONFIG_FILE), "w") ) diff --git a/docs/weather-skills.md b/docs/weather-skills.md index dd58636665..59312da126 100644 --- a/docs/weather-skills.md +++ b/docs/weather-skills.md @@ -81,13 +81,11 @@ aea add skill fetchai/weather_station:0.17.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 -``` - -In `weather_station/aea-config.yaml` add -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ```

@@ -118,13 +116,11 @@ aea add skill fetchai/weather_client:0.17.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 -``` - -In `my_weather_client/aea-config.yaml` add -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ```

@@ -166,20 +162,37 @@ First, run the weather station AEA: aea run ``` -Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) This is the entry peer address for the local agent communication network created by the weather station. -Then, update the configuration of the weather client AEA's p2p connection (in `vendor/fetchai/connections/p2p_libp2p/connection.yaml`) replace the following: + +Then, in the weather client, run this command (replace `SOME_ADDRESS` with the correct value as described above): +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +This allows the weather client to connect to the same local agent communication network as the weather station. Then run the weather client AEA: ``` bash From 6f0031a74501f83665def7bb9f0ec20efc736c18 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Mon, 28 Dec 2020 10:55:23 +0000 Subject: [PATCH 053/204] fix docs tests weather skill --- .../md_files/bash-weather-skills.md | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-weather-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-weather-skills.md index 14059dabab..650877afde 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-weather-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-weather-skills.md @@ -14,6 +14,11 @@ aea add skill fetchai/weather_station:0.17.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` ``` bash aea fetch fetchai/weather_client:0.20.0 --alias my_weather_client @@ -31,6 +36,11 @@ aea add skill fetchai/weather_client:0.17.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` ``` bash aea generate-key fetchai @@ -49,6 +59,16 @@ aea generate-wealth fetchai aea run ``` ``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +``` bash aea run ``` ``` bash @@ -57,19 +77,13 @@ aea delete my_weather_station aea delete my_weather_client ``` ``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml +--- +public_id: fetchai/p2p_libp2p:0.13.0 +type: connection config: delegate_uri: 127.0.0.1:11001 - entry_peers: ['SOME_ADDRESS'] + entry_peers: + - SOME_ADDRESS local_uri: 127.0.0.1:9001 log_file: libp2p_node.log public_uri: 127.0.0.1:9001 From 36a86e854a76c13c76a17b3e08108f07c1dedbad Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Mon, 28 Dec 2020 11:07:26 +0000 Subject: [PATCH 054/204] fix thermometer skills demo --- docs/thermometer-skills.md | 59 +++++++++++-------- .../md_files/bash-thermometer-skills.md | 36 +++++++---- 2 files changed, 59 insertions(+), 36 deletions(-) diff --git a/docs/thermometer-skills.md b/docs/thermometer-skills.md index 8d03d4a719..c85449aba6 100644 --- a/docs/thermometer-skills.md +++ b/docs/thermometer-skills.md @@ -5,14 +5,10 @@ The AEA thermometer skills demonstrate an interaction between two AEAs. ## Discussion -The scope of the specific demo is to demonstrate how to create a very simple AEA with the usage of the AEA framework, a Raspberry Pi, and a thermometer sensor. The thermometer AEA -will read data from the sensor each time a client requests and will deliver to the client upon payment. To keep the demo simple we avoided the usage of a database since this would increase the complexity. As a result, the AEA can provide only one reading from the sensor. -This demo does not utilise a smart contract. As a result, we interact with a ledger only to complete a transaction. - -Since the AEA framework enables us to use third-party libraries hosted on PyPI we can directly reference the external dependencies. -The `aea install` command will install each dependency that the specific AEA needs and is listed in the skill's YAML file. -The AEA must run inside a Raspberry Pi or any other Linux system, and the sensor must be connected to the USB port. +The scope of the specific demo is to demonstrate how to create a very simple AEA with the usage of the AEA framework and a thermometer sensor. The thermometer AEA will read data from the sensor each time a client requests and will deliver to the client upon payment. To keep the demo simple we avoided the usage of a database since this would increase the complexity. As a result, the AEA can provide only one reading from the sensor. This demo does not utilise a smart contract. As a result, we interact with a ledger only to complete a transaction. + ## Communication This diagram shows the communication between the various entities as data is successfully sold by the thermometer AEA to the client. @@ -82,13 +78,11 @@ aea add skill fetchai/thermometer:0.17.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 -``` - -In `my_thermometer_aea/aea-config.yaml` add -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ```

@@ -118,13 +112,11 @@ aea add skill fetchai/thermometer_client:0.17.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 -``` - -In `my_thermometer_aea/aea-config.yaml` add -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ```

@@ -165,20 +157,37 @@ First, run the thermometer AEA: aea run ``` -Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) This is the entry peer address for the local agent communication network created by the thermometer AEA. -Then, update the configuration of the thermometer client AEA's p2p connection (in `vendor/fetchai/connections/p2p_libp2p/connection.yaml`) replace the following: + + +Then, in the thermometer client, run this command (replace `SOME_ADDRESS` with the correct value as described above): +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +This allows the thermometer client to connect to the same local agent communication network as the thermometer AEA. Then run the thermometer client AEA: ``` bash diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-thermometer-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-thermometer-skills.md index a8facf8281..19971a3708 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-thermometer-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-thermometer-skills.md @@ -14,6 +14,11 @@ aea add skill fetchai/thermometer:0.17.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` ``` bash aea fetch fetchai/thermometer_client:0.18.0 --alias my_thermometer_client @@ -31,6 +36,11 @@ aea add skill fetchai/thermometer_client:0.17.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` ``` bash aea generate-key fetchai @@ -49,6 +59,16 @@ aea generate-wealth fetchai aea run ``` ``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +``` bash aea run ``` ``` bash @@ -57,19 +77,13 @@ aea delete my_thermometer_aea aea delete my_thermometer_client ``` ``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml +--- +public_id: fetchai/p2p_libp2p:0.13.0 +type: connection config: delegate_uri: 127.0.0.1:11001 - entry_peers: ['SOME_ADDRESS'] + entry_peers: + - SOME_ADDRESS local_uri: 127.0.0.1:9001 log_file: libp2p_node.log public_uri: 127.0.0.1:9001 From 6a9bd722f5333ffdb99b3a3640cf8a64f2875608 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 28 Dec 2020 14:29:22 +0100 Subject: [PATCH 055/204] update car park skill docs --- docs/car-park-skills.md | 35 ++++++++++++++----- .../md_files/bash-car-park-skills.md | 21 ++++++++++- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/docs/car-park-skills.md b/docs/car-park-skills.md index 5d28eb1ff4..f1946f795f 100644 --- a/docs/car-park-skills.md +++ b/docs/car-park-skills.md @@ -75,13 +75,11 @@ aea add skill fetchai/carpark_detection:0.17.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 -``` - -In `car_detector/aea-config.yaml` add -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ```

@@ -159,19 +157,38 @@ aea run ``` Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) +This is the entry peer address for the local agent communication network created by the car data seller. -Then, update the configuration of the car data buyer AEA's p2p connection (in `vendor/fetchai/connections/p2p_libp2p/connection.yaml`) replace the following: + + +Then, in the car data buyer, run this command (replace `SOME_ADDRESS` with the correct value as described above): +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +This allows the car data buyer to connect to the same local agent communication network as the car data seller. Then run the buyer AEA: ``` bash diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md index b6e261abda..1b5372bb47 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md @@ -14,6 +14,11 @@ aea add skill fetchai/carpark_detection:0.17.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` ``` bash aea fetch fetchai/car_data_buyer:0.20.0 @@ -49,6 +54,16 @@ aea generate-wealth fetchai aea run ``` ``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +``` bash aea run ``` ``` bash @@ -67,9 +82,13 @@ default_routing: fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 ``` ``` yaml +--- +public_id: fetchai/p2p_libp2p:0.13.0 +type: connection config: delegate_uri: 127.0.0.1:11001 - entry_peers: ['SOME_ADDRESS'] + entry_peers: + - SOME_ADDRESS local_uri: 127.0.0.1:9001 log_file: libp2p_node.log public_uri: 127.0.0.1:9001 From 991616e9d203ae5cd9d9e38dc0264bbbefae30c1 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 28 Dec 2020 14:43:29 +0100 Subject: [PATCH 056/204] update ml skill docs --- docs/car-park-skills.md | 15 +++--- docs/ml-skills.md | 49 +++++++++++++------ .../md_files/bash-car-park-skills.md | 5 ++ .../test_bash_yaml/md_files/bash-ml-skills.md | 26 +++++++++- 4 files changed, 70 insertions(+), 25 deletions(-) diff --git a/docs/car-park-skills.md b/docs/car-park-skills.md index f1946f795f..3df566a160 100644 --- a/docs/car-park-skills.md +++ b/docs/car-park-skills.md @@ -109,14 +109,13 @@ aea add skill fetchai/carpark_client:0.18.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` -In `car_data_buyer/aea-config.yaml` add -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -```

@@ -159,7 +158,7 @@ aea run Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) This is the entry peer address for the local agent communication network created by the car data seller. - Then, in the car data buyer, run this command (replace `SOME_ADDRESS` with the correct value as described above): diff --git a/docs/ml-skills.md b/docs/ml-skills.md index 7936d7d8ec..d4bb109a45 100644 --- a/docs/ml-skills.md +++ b/docs/ml-skills.md @@ -80,17 +80,15 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/ml_data_provider:0.17.0 aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' aea install aea build ``` -In `ml_data_provider/aea-config.yaml` add -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -

@@ -116,16 +114,15 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/ml_train:0.18.0 aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' aea install aea build ``` -In `ml_model_trainer/aea-config.yaml` add -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -```

@@ -166,19 +163,39 @@ aea run ``` Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) +This is the entry peer address for the local agent communication network created by the ML data provider. -Then, update the configuration of the model trainer AEA's p2p connection (in `vendor/fetchai/connections/p2p_libp2p/connection.yaml`) replace the following: + +Then, in the ML model trainer, run this command (replace `SOME_ADDRESS` with the correct value as described above): +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +This allows the model trainer to connect to the same local agent communication network as the data provider. + Then run the model trainer AEA: ``` bash diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md index 1b5372bb47..3d25dba57c 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md @@ -36,6 +36,11 @@ aea add skill fetchai/carpark_client:0.18.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` ``` bash aea generate-key fetchai diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md index 5ac88ca8f5..465f527eb2 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md @@ -12,6 +12,11 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/ml_data_provider:0.17.0 aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' aea install aea build ``` @@ -29,6 +34,11 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/ml_train:0.18.0 aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' aea install aea build ``` @@ -49,6 +59,16 @@ aea generate-wealth fetchai aea run ``` ``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +``` bash aea run ``` ``` bash @@ -67,9 +87,13 @@ default_routing: fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 ``` ``` yaml +--- +public_id: fetchai/p2p_libp2p:0.13.0 +type: connection config: delegate_uri: 127.0.0.1:11001 - entry_peers: ['SOME_ADDRESS'] + entry_peers: + - SOME_ADDRESS local_uri: 127.0.0.1:9001 log_file: libp2p_node.log public_uri: 127.0.0.1:9001 From 1809336ceb50a5aeee2bbcfaddd15851de713dd9 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 28 Dec 2020 19:28:23 +0100 Subject: [PATCH 057/204] try to fix some issues in ML skills --- .../fetchai/protocols/ml_trade/dialogues.py | 18 ++- .../fetchai/protocols/ml_trade/protocol.yaml | 2 +- .../skills/ml_data_provider/skill.yaml | 2 +- .../skills/ml_data_provider/strategy.py | 41 +++++ .../fetchai/skills/ml_train/behaviours.py | 141 +++++++++++++++++- packages/fetchai/skills/ml_train/handlers.py | 4 +- packages/fetchai/skills/ml_train/skill.yaml | 6 +- packages/fetchai/skills/ml_train/strategy.py | 27 ++++ packages/hashes.csv | 6 +- 9 files changed, 231 insertions(+), 16 deletions(-) diff --git a/packages/fetchai/protocols/ml_trade/dialogues.py b/packages/fetchai/protocols/ml_trade/dialogues.py index 3f89804c13..0d093869fb 100644 --- a/packages/fetchai/protocols/ml_trade/dialogues.py +++ b/packages/fetchai/protocols/ml_trade/dialogues.py @@ -25,9 +25,11 @@ """ from abc import ABC -from typing import Callable, FrozenSet, Type, cast +from typing import Callable, FrozenSet, Optional, Type, cast from aea.common import Address +from aea.exceptions import AEAEnforceError, enforce +from aea.helpers.transaction.base import Terms from aea.protocols.base import Message from aea.protocols.dialogue.base import Dialogue, DialogueLabel, Dialogues @@ -83,6 +85,20 @@ def __init__( self_address=self_address, role=role, ) + self._terms = None # type: Optional[Terms] + + @property + def terms(self) -> Terms: + """Get terms.""" + if self._terms is None: + raise AEAEnforceError("Terms not set!") + return self._terms + + @terms.setter + def terms(self, terms: Terms) -> None: + """Set terms.""" + enforce(self._terms is None, "Terms already set!") + self._terms = terms class MlTradeDialogues(Dialogues, ABC): diff --git a/packages/fetchai/protocols/ml_trade/protocol.yaml b/packages/fetchai/protocols/ml_trade/protocol.yaml index 95a98e39b2..087959bc96 100644 --- a/packages/fetchai/protocols/ml_trade/protocol.yaml +++ b/packages/fetchai/protocols/ml_trade/protocol.yaml @@ -9,7 +9,7 @@ fingerprint: README.md: QmccSUTpVzDpaAqpGPGXWY5aG9Dp1yStX4Sn2tqCjsKVEC __init__.py: QmcCS9uUQTTS2w85dTNiN5rQ14wyBhmBkr7pPPPcbLphcn custom_types.py: QmPa6mxbN8WShsniQxJACfzAPRjGzYLbUFGoVU4N9DewUw - dialogues.py: Qmbras4sV8Fz7cnXwXMJLM5G4Y6rvSWvXakfTjDbyddbY5 + dialogues.py: QmUUV9HXZojbFvw9mm58E2PDUETjbSfueSH8fCaysuaqPo message.py: QmYXwvWBCQkfPKGB8cbNPeomxS248K4CvgW8djNuXCq7Qa ml_trade.proto: QmNvSW9EUYiyu5SLhCVVaAdzLi1yjM94mnUeNSBzu2GRYH ml_trade_pb2.py: QmcrA4D7C9LFPcj6QKe8uUxYdHfHpRmZ5Dm4U5dKpT9nPh diff --git a/packages/fetchai/skills/ml_data_provider/skill.yaml b/packages/fetchai/skills/ml_data_provider/skill.yaml index 3b2b8add61..a8442a9382 100644 --- a/packages/fetchai/skills/ml_data_provider/skill.yaml +++ b/packages/fetchai/skills/ml_data_provider/skill.yaml @@ -12,7 +12,7 @@ fingerprint: behaviours.py: QmWgXU9qgahXwMKNqLLfDiGNYJozSXv2SVMkoPDQncC7ok dialogues.py: QmVwLCVudrwmJ33t5npXFDfRp8UYK7QFH4djanZgzziPTw handlers.py: QmWCVSnLMSLLPB9koJVc7g12YsfW3eCgkMET8TTpwDDW69 - strategy.py: QmXsAvSWe97U5Hv7ZfdTH7pfbYnx2AEyhfgAWP8qk1HnCh + strategy.py: QmRro9Lztj4AwUPcBFWsQV7CWVjUmZ12Xp4Q4rsvPfbyKL fingerprint_ignore_patterns: [] connections: [] contracts: [] diff --git a/packages/fetchai/skills/ml_data_provider/strategy.py b/packages/fetchai/skills/ml_data_provider/strategy.py index b6b24853db..75901b6c08 100644 --- a/packages/fetchai/skills/ml_data_provider/strategy.py +++ b/packages/fetchai/skills/ml_data_provider/strategy.py @@ -25,6 +25,7 @@ from aea.exceptions import enforce from aea.helpers.search.generic import ( AGENT_LOCATION_MODEL, + AGENT_PERSONALITY_MODEL, AGENT_REMOVE_SERVICE_MODEL, AGENT_SET_SERVICE_MODEL, SIMPLE_DATA_MODEL, @@ -39,7 +40,9 @@ DEFAULT_BUYER_TX_FEE = 0 DEFAULT_LOCATION = {"longitude": 0.1270, "latitude": 51.5194} +DEFAULT_PERSONALITY_DATA = {"piece": "genus", "value": "data"} DEFAULT_SERVICE_DATA = {"key": "dataset_id", "value": "fmnist"} +DEFAULT_CLASSIFICATION = {"piece": "classification", "value": "seller"} class Strategy(Model): @@ -63,6 +66,22 @@ def __init__(self, **kwargs) -> None: latitude=location["latitude"], longitude=location["longitude"] ) } + self._set_personality_data = kwargs.pop( + "personality_data", DEFAULT_PERSONALITY_DATA + ) + enforce( + len(self._set_personality_data) == 2 + and "piece" in self._set_personality_data + and "value" in self._set_personality_data, + "personality_data must contain keys `key` and `value`", + ) + self._set_classification = kwargs.pop("classification", DEFAULT_CLASSIFICATION) + enforce( + len(self._set_classification) == 2 + and "piece" in self._set_classification + and "value" in self._set_classification, + "classification must contain keys `key` and `value`", + ) self._set_service_data = kwargs.pop("service_data", DEFAULT_SERVICE_DATA) enforce( len(self._set_service_data) == 2 @@ -114,6 +133,28 @@ def get_location_description(self) -> Description: ) return description + def get_register_personality_description(self) -> Description: + """ + Get the register personality description. + + :return: a description of the personality + """ + description = Description( + self._set_personality_data, data_model=AGENT_PERSONALITY_MODEL, + ) + return description + + def get_register_classification_description(self) -> Description: + """ + Get the register classification description. + + :return: a description of the classification + """ + description = Description( + self._set_classification, data_model=AGENT_PERSONALITY_MODEL, + ) + return description + def get_register_service_description(self) -> Description: """ Get the register service description. diff --git a/packages/fetchai/skills/ml_train/behaviours.py b/packages/fetchai/skills/ml_train/behaviours.py index 8c850cb7d9..f1e81bce0a 100644 --- a/packages/fetchai/skills/ml_train/behaviours.py +++ b/packages/fetchai/skills/ml_train/behaviours.py @@ -19,11 +19,144 @@ """This package contains the behaviours.""" -from packages.fetchai.skills.generic_buyer.behaviours import ( - GenericSearchBehaviour, - GenericTransactionBehaviour, +from typing import List, Optional, Set, cast + +from aea.protocols.dialogue.base import DialogueLabel +from aea.skills.behaviours import TickerBehaviour +from ml_model_trainer.vendor.fetchai.skills.ml_train.dialogues import ( + LedgerApiDialogue, + LedgerApiDialogues, +) + +from packages.fetchai.connections.ledger.base import ( + CONNECTION_ID as LEDGER_CONNECTION_PUBLIC_ID, ) +from packages.fetchai.protocols.ledger_api import LedgerApiMessage +from packages.fetchai.protocols.ml_trade.dialogues import ( + MlTradeDialogue, + MlTradeDialogues, +) +from packages.fetchai.skills.generic_buyer.behaviours import GenericSearchBehaviour +from packages.fetchai.skills.ml_train.strategy import Strategy + + +DEFAULT_MAX_PROCESSING = 120 +DEFAULT_TX_INTERVAL = 2.0 +DEFAULT_SEARCH_INTERVAL = 5.0 +LEDGER_API_ADDRESS = str(LEDGER_CONNECTION_PUBLIC_ID) SearchBehaviour = GenericSearchBehaviour -TransactionBehaviour = GenericTransactionBehaviour + + +class TransactionBehaviour(TickerBehaviour): + """A behaviour to sequentially submit transactions to the blockchain.""" + + def __init__(self, **kwargs): + """Initialize the transaction behaviour.""" + tx_interval = cast( + float, kwargs.pop("transaction_interval", DEFAULT_TX_INTERVAL) + ) + self.max_processing = cast( + float, kwargs.pop("max_processing", DEFAULT_MAX_PROCESSING) + ) + self.processing_time = 0.0 + self.waiting: List[MlTradeDialogue] = [] + self.processing: Optional[LedgerApiDialogue] = None + self.timedout: Set[DialogueLabel] = set() + super().__init__(tick_interval=tx_interval, **kwargs) + + def setup(self) -> None: + """Setup behaviour.""" + pass + + def act(self) -> None: + """ + Implement the act. + + :return: None + """ + if self.processing is not None: + if self.processing_time <= self.max_processing: + # already processing + self.processing_time += self.tick_interval + return + self._timeout_processing() + if len(self.waiting) == 0: + # nothing to process + return + self._start_processing() + + def _start_processing(self) -> None: + """Process the next transaction.""" + ml_trade_dialogue = self.waiting.pop(0) + self.context.logger.info( + f"Processing transaction, {len(self.waiting)} transactions remaining" + ) + strategy = cast(Strategy, self.context.strategy) + proposal = ml_trade_dialogue.last_message.terms + terms = strategy.terms_from_proposal( + proposal, ml_trade_dialogue.last_message.sender + ) + ml_trade_dialogue.terms = terms + + ledger_api_dialogues = cast( + LedgerApiDialogues, self.context.ledger_api_dialogues + ) + ledger_api_msg, ledger_api_dialogue = ledger_api_dialogues.create( + counterparty=LEDGER_API_ADDRESS, + performative=LedgerApiMessage.Performative.GET_RAW_TRANSACTION, + terms=terms, + ) + ledger_api_dialogue = cast(LedgerApiDialogue, ledger_api_dialogue) + ledger_api_dialogue.associated_ml_trade_dialogue = ml_trade_dialogue + self.processing_time = 0.0 + self.processing = ledger_api_dialogue + self.context.logger.info( + f"requesting transfer transaction from ledger api for message={ledger_api_msg}..." + ) + self.context.outbox.put_message(message=ledger_api_msg) + + def teardown(self) -> None: + """Teardown behaviour.""" + pass + + def _timeout_processing(self) -> None: + """Timeout processing.""" + if self.processing is None: + return + self.timedout.add(self.processing.dialogue_label) + self.waiting.append(self.processing.associated_ml_trade_dialogue) + self.processing_time = 0.0 + self.processing = None + + def finish_processing(self, ledger_api_dialogue: LedgerApiDialogue) -> None: + """ + Finish processing. + + :param ledger_api_dialogue: the ledger api dialogue + """ + if self.processing == ledger_api_dialogue: + self.processing_time = 0.0 + self.processing = None + return + if ledger_api_dialogue.dialogue_label not in self.timedout: + raise ValueError( + f"Non-matching dialogues in transaction behaviour: {self.processing} and {ledger_api_dialogue}" + ) + self.timedout.remove(ledger_api_dialogue.dialogue_label) + self.context.logger.debug( + f"Timeout dialogue in transaction processing: {ledger_api_dialogue}" + ) + # don't reset, as another might be processing + + def failed_processing(self, ledger_api_dialogue: LedgerApiDialogue) -> None: + """ + Failed processing. + + Currently, we retry processing indefinitely. + + :param ledger_api_dialogue: the ledger api dialogue + """ + self.finish_processing(ledger_api_dialogue) + self.waiting.append(ledger_api_dialogue.associated_ml_trade_dialogue) diff --git a/packages/fetchai/skills/ml_train/handlers.py b/packages/fetchai/skills/ml_train/handlers.py index 214b0f8ac1..0f2c4250f6 100644 --- a/packages/fetchai/skills/ml_train/handlers.py +++ b/packages/fetchai/skills/ml_train/handlers.py @@ -177,9 +177,7 @@ def _handle_terms( tx_behaviour = cast( TransactionBehaviour, self.context.behaviours.transaction ) - tx_behaviour.waiting.append( - (ledger_api_dialogue, cast(LedgerApiMessage, ledger_api_msg)) # type: ignore - ) + tx_behaviour.waiting.append(ml_trade_dialogue) else: # accept directly with a dummy transaction digest, no settlement ml_accept = ml_trade_dialogue.reply( diff --git a/packages/fetchai/skills/ml_train/skill.yaml b/packages/fetchai/skills/ml_train/skill.yaml index 7914c97dfe..ef75142e4d 100644 --- a/packages/fetchai/skills/ml_train/skill.yaml +++ b/packages/fetchai/skills/ml_train/skill.yaml @@ -9,12 +9,12 @@ aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmeHcYRhDJi6gqAHpK29UJz3doiyaicoqeTrPddqM3gh64 __init__.py: QmTurqhUDy13mCSrqmbd598Eqz7A5PAmDrL9UpeNg24dDr - behaviours.py: QmXTG79nbRCELAJao6zasPDNxoDX4YYqVbJh6aRc5FXdyZ + behaviours.py: QmXiV4QSb2CfiBMNL8os6GWvtHaKj7herqYUZJ9RcHdQLL dialogues.py: QmVLXHVwXXLnK4zJzPBVhxzgu7ZmohRZ7oj3yG1LPFrmhx - handlers.py: QmaSUgDzw2ype25JQf9jNvgJ4uPxzmn58Ue6Atcs4w6DoH + handlers.py: QmYi7BaxgQcYBhSDbmEDbNcqSYPsSJTYAPQzy2SGTBVAtx ml_model.py: QmTfshn6dFnz9gKXZt7aJJczRH14bN7nk6TybwFpzkEPnk model.json: QmdV2tGrRY6VQ5VLgUa4yqAhPDG6X8tYsWecypq8nox9Td - strategy.py: QmPiuoPn8nUBnsx7FVuDjXbHB71TQEeezqW6HZanvAkszb + strategy.py: QmaeAuVjk4aj35qP5ypSf52rVK2uXfQqdMQFRZKH1LgufA tasks.py: QmahJRCf6V61FsqrKgMMUyJ8F7PRd6C2bjunZg2XtM9fpF fingerprint_ignore_patterns: [] connections: diff --git a/packages/fetchai/skills/ml_train/strategy.py b/packages/fetchai/skills/ml_train/strategy.py index 75abebd629..0f004556e5 100644 --- a/packages/fetchai/skills/ml_train/strategy.py +++ b/packages/fetchai/skills/ml_train/strategy.py @@ -28,6 +28,7 @@ Location, Query, ) +from aea.helpers.transaction.base import Address, Terms from aea.skills.base import Model @@ -196,3 +197,29 @@ def is_affordable_terms(self, terms: Description) -> bool: else: result = True return result + + def terms_from_proposal( + self, proposal: Description, counterparty_address: Address + ) -> Terms: + """ + Get the terms from a proposal. + + :param proposal: the proposal + :return: terms + """ + buyer_address = self.context.agent_addresses[proposal.values["ledger_id"]] + terms = Terms( + ledger_id=proposal.values["ledger_id"], + sender_address=buyer_address, + counterparty_address=counterparty_address, + amount_by_currency_id={ + proposal.values["currency_id"]: -proposal.values["price"] + }, + quantities_by_good_id={ + proposal.values["service_id"]: proposal.values["quantity"] + }, + is_sender_payable_tx_fee=True, + nonce=proposal.values["tx_nonce"], + fee_by_currency_id={proposal.values["currency_id"]: self._max_buyer_tx_fee}, + ) + return terms diff --git a/packages/hashes.csv b/packages/hashes.csv index d06f2cb96c..6e087b5582 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -55,7 +55,7 @@ fetchai/protocols/fipa,QmdZevSTKavH7wCVXeZyiQ5y3sDAkhEQ3SiwhaiXWYgnDJ fetchai/protocols/gym,QmW8fqeoXvsgdjZzZt6Wfj6vfCJBEweAUFdUk1dqNyBa1m fetchai/protocols/http,QmYF5MEFZbJUo2xTLJwCt6cTkADbytUqfeWvsGzvuy6hJF fetchai/protocols/ledger_api,QmWjvwNBpomTDCPEfA86dhHihr2HkCpnveCVZDWU7x5m42 -fetchai/protocols/ml_trade,QmezUacB3b2N31wcs6zxCgurFWubEFJXrfPdaT368LVhTX +fetchai/protocols/ml_trade,QmW8pMfbErnFK7vnPAW6ikazRKt1oBvFKPTZUvHzGkW7xr fetchai/protocols/oef_search,QmQXGHb6GRgeeeYGy5KVPz6bkaWxo3S6tm3fHqoNJ3Zoij fetchai/protocols/prometheus,QmR9KCLCWMMwJRkGV313WGxYwk49bLtg6SxoAavYb3C97h fetchai/protocols/register,QmNwCo2HYKjZzHbjPfmAZnYUBFVURbbFR2JZAye9Wg2WmY @@ -80,8 +80,8 @@ fetchai/skills/generic_buyer,QmNRP6rWnTNMwDYygzzN6BY7kUxh2DMYrjNaQVd19M4Nv4 fetchai/skills/generic_seller,QmR7xuDhrnq8gXMe9pk65LykCydkcB8mPh3UgPGZUgvpQ6 fetchai/skills/gym,Qmb4Bhfhhiwu9UsvSyJdLworYm41agCA6WffrYGxN3Sr1x fetchai/skills/http_echo,QmRJRubhKEWPmwxqEac3Zk9AXbAR4J4oB5k56CWuj9zPFE -fetchai/skills/ml_data_provider,QmQCcpUtyQ5FvWyjfxtp7VXVoRMZaRBVAfuDdkmsQxDnSm -fetchai/skills/ml_train,QmPjCNUHzYi2iFpF9FdWMnQeKen6ZHhWjYbCevWsDzHzwo +fetchai/skills/ml_data_provider,QmbdLMEdhRuoMi7BRJM1K6K6HKBPb23s6PpYppBuz8vgiE +fetchai/skills/ml_train,QmSYF6dwth9GEpTxrc4LLmrM7LvcvFLCs7Hpim7trHLCuQ fetchai/skills/registration_aw1,QmQiDCMeMnkQ4D3eu1mjAXoJKo51976KrAkfYWWNFKTGnA fetchai/skills/scaffold,QmezXGa1jPSteM9c6iH1hCHJVarqbLaG7mwaJmkkAJ6XUx fetchai/skills/simple_buyer,QmXLnLkkBDMq6KcFZ6XRcNW57djKCcuu7XicJGykcHZCM6 From fb6428a82ca0f0d26356b77a43459bd72b2d81ec Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Fri, 25 Dec 2020 12:23:45 +0300 Subject: [PATCH 058/204] env vars support --- aea/aea_builder.py | 61 +-- aea/cli/add_key.py | 5 +- aea/cli/config.py | 284 +------------ aea/cli/registry/fetch.py | 5 +- aea/cli/registry/push.py | 30 +- aea/cli/remove.py | 5 +- aea/cli/utils/click_utils.py | 4 +- aea/cli/utils/constants.py | 8 +- aea/cli/utils/context.py | 3 +- aea/cli/utils/decorators.py | 3 +- aea/cli/utils/formatting.py | 3 +- aea/cli/utils/package_utils.py | 24 +- aea/configurations/base.py | 102 ++--- aea/configurations/loader.py | 9 +- aea/configurations/manager.py | 410 +++++++++++++++++++ aea/configurations/validation.py | 96 ++++- aea/crypto/helpers.py | 74 +--- aea/helpers/constants.py | 27 ++ aea/helpers/env_vars.py | 109 +++++ aea/helpers/storage/backends/base.py | 6 +- aea/helpers/yaml_utils.py | 38 +- aea/manager/project.py | 2 +- aea/test_tools/generic.py | 8 +- tests/conftest.py | 3 +- tests/test_aea_builder.py | 14 - tests/test_cli/test_registry/test_push.py | 11 +- tests/test_cli/test_transfer.py | 6 +- tests/test_configurations/test_base.py | 22 - tests/test_configurations/test_manager.py | 160 ++++++++ tests/test_configurations/test_validation.py | 41 ++ tests/test_crypto/test_helpers.py | 8 - tests/test_helpers/test_yaml_utils.py | 67 --- tests/test_launcher.py | 9 +- 33 files changed, 973 insertions(+), 684 deletions(-) create mode 100644 aea/configurations/manager.py create mode 100644 aea/helpers/constants.py create mode 100644 aea/helpers/env_vars.py create mode 100644 tests/test_configurations/test_manager.py create mode 100644 tests/test_configurations/test_validation.py diff --git a/aea/aea_builder.py b/aea/aea_builder.py index b1f9e52063..8c6f0b0957 100644 --- a/aea/aea_builder.py +++ b/aea/aea_builder.py @@ -68,11 +68,13 @@ SIGNING_PROTOCOL, SKILLS, STATE_UPDATE_PROTOCOL, - VENDOR, ) from aea.configurations.loader import ConfigLoader, load_component_configuration +from aea.configurations.manager import ( + AgentConfigManager, + find_component_directory_from_component_id, +) from aea.configurations.pypi import is_satisfiable, merge_dependencies -from aea.crypto.helpers import verify_or_create_private_keys from aea.crypto.ledger_apis import DEFAULT_CURRENCY_DENOMINATIONS from aea.crypto.wallet import Wallet from aea.decision_maker.base import DecisionMakerHandler @@ -1349,36 +1351,9 @@ def _check_pypi_dependencies(self, configuration: ComponentConfiguration): f"Conflict on package {pkg_name}: specifier set '{dep_info.version}' not satisfiable." ) - @staticmethod - def find_component_directory_from_component_id( - aea_project_directory: Path, component_id: ComponentId - ) -> Path: - """Find a component directory from component id.""" - # search in vendor first - vendor_package_path = ( - aea_project_directory - / VENDOR - / component_id.public_id.author - / component_id.component_type.to_plural() - / component_id.public_id.name - ) - if vendor_package_path.exists() and vendor_package_path.is_dir(): - return vendor_package_path - - # search in custom packages. - custom_package_path = ( - aea_project_directory - / component_id.component_type.to_plural() - / component_id.public_id.name - ) - if custom_package_path.exists() and custom_package_path.is_dir(): - return custom_package_path - - raise ValueError("Package {} not found.".format(component_id)) - @classmethod def try_to_load_agent_configuration_file( - cls, aea_project_path: Union[Path, str] + cls, aea_project_path: Union[str, Path] ) -> AgentConfig: """Try to load the agent configuration file..""" try: @@ -1508,8 +1483,8 @@ def set_from_configuration( agent_configuration.component_configurations ) + @staticmethod def _find_import_order( - self, component_ids: List[ComponentId], aea_project_path: Path, skip_consistency_check: bool, @@ -1528,7 +1503,7 @@ def _find_import_order( ComponentId, Set[ComponentId] ] = defaultdict(set) for component_id in component_ids: - component_path = self.find_component_directory_from_component_id( + component_path = find_component_directory_from_component_id( aea_project_path, component_id ) configuration = load_component_configuration( @@ -1576,18 +1551,18 @@ def from_aea_project( :return: an AEABuilder. """ aea_project_path = Path(aea_project_path) - load_env_file(str(aea_project_path / DEFAULT_ENV_DOTFILE)) + cls.try_to_load_agent_configuration_file(aea_project_path) - # load to verify - agent_configuration = cls.try_to_load_agent_configuration_file(aea_project_path) - logging.config.dictConfig(agent_configuration.logging_config) # type: ignore - - verify_or_create_private_keys( - aea_project_path=aea_project_path, exit_on_error=False - ) + load_env_file(str(aea_project_path / DEFAULT_ENV_DOTFILE)) + # check and create missing, do not replace env variables. updates config + AgentConfigManager.verify_or_create_private_keys( + aea_project_path, substitude_env_vars=False + ).dump_config() - # load agent configuration file - agent_configuration = cls.try_to_load_agent_configuration_file(aea_project_path) + # just validate + agent_configuration = AgentConfigManager.verify_or_create_private_keys( + aea_project_path, substitude_env_vars=True + ).agent_config builder = AEABuilder(with_default_packages=False) builder.set_from_configuration( @@ -1691,7 +1666,7 @@ def _add_components_of_type( component_ids, aea_project_path, skip_consistency_check ) for component_id in import_order: - component_path = self.find_component_directory_from_component_id( + component_path = find_component_directory_from_component_id( aea_project_path, component_id ) self.add_component( diff --git a/aea/cli/add_key.py b/aea/cli/add_key.py index 12d13de33f..d7ff697e5a 100644 --- a/aea/cli/add_key.py +++ b/aea/cli/add_key.py @@ -88,6 +88,5 @@ def _try_add_key(ctx: Context, type_: str, filepath: str, connection: bool = Fal ctx.agent_config.private_key_paths.create(type_, filepath) except ValueError as e: # pragma: no cover raise click.ClickException(str(e)) - ctx.agent_loader.dump( - ctx.agent_config, open(os.path.join(ctx.cwd, DEFAULT_AEA_CONFIG_FILE), "w") - ) + with open(os.path.join(ctx.cwd, DEFAULT_AEA_CONFIG_FILE), "w") as fp: + ctx.agent_loader.dump(ctx.agent_config, fp) diff --git a/aea/cli/config.py b/aea/cli/config.py index 409264b716..efd3b2774e 100644 --- a/aea/cli/config.py +++ b/aea/cli/config.py @@ -19,49 +19,18 @@ """Implementation of the 'aea config' subcommand.""" import contextlib import json -from copy import deepcopy -from pathlib import Path -from typing import Dict, List, NewType, Optional, Tuple, Union, cast +from typing import Optional, cast import click from click.exceptions import ClickException -from aea.aea_builder import AEABuilder -from aea.cli.utils.config import ( - _try_get_component_id_from_prefix, - _try_get_configuration_object_from_aea_config, - handle_dotted_path, -) -from aea.cli.utils.constants import ( - CONFIG_SUPPORTED_KEY_TYPES, - FALSE_EQUIVALENTS, - FROM_STRING_TO_TYPE, -) +from aea.cli.utils.constants import CONFIG_SUPPORTED_KEY_TYPES from aea.cli.utils.context import Context from aea.cli.utils.decorators import check_aea_project, pass_ctx -from aea.configurations.base import ( - AgentConfig, - ComponentConfiguration, - ComponentId, - DEFAULT_AEA_CONFIG_FILE, - PackageType, -) -from aea.configurations.loader import ConfigLoader, load_component_configuration +from aea.configurations.manager import AgentConfigManager, VariableDoesNotExist from aea.configurations.validation import ExtraPropertiesError from aea.exceptions import AEAException -from aea.helpers.storage.backends.base import JSON_TYPES - - -class VariableDoesNotExist(ValueError): - """Variable does not exist in a config exception.""" - - -JsonPath = List[str] -VariablePath = Union[str, JsonPath] - - -NotExistsType = NewType("NotExistsType", object) -NotExists = NotExistsType(None) +from aea.helpers.env_vars import convert_value_str_to_type @click.group() @@ -77,7 +46,8 @@ def config(click_context): # pylint: disable=unused-argument def get(ctx: Context, json_path: str): """Get a field.""" try: - value = AgentConfigManager(ctx.agent_config, ctx.cwd).get_variable(json_path) + agent_config_manager = AgentConfigManager.load(ctx.cwd) + value = agent_config_manager.get_variable(json_path) except (ValueError, AEAException) as e: raise ClickException(*e.args) @@ -103,7 +73,7 @@ def set_command( ): """Set a field.""" try: - agent_config_manager = AgentConfigManager(ctx.agent_config, ctx.cwd) + agent_config_manager = AgentConfigManager.load(ctx.cwd) current_value = None with contextlib.suppress(VariableDoesNotExist): @@ -112,18 +82,16 @@ def set_command( # type was not specified, tried to auto determine if type_ is None: # apply str as default type - converted_value = AgentConfigManager.convert_value_str_to_type(value, "str") + converted_value = convert_value_str_to_type(value, "str") if current_value is not None: # try to convert to original value's type with contextlib.suppress(Exception): - converted_value = AgentConfigManager.convert_value_str_to_type( + converted_value = convert_value_str_to_type( value, type(current_value).__name__ ) else: # convert to type specified by user - converted_value = AgentConfigManager.convert_value_str_to_type( - value, cast(str, type_) - ) + converted_value = convert_value_str_to_type(value, cast(str, type_)) agent_config_manager.set_variable(json_path, converted_value) agent_config_manager.dump_config() @@ -131,235 +99,3 @@ def set_command( raise ClickException(f"Attribute `{e.args[0][0]}` is not allowed to change!") except (ValueError, AEAException) as e: raise ClickException(*e.args) - - -class AgentConfigManager: - """AeaConfig manager.""" - - component_configurations = "component_configurations" - - def __init__( - self, agent_config: AgentConfig, aea_project_directory: Union[str, Path] - ) -> None: - """ - Init manager. - - :param agent_config: AgentConfig to manage. - :param aea_project_directory: directory where project for agent_config placed. - """ - self.agent_config = agent_config - self.aea_project_directory = aea_project_directory - - def load_component_configuration( - self, component_id: ComponentId, skip_consistency_check: bool = True, - ) -> ComponentConfiguration: - """ - Load component configuration from the project directory. - - :param component_id: Id of the component to load config for. - :param skip_consistency_check: bool. - - :return: ComponentConfiguration - """ - path = AEABuilder.find_component_directory_from_component_id( - aea_project_directory=Path(self.aea_project_directory), - component_id=component_id, - ) - return load_component_configuration( - component_type=component_id.component_type, - directory=path, - skip_consistency_check=skip_consistency_check, - ) - - @property - def agent_config_file_path(self) -> Path: - """Return agent config file path.""" - return Path(self.aea_project_directory) / DEFAULT_AEA_CONFIG_FILE - - @classmethod - def load(cls, aea_project_path: str) -> "AgentConfigManager": - """Create AgentConfigManager instance from agent project path.""" - raise NotImplementedError # pragma: nocover - - def set_variable(self, path: VariablePath, value: JSON_TYPES) -> None: - """ - Set config variable. - - :param path: str dotted path or List[Union[ComponentId, str]] - :param value: one of the json friendly objects. - - :return: None - """ - component_id, json_path = self._parse_path(path) - data = self._make_dict_for_path_and_value(json_path, value) - overrides = {} - if component_id: - overrides[self.component_configurations] = {component_id: data} - else: - # agent - overrides.update(data) - - self.update_config(overrides) - - @staticmethod - def _make_dict_for_path_and_value(json_path: JsonPath, value: JSON_TYPES) -> Dict: - """ - Turn json_path and value into overrides dict. - - :param json_path: List[str] represents config variable path: - :param value: json friendly value - - :return: dict of overrides - """ - data: Dict = {} - nested = data - for key in json_path[:-1]: - nested[key] = {} - nested = nested[key] - nested[json_path[-1]] = value - return data - - def get_variable(self, path: VariablePath) -> JSON_TYPES: - """ - Set config variable. - - :param path: str dotted path or List[Union[ComponentId, str]] - - :return: json friendly value. - """ - component_id, json_path = self._parse_path(path) - - if component_id: - configrations_data = [ - _try_get_configuration_object_from_aea_config( - self.agent_config, component_id - ) - or {}, - self.load_component_configuration(component_id).json, - ] - else: - configrations_data = [self.agent_config.json] - - for data in configrations_data: - value = self._get_value_for_json_path(data, json_path) - if value is not NotExists: - return cast(JSON_TYPES, value) - - raise VariableDoesNotExist( - f"Attribute `{'.'.join(json_path)}` for {'{}({}) config'.format(component_id.component_type,component_id.public_id) if component_id else 'AgentConfig'} does not exist" - ) - - @staticmethod - def _get_value_for_json_path( - data: Dict, json_path: JsonPath - ) -> Union[NotExistsType, JSON_TYPES]: - """ - Get value by json path from the dict object. - - :param data: dict to get value from - :param json_path: List[str] - - :return: one of the json values of NotExists if value not presents in data dict. - """ - value = json.loads(json.dumps(data)) # in case or ordered dict - prev_key = "" - for key in json_path: - if not isinstance(value, dict): - raise ValueError(f"Attribute '{prev_key}' is not a dictionary.") - - if key not in value: - return NotExists - value = value[key] - prev_key = key - return value - - def _parse_path(self, path: VariablePath) -> Tuple[Optional[ComponentId], JsonPath]: - """ - Get component_id and json path from dotted path or list of str with first optional component id. - - :param path: dotted path str, list of str with first optional component id - - :return: Tuple of optonal component id if path related to component and List[str] - """ - if isinstance(path, str): - json_path, *_, component_id = handle_dotted_path( - path, self.agent_config.author - ) - else: # pragma: nocover - if isinstance(path[0], ComponentId): - json_path = path[1:] - component_id = path[0] - else: - component_id = None - json_path = path - if component_id: - component_id = _try_get_component_id_from_prefix( - set(self.agent_config.all_components_id), component_id.component_prefix - ) - return component_id, json_path - - def update_config(self, overrides: Dict) -> None: - """ - Apply overrides for agent config. - - Validates and applies agent config and component overrides. - Does not save it on the disc! - - :param overrides: overrided values dictionary - - :return: None - """ - for component_id, obj in overrides.get("component_configurations", {}).items(): - component_configuration = self.load_component_configuration(component_id) - component_configuration.check_overrides_valid(obj) - - self.agent_config.update(overrides) - - @property - def json(self) -> Dict: - """Return current agent config json representation.""" - return self.agent_config.json - - def dump_config(self) -> None: - """Save agent config on the disc.""" - config_data = self.json - self.agent_config.validate_config_data(config_data) - with open(self.agent_config_file_path, "w") as file_pointer: - ConfigLoader.from_configuration_type(PackageType.AGENT).dump( - self.agent_config, file_pointer - ) - - @staticmethod - def convert_value_str_to_type(value: str, type_str: str) -> JSON_TYPES: - """Convert value by type name to native python type.""" - try: - type_ = FROM_STRING_TO_TYPE[type_str] - if type_ == bool: - return value not in FALSE_EQUIVALENTS - if type_ is None: - return None - if type_ in (dict, list): - return json.loads(value) - return type_(value) - except (ValueError, json.decoder.JSONDecodeError): # pragma: no cover - raise ValueError("Cannot convert {} to type {}".format(value, type_)) - - def get_overridables(self) -> Tuple[Dict, Dict[ComponentId, Dict]]: - """Get config overridables.""" - agent_overridable = self.agent_config.get_overridable() - - components_overridables: Dict[ComponentId, Dict] = {} - for component_id in self.agent_config.all_components_id: - obj = {} - component_config = self.load_component_configuration( - component_id, skip_consistency_check=True - ) - obj.update(component_config.get_overridable()) - obj.update( - deepcopy( - self.agent_config.component_configurations.get(component_id, {}) - ) - ) - if obj: - components_overridables[component_id] = obj - return agent_overridable, components_overridables diff --git a/aea/cli/registry/fetch.py b/aea/cli/registry/fetch.py index 9682b58062..78963c0a67 100644 --- a/aea/cli/registry/fetch.py +++ b/aea/cli/registry/fetch.py @@ -77,9 +77,8 @@ def fetch_agent( if alias is not None: ctx.agent_config.agent_name = alias - ctx.agent_loader.dump( - ctx.agent_config, open(os.path.join(ctx.cwd, DEFAULT_AEA_CONFIG_FILE), "w") - ) + with open(os.path.join(ctx.cwd, DEFAULT_AEA_CONFIG_FILE), "w") as fp: + ctx.agent_loader.dump(ctx.agent_config, fp) click.echo("Fetching dependencies...") for item_type in (CONNECTION, CONTRACT, SKILL, PROTOCOL): diff --git a/aea/cli/registry/push.py b/aea/cli/registry/push.py index 4f7b65842d..03ad2c73b2 100644 --- a/aea/cli/registry/push.py +++ b/aea/cli/registry/push.py @@ -144,16 +144,20 @@ def push_item(ctx: Context, item_type: str, item_id: PublicId) -> None: if deps_list: data.update({key: deps_list}) - files = {"file": open(output_filepath, "rb")} - readme_path = os.path.join(item_path, DEFAULT_README_FILE) - if is_readme_present(readme_path): - files["readme"] = open(readme_path, "rb") - - path = "/{}/create".format(item_type_plural) - logger.debug("Pushing {} {} to Registry ...".format(item_id.name, item_type)) - resp = request_api("POST", path, data=data, is_auth=True, files=files) - click.echo( - "Successfully pushed {} {} to the Registry. Public ID: {}".format( - item_type, item_id.name, resp["public_id"] - ) - ) + try: + files = {"file": open(output_filepath, "rb")} + readme_path = os.path.join(item_path, DEFAULT_README_FILE) + if is_readme_present(readme_path): + files["readme"] = open(readme_path, "rb") + + path = "/{}/create".format(item_type_plural) + logger.debug("Pushing {} {} to Registry ...".format(item_id.name, item_type)) + resp = request_api("POST", path, data=data, is_auth=True, files=files) + finally: + for fd in files.values(): + fd.close() + click.echo( + "Successfully pushed {} {} to the Registry. Public ID: {}".format( + item_type, item_id.name, resp["public_id"] + ) + ) diff --git a/aea/cli/remove.py b/aea/cli/remove.py index 4ae4aa95aa..c0ceb390d4 100644 --- a/aea/cli/remove.py +++ b/aea/cli/remove.py @@ -17,7 +17,6 @@ # # ------------------------------------------------------------------------------ """Implementation of the 'aea remove' subcommand.""" - import os import shutil from collections import defaultdict @@ -27,7 +26,6 @@ import click -from aea.aea_builder import AEABuilder from aea.cli.utils.click_utils import PublicIdParameter from aea.cli.utils.config import load_item_config, try_to_load_agent_config from aea.cli.utils.context import Context @@ -52,6 +50,7 @@ PROTOCOL, SKILL, ) +from aea.configurations.manager import find_component_directory_from_component_id @click.group() @@ -170,7 +169,7 @@ def get_item_config(cls, package_id: PackageId) -> PackageConfiguration: def get_component_directory(package_id: PackageId) -> Path: """Return path for package.""" try: - return AEABuilder.find_component_directory_from_component_id( + return find_component_directory_from_component_id( Path("."), ComponentId(str(package_id.package_type), package_id.public_id), ) diff --git a/aea/cli/utils/click_utils.py b/aea/cli/utils/click_utils.py index 4160d7977e..2fff7d58be 100644 --- a/aea/cli/utils/click_utils.py +++ b/aea/cli/utils/click_utils.py @@ -108,8 +108,8 @@ def convert(self, value, param, ctx): try: # check that the target folder is an AEA project. os.chdir(path) - fp = open(DEFAULT_AEA_CONFIG_FILE, mode="r", encoding="utf-8") - ctx.obj.agent_config = ctx.obj.agent_loader.load(fp) + with open(DEFAULT_AEA_CONFIG_FILE, mode="r", encoding="utf-8") as fp: + ctx.obj.agent_config = ctx.obj.agent_loader.load(fp) try_to_load_agent_config(ctx.obj) # everything ok - return the parameter to the command return value diff --git a/aea/cli/utils/constants.py b/aea/cli/utils/constants.py index 4ad2a8e4d3..4818e17233 100644 --- a/aea/cli/utils/constants.py +++ b/aea/cli/utils/constants.py @@ -16,9 +16,7 @@ # limitations under the License. # # ------------------------------------------------------------------------------ - """Module with constants of the aea cli.""" - import os from pathlib import Path from typing import Dict @@ -40,6 +38,7 @@ SKILLS, VENDOR, ) +from aea.helpers.constants import FROM_STRING_TO_TYPE AEA_DIR = str(Path(".")) @@ -60,9 +59,6 @@ ] -FROM_STRING_TO_TYPE = dict( - str=str, int=int, bool=bool, float=float, dict=dict, list=list, none=None, -) CONFIG_SUPPORTED_KEY_TYPES = list(FROM_STRING_TO_TYPE.keys()) ALLOWED_PATH_ROOTS = [ @@ -79,6 +75,6 @@ CONNECTIONS: DEFAULT_CONNECTION_CONFIG_FILE, CONTRACTS: DEFAULT_CONTRACT_CONFIG_FILE, } # type: Dict[str, str] -FALSE_EQUIVALENTS = ["f", "false", "False", "0"] + REQUIREMENTS = "requirements.txt" diff --git a/aea/cli/utils/context.py b/aea/cli/utils/context.py index fa58f58f5d..b96e09655a 100644 --- a/aea/cli/utils/context.py +++ b/aea/cli/utils/context.py @@ -105,7 +105,8 @@ def _get_item_dependencies(item_type, public_id: PublicId) -> Dependencies: if not path.exists(): path = Path(item_type_plural, public_id.name, default_config_file_name) config_loader = ConfigLoader.from_configuration_type(item_type) - config = config_loader.load(path.open()) + with path.open() as fp: + config = config_loader.load(fp) deps = cast(Dependencies, config.dependencies) return deps diff --git a/aea/cli/utils/decorators.py b/aea/cli/utils/decorators.py index a43b32fc92..65a9e6e1c9 100644 --- a/aea/cli/utils/decorators.py +++ b/aea/cli/utils/decorators.py @@ -91,7 +91,8 @@ def _validate_config_consistency(ctx: Context): # load the configuration file. try: - package_configuration = loader.load(configuration_file_path.open("r")) + with configuration_file_path.open("r") as fp: + package_configuration = loader.load(fp) except ValidationError as e: raise ValueError( "{} configuration file not valid: {}".format( diff --git a/aea/cli/utils/formatting.py b/aea/cli/utils/formatting.py index 9df1724790..2ae3e02aa0 100644 --- a/aea/cli/utils/formatting.py +++ b/aea/cli/utils/formatting.py @@ -51,7 +51,8 @@ def format_items(items): def retrieve_details(name: str, loader: ConfigLoader, config_filepath: str) -> Dict: """Return description of a protocol, skill, connection.""" - config = loader.load(open(str(config_filepath))) + with open(str(config_filepath)) as fp: + config = loader.load(fp) item_name = config.agent_name if isinstance(config, AgentConfig) else config.name enforce(item_name == name, "Item names do not match!") return { diff --git a/aea/cli/utils/package_utils.py b/aea/cli/utils/package_utils.py index 898e73f595..5179bfa5cb 100644 --- a/aea/cli/utils/package_utils.py +++ b/aea/cli/utils/package_utils.py @@ -60,8 +60,8 @@ VENDOR, ) from aea.configurations.loader import ConfigLoader +from aea.configurations.manager import AgentConfigManager from aea.configurations.utils import replace_component_ids -from aea.crypto.helpers import verify_or_create_private_keys from aea.crypto.ledger_apis import DEFAULT_LEDGER_CONFIGS, LedgerApis from aea.crypto.wallet import Wallet from aea.exceptions import AEAEnforceError @@ -82,7 +82,12 @@ def verify_or_create_private_keys_ctx( :param ctx: Context """ try: - agent_config = verify_or_create_private_keys(aea_project_path, exit_on_error) + AgentConfigManager.verify_or_create_private_keys( + aea_project_path, exit_on_error, substitude_env_vars=False + ).dump_config() + agent_config = AgentConfigManager.verify_or_create_private_keys( + aea_project_path, exit_on_error + ).agent_config if ctx is not None: ctx.agent_config = agent_config except ValueError as e: # pragma: nocover @@ -279,9 +284,8 @@ def find_item_locally( item_configuration_loader = ConfigLoader.from_configuration_type( PackageType(item_type) ) - item_configuration = item_configuration_loader.load( - item_configuration_filepath.open() - ) + with item_configuration_filepath.open() as fp: + item_configuration = item_configuration_loader.load(fp) except ValidationError as e: raise click.ClickException( "{} configuration file not valid: {}".format(item_type.capitalize(), str(e)) @@ -331,9 +335,8 @@ def find_item_in_distribution( # pylint: disable=unused-argument item_configuration_loader = ConfigLoader.from_configuration_type( PackageType(item_type) ) - item_configuration = item_configuration_loader.load( - item_configuration_filepath.open() - ) + with item_configuration_filepath.open() as fp: + item_configuration = item_configuration_loader.load(fp) except ValidationError as e: raise click.ClickException( "{} configuration file not valid: {}".format(item_type.capitalize(), str(e)) @@ -425,9 +428,8 @@ def register_item(ctx: Context, item_type: str, item_public_id: PublicId) -> Non ) supported_items = get_items(ctx.agent_config, item_type) supported_items.add(item_public_id) - ctx.agent_loader.dump( - ctx.agent_config, open(os.path.join(ctx.cwd, DEFAULT_AEA_CONFIG_FILE), "w") - ) + with open(os.path.join(ctx.cwd, DEFAULT_AEA_CONFIG_FILE), "w") as fp: + ctx.agent_loader.dump(ctx.agent_config, fp) def is_item_present_unified(ctx: Context, item_type: str, item_public_id: PublicId): diff --git a/aea/configurations/base.py b/aea/configurations/base.py index 4c6a8e692a..eca468f924 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -72,13 +72,12 @@ PackageVersion, PublicId, ) -from aea.configurations.validation import ConfigValidator +from aea.configurations.validation import ConfigValidator, validate_data_with_pattern from aea.exceptions import enforce from aea.helpers.base import ( CertRequest, SimpleId, SimpleIdOrStr, - dict_to_path_value, load_module, recursive_update, ) @@ -312,7 +311,7 @@ def package_dependencies(self) -> Set[ComponentId]: """Get the package dependencies.""" return set() - def update(self, data: Dict) -> None: + def update(self, data: Dict, env_vars_friendly: bool = False) -> None: """ Update configuration with other data. @@ -322,15 +321,19 @@ def update(self, data: Dict) -> None: if not data: # do nothing if nothing to update return - self.check_overrides_valid(data) + self.check_overrides_valid(data, env_vars_friendly=env_vars_friendly) self._create_or_update_from_json( obj=self.make_resulting_config_data(data), instance=self ) @classmethod - def validate_config_data(cls, json_data: Dict) -> None: + def validate_config_data( + cls, json_data: Dict, env_vars_friendly: bool = False + ) -> None: """Perform config validation.""" - ConfigValidator(cls.schema).validate(json_data) + ConfigValidator(cls.schema, env_vars_friendly=env_vars_friendly).validate( + json_data + ) @classmethod def from_json(cls, obj: Dict): @@ -344,51 +347,6 @@ def _create_or_update_from_json( """Create new config object or updates existing one from json data.""" raise NotImplementedError # pragma: nocover - @staticmethod - def _compare_data_to_pattern( - data: dict, pattern: dict, excludes: Optional[List[Tuple[str]]] = None - ) -> List[str]: - excludes = excludes or [] - pattern_path_value = { - tuple(path): value for path, value in dict_to_path_value(pattern) - } - data_path_value = { - tuple(path): value for path, value in dict_to_path_value(data) - } - errors = [] - - def check_excludes(path): - for exclude in excludes: - if len(exclude) > len(path): # pragma: nocover - continue - - if path[: len(exclude)] == exclude: - return True - return False - - for path, new_value in data_path_value.items(): - if check_excludes(path): - continue - - if path not in pattern_path_value: - errors.append( - f"Attribute `{'.'.join(path)}` is not allowed to be updated!" - ) - continue - - pattern_value = pattern_path_value[path] - - if pattern_value is None: - # not possible to determine data type for optional value not set - # it will be checked with jsonschema later - continue # pragma: nocover - - if not issubclass(type(new_value), type(pattern_value)): - errors.append( - f"For attribute `{'.'.join(path)}` `{type(pattern_value).__name__}` data type is expected, but `{type(new_value).__name__}` was provided!" - ) - return errors - def make_resulting_config_data(self, overrides: Dict) -> Dict: """Make config data with overrides applied. @@ -398,19 +356,28 @@ def make_resulting_config_data(self, overrides: Dict) -> Dict: recursive_update(current_config, overrides, allow_new_values=True) return current_config - def check_overrides_valid(self, overrides: Dict) -> None: + def check_overrides_valid( + self, overrides: Dict, env_vars_friendly: bool = False + ) -> None: """Check overrides is correct, return list of errors if present.""" # check for permited overrides - self._check_overrides_corresponds_to_overridable(overrides) + self._check_overrides_corresponds_to_overridable( + overrides, env_vars_friendly=env_vars_friendly + ) # check resulting config with applied overrides passes validation result_config = self.make_resulting_config_data(overrides) - self.validate_config_data(result_config) + self.validate_config_data(result_config, env_vars_friendly=env_vars_friendly) - def _check_overrides_corresponds_to_overridable(self, overrides: Dict) -> None: + def _check_overrides_corresponds_to_overridable( + self, overrides: Dict, env_vars_friendly: bool = False + ) -> None: """Check overrides is correct, return list of errors if present.""" - errors_list = self._compare_data_to_pattern( - overrides, self.get_overridable(), excludes=self.CHECK_EXCLUDES + errors_list = validate_data_with_pattern( + overrides, + self.get_overridable(), + excludes=self.CHECK_EXCLUDES, + skip_env_vars=env_vars_friendly, ) if errors_list: raise ValueError(errors_list[0]) @@ -732,19 +699,6 @@ def _create_or_update_from_json( return instance - def update(self, data: Dict) -> None: - """ - Update configuration with other data. - - This method does side-effect on the configuration object. - - :param data: the data to populate or replace. - :return: None - """ - new_config = data.get("config", {}) - recursive_update(self.config, new_config, allow_new_values=True) - self.is_abstract = data.get("is_abstract", self.is_abstract) - class ProtocolConfig(ComponentConfiguration): """Handle protocol configuration.""" @@ -1437,7 +1391,7 @@ def all_components_id(self) -> List[ComponentId]: return result - def update(self, data: Dict) -> None: + def update(self, data: Dict, env_vars_friendly: bool = False) -> None: """ Update configuration with other data. @@ -1463,9 +1417,9 @@ def update(self, data: Dict) -> None: allow_new_values=True, ) - self.check_overrides_valid(data) - super().update(data) - self.validate_config_data(self.json) + self.check_overrides_valid(data, env_vars_friendly=env_vars_friendly) + super().update(data, env_vars_friendly=env_vars_friendly) + self.validate_config_data(self.json, env_vars_friendly=env_vars_friendly) self.component_configurations = updated_component_configurations diff --git a/aea/configurations/loader.py b/aea/configurations/loader.py index e71886fff7..2ce50cc38d 100644 --- a/aea/configurations/loader.py +++ b/aea/configurations/loader.py @@ -192,7 +192,7 @@ def _load_from_json(self, configuration_file_json: Dict) -> T: return configuration_obj def load_agent_config_from_json( - self, configuration_json: List[Dict] + self, configuration_json: List[Dict], validate: bool = True ) -> AgentConfig: """ Load agent configuration from configuration json data. @@ -204,7 +204,8 @@ def load_agent_config_from_json( if len(configuration_json) == 0: raise ValueError("Agent configuration file was empty.") agent_config_json = configuration_json[0] - self.validate(agent_config_json) + if validate: + self.validate(agent_config_json) key_order = list(agent_config_json.keys()) agent_configuration_obj = cast( AgentConfig, self.configuration_class.from_json(agent_config_json) @@ -346,8 +347,8 @@ def _load_configuration_object( ) configuration_filepath = directory / configuration_filename try: - fp = open(configuration_filepath) - configuration_object = configuration_loader.load(fp) + with open(configuration_filepath) as fp: + configuration_object = configuration_loader.load(fp) except FileNotFoundError: raise FileNotFoundError( "{} configuration not found: {}".format( diff --git a/aea/configurations/manager.py b/aea/configurations/manager.py new file mode 100644 index 0000000000..ae7461c002 --- /dev/null +++ b/aea/configurations/manager.py @@ -0,0 +1,410 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""Implementation of the 'aea config' subcommand.""" +import json +import os +from copy import deepcopy +from pathlib import Path +from typing import Dict, List, NewType, Optional, Tuple, Union, cast + +from aea.cli.utils.config import ( + _try_get_component_id_from_prefix, + _try_get_configuration_object_from_aea_config, + handle_dotted_path, +) +from aea.configurations.base import ( + AgentConfig, + ComponentConfiguration, + ComponentId, + DEFAULT_AEA_CONFIG_FILE, + PackageType, +) +from aea.configurations.constants import PRIVATE_KEY_PATH_SCHEMA, VENDOR +from aea.configurations.loader import ConfigLoader, load_component_configuration +from aea.crypto.helpers import create_private_key, try_validate_private_key_path +from aea.crypto.registries import crypto_registry +from aea.helpers.env_vars import apply_env_variables, is_env_variable +from aea.helpers.storage.backends.base import JSON_TYPES +from aea.helpers.yaml_utils import yaml_load_all + + +JsonPath = List[str] +VariablePath = Union[str, JsonPath] + + +class VariableDoesNotExist(ValueError): + """Variable does not exist in a config exception.""" + + +NotExistsType = NewType("NotExistsType", object) +NotExists = NotExistsType(None) + + +def find_component_directory_from_component_id( + aea_project_directory: Path, component_id: ComponentId +) -> Path: + """Find a component directory from component id.""" + # search in vendor first + vendor_package_path = ( + aea_project_directory + / VENDOR + / component_id.public_id.author + / component_id.component_type.to_plural() + / component_id.public_id.name + ) + if vendor_package_path.exists() and vendor_package_path.is_dir(): + return vendor_package_path + + # search in custom packages. + custom_package_path = ( + aea_project_directory + / component_id.component_type.to_plural() + / component_id.public_id.name + ) + if custom_package_path.exists() and custom_package_path.is_dir(): + return custom_package_path + + raise ValueError("Package {} not found.".format(component_id)) + + +class AgentConfigManager: + """AeaConfig manager.""" + + component_configurations = "component_configurations" + _loader = ConfigLoader.from_configuration_type(PackageType.AGENT) + + def __init__( + self, + agent_config: AgentConfig, + aea_project_directory: Union[str, Path], + env_vars_friendly: bool = False, + ) -> None: + """ + Init manager. + + :param agent_config: AgentConfig to manage. + :param aea_project_directory: directory where project for agent_config placed. + """ + self.agent_config = agent_config + self.aea_project_directory = aea_project_directory + self.env_vars_friendly = env_vars_friendly + + def load_component_configuration( + self, component_id: ComponentId, skip_consistency_check: bool = True, + ) -> ComponentConfiguration: + """ + Load component configuration from the project directory. + + :param component_id: Id of the component to load config for. + :param skip_consistency_check: bool. + + :return: ComponentConfiguration + """ + path = find_component_directory_from_component_id( + aea_project_directory=Path(self.aea_project_directory), + component_id=component_id, + ) + return load_component_configuration( + component_type=component_id.component_type, + directory=path, + skip_consistency_check=skip_consistency_check, + ) + + @property + def agent_config_file_path(self) -> Path: + """Return agent config file path.""" + return self._get_agent_config_file_path(self.aea_project_directory) + + @classmethod + def _get_agent_config_file_path(cls, aea_project_path) -> Path: + """Get agent config file path for aea project path.""" + return Path(aea_project_path) / DEFAULT_AEA_CONFIG_FILE + + @classmethod + def load( + cls, aea_project_path: Union[Path, str], substitude_env_vars: bool = False + ) -> "AgentConfigManager": + """Create AgentConfigManager instance from agent project path.""" + data = cls._load_config_data(Path(aea_project_path)) + if substitude_env_vars: + data = cast(List[Dict], apply_env_variables(data, os.environ)) + agent_config = cls._loader.load_agent_config_from_json(data, validate=False) + instance = cls( + agent_config, aea_project_path, env_vars_friendly=not substitude_env_vars + ) + instance.validate_current_config() + return instance + + @classmethod + def _load_config_data(cls, aea_project_path: Path) -> List[Dict]: + with cls._get_agent_config_file_path(aea_project_path).open() as fp: + data = yaml_load_all(fp) + return data + + def set_variable(self, path: VariablePath, value: JSON_TYPES) -> None: + """ + Set config variable. + + :param path: str dotted path or List[Union[ComponentId, str]] + :param value: one of the json friendly objects. + + :return: None + """ + component_id, json_path = self._parse_path(path) + data = self._make_dict_for_path_and_value(json_path, value) + overrides = {} + if component_id: + overrides[self.component_configurations] = {component_id: data} + else: + # agent + overrides.update(data) + + self.update_config(overrides) + + @staticmethod + def _make_dict_for_path_and_value(json_path: JsonPath, value: JSON_TYPES) -> Dict: + """ + Turn json_path and value into overrides dict. + + :param json_path: List[str] represents config variable path: + :param value: json friendly value + + :return: dict of overrides + """ + data: Dict = {} + nested = data + for key in json_path[:-1]: + nested[key] = {} + nested = nested[key] + nested[json_path[-1]] = value + return data + + def get_variable(self, path: VariablePath) -> JSON_TYPES: + """ + Set config variable. + + :param path: str dotted path or List[Union[ComponentId, str]] + + :return: json friendly value. + """ + component_id, json_path = self._parse_path(path) + + if component_id: + configrations_data = [ + _try_get_configuration_object_from_aea_config( + self.agent_config, component_id + ) + or {}, + self.load_component_configuration(component_id).json, + ] + else: + configrations_data = [self.agent_config.json] + + for data in configrations_data: + value = self._get_value_for_json_path(data, json_path) + if value is not NotExists: + return cast(JSON_TYPES, value) + + raise VariableDoesNotExist( + f"Attribute `{'.'.join(json_path)}` for {'{}({}) config'.format(component_id.component_type,component_id.public_id) if component_id else 'AgentConfig'} does not exist" + ) + + @staticmethod + def _get_value_for_json_path( + data: Dict, json_path: JsonPath + ) -> Union[NotExistsType, JSON_TYPES]: + """ + Get value by json path from the dict object. + + :param data: dict to get value from + :param json_path: List[str] + + :return: one of the json values of NotExists if value not presents in data dict. + """ + value = json.loads(json.dumps(data)) # in case or ordered dict + prev_key = "" + for key in json_path: + if not isinstance(value, dict): + raise ValueError(f"Attribute '{prev_key}' is not a dictionary.") + + if key not in value: + return NotExists + value = value[key] + prev_key = key + return value + + def _parse_path(self, path: VariablePath) -> Tuple[Optional[ComponentId], JsonPath]: + """ + Get component_id and json path from dotted path or list of str with first optional component id. + + :param path: dotted path str, list of str with first optional component id + + :return: Tuple of optonal component id if path related to component and List[str] + """ + if isinstance(path, str): + json_path, *_, component_id = handle_dotted_path( + path, self.agent_config.author + ) + else: # pragma: nocover + if isinstance(path[0], ComponentId): + json_path = path[1:] + component_id = path[0] + else: + component_id = None + json_path = path + if component_id: + component_id = _try_get_component_id_from_prefix( + set(self.agent_config.all_components_id), component_id.component_prefix + ) + return component_id, json_path + + def update_config(self, overrides: Dict) -> None: + """ + Apply overrides for agent config. + + Validates and applies agent config and component overrides. + Does not save it on the disc! + + :param overrides: overrided values dictionary + + :return: None + """ + for component_id, obj in overrides.get("component_configurations", {}).items(): + component_configuration = self.load_component_configuration(component_id) + component_configuration.check_overrides_valid( + obj, env_vars_friendly=self.env_vars_friendly + ) + + self.agent_config.update(overrides, env_vars_friendly=self.env_vars_friendly) + + def validate_current_config(self): + """Check is current config valid.""" + for component_id, obj in self.agent_config.component_configurations.items(): + component_configuration = self.load_component_configuration(component_id) + component_configuration.check_overrides_valid( + obj, env_vars_friendly=self.env_vars_friendly + ) + self.agent_config.validate_config_data( + self.agent_config.json, env_vars_friendly=self.env_vars_friendly + ) + + @property + def json(self) -> Dict: + """Return current agent config json representation.""" + return self.agent_config.json + + def dump_config(self) -> None: + """Save agent config on the disc.""" + config_data = self.json + self.agent_config.validate_config_data( + config_data, env_vars_friendly=self.env_vars_friendly + ) + with open(self.agent_config_file_path, "w") as file_pointer: + ConfigLoader.from_configuration_type(PackageType.AGENT).dump( + self.agent_config, file_pointer + ) + + @classmethod + def verify_or_create_private_keys( + cls, + aea_project_path: Union[Path, str], + exit_on_error: bool = True, + substitude_env_vars: bool = False, + ) -> "AgentConfigManager": + """ + Verify or create private keys. + + Does not saves the config! Use AgentConfigManager.dump_config() + + :param aea_project_path: path to an AEA project. + :param exit_on_error: whether we should exit the program on error. + :param substitude_env_vars: replace env vars with values, does not dump config + + :return: the agent configuration manager. + """ + aea_project_path = Path(aea_project_path) + agent_config_manager = cls.load( + aea_project_path, substitude_env_vars=substitude_env_vars + ) + aea_conf = agent_config_manager.agent_config + + for identifier, _ in aea_conf.private_key_paths.read_all(): + if identifier not in crypto_registry.supported_ids: # pragma: nocover + raise ValueError( + "Unsupported identifier `{}` in private key paths. Supported identifiers: {}.".format( + identifier, sorted(crypto_registry.supported_ids) + ) + ) + + for identifier in crypto_registry.supported_ids: + config_private_key_path = aea_conf.private_key_paths.read(identifier) + + if is_env_variable(config_private_key_path): + # skip env var definition + continue + + if config_private_key_path is None: + private_key_path = PRIVATE_KEY_PATH_SCHEMA.format(identifier) + if identifier == aea_conf.default_ledger: # pragma: nocover + if os.path.exists(private_key_path): + raise ValueError( + "File {} for private key {} already exists. Add to aea-config.yaml.".format( + repr(config_private_key_path), identifier + ) + ) + create_private_key( + identifier, + private_key_file=str(aea_project_path / private_key_path), + ) + aea_conf.private_key_paths.update(identifier, private_key_path) + else: + try: + try_validate_private_key_path( + identifier, + str(aea_project_path / config_private_key_path), + exit_on_error=exit_on_error, + ) + except FileNotFoundError: # pragma: no cover + raise ValueError( + "File {} for private key {} not found.".format( + repr(config_private_key_path), identifier, + ) + ) + + return agent_config_manager + + def get_overridables(self) -> Tuple[Dict, Dict[ComponentId, Dict]]: + """Get config overridables.""" + agent_overridable = self.agent_config.get_overridable() + + components_overridables: Dict[ComponentId, Dict] = {} + for component_id in self.agent_config.all_components_id: + obj = {} + component_config = self.load_component_configuration( + component_id, skip_consistency_check=True + ) + obj.update(component_config.get_overridable()) + obj.update( + deepcopy( + self.agent_config.component_configurations.get(component_id, {}) + ) + ) + if obj: + components_overridables[component_id] = obj + return agent_overridable, components_overridables diff --git a/aea/configurations/validation.py b/aea/configurations/validation.py index 767325ed59..49ee51fa47 100644 --- a/aea/configurations/validation.py +++ b/aea/configurations/validation.py @@ -22,16 +22,19 @@ import os from copy import deepcopy from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional, Tuple import jsonschema from jsonschema import Draft4Validator +from jsonschema._types import TypeChecker from jsonschema._utils import find_additional_properties from jsonschema._validators import additionalProperties from jsonschema.validators import extend from aea.configurations.constants import AGENT from aea.configurations.data_types import ComponentId, ComponentType, PublicId +from aea.helpers.base import dict_to_path_value +from aea.helpers.env_vars import is_env_variable _CUR_DIR = os.path.dirname(inspect.getfile(inspect.currentframe())) # type: ignore @@ -86,6 +89,16 @@ def __repr__(self) -> str: # pragma: nocover return str(self) +class CustomTypeChecker(TypeChecker): + """Custom type checker to handle env variables.""" + + def is_type(self, instance, type): # pylint: disable=redefined-builtin + """Check is instance of type.""" + if is_env_variable(instance): + return True + return super().is_type(instance, type) + + def ownAdditionalProperties(validator, aP, instance, schema): """Additioinal properties validator.""" for _ in additionalProperties(validator, aP, instance, schema): @@ -99,20 +112,36 @@ def ownAdditionalProperties(validator, aP, instance, schema): ) +EnvVarsFriendlyDraft4Validator = extend( + validator=Draft4Validator, + type_checker=CustomTypeChecker( + Draft4Validator.TYPE_CHECKER._type_checkers # pylint: disable=protected-access + ), +) + + class ConfigValidator: """Configuration validator implementation.""" - def __init__(self, schema_filename: str): + def __init__(self, schema_filename: str, env_vars_friendly: bool = False): """ Initialize the parser for configuration files. :param schema_filename: the path to the JSON-schema file in 'aea/configurations/schemas'. """ base_uri = Path(_SCHEMAS_DIR) - self._schema = json.load((base_uri / schema_filename).open()) + with (base_uri / schema_filename).open() as fp: + self._schema = json.load(fp) root_path = make_jsonschema_base_uri(base_uri) self._resolver = jsonschema.RefResolver(root_path, self._schema) - self._validator = OwnDraft4Validator(self._schema, resolver=self._resolver) + self.env_vars_friendly = env_vars_friendly + + if env_vars_friendly: + self._validator = EnvVarsFriendlyDraft4Validator( + self._schema, resolver=self._resolver + ) + else: + self._validator = OwnDraft4Validator(self._schema, resolver=self._resolver) @staticmethod def split_component_id_and_config( @@ -142,7 +171,10 @@ def split_component_id_and_config( @classmethod def validate_component_configuration( - cls, component_id: ComponentId, configuration: Dict + cls, + component_id: ComponentId, + configuration: Dict, + env_vars_friendly: bool = False, ) -> None: """ Validate the component configuration of an agent configuration file. @@ -158,7 +190,7 @@ def validate_component_configuration( component_id.component_type ) try: - cls(schema_file).validate( + cls(schema_file, env_vars_friendly=env_vars_friendly).validate( dict( type=str(component_id.component_type), public_id=str(component_id.public_id), @@ -216,3 +248,55 @@ def required_fields(self) -> List[str]: :return: list of required fields. """ return self._schema["required"] + + +def validate_data_with_pattern( + data: dict, + pattern: dict, + excludes: Optional[List[Tuple[str]]] = None, + skip_env_vars: bool = False, +) -> List[str]: + """Validate data dict with pattern dict for attributes present and type match.""" + excludes = excludes or [] + pattern_path_value = { + tuple(path): value for path, value in dict_to_path_value(pattern) + } + data_path_value = {tuple(path): value for path, value in dict_to_path_value(data)} + errors = [] + + def check_excludes(path): + for exclude in excludes: + if len(exclude) > len(path): # pragma: nocover + continue + + if path[: len(exclude)] == exclude: + return True + return False + + for path, new_value in data_path_value.items(): + if check_excludes(path): + continue + + if path not in pattern_path_value: + errors.append(f"Attribute `{'.'.join(path)}` is not allowed to be updated!") + continue + + pattern_value = pattern_path_value[path] + + if pattern_value is None: + # not possible to determine data type for optional value not set + # it will be checked with jsonschema later + continue # pragma: nocover + + if skip_env_vars and ( + is_env_variable(pattern_value) or is_env_variable(new_value) + ): + # one of the values is env variable: skip data type check + continue + + if not issubclass(type(new_value), type(pattern_value)): + errors.append( + f"For attribute `{'.'.join(path)}` `{type(pattern_value).__name__}` data type is expected, but `{type(new_value).__name__}` was provided!" + ) + + return errors diff --git a/aea/crypto/helpers.py b/aea/crypto/helpers.py index a12e038614..16d01353e0 100644 --- a/aea/crypto/helpers.py +++ b/aea/crypto/helpers.py @@ -18,82 +18,17 @@ # ------------------------------------------------------------------------------ """Module wrapping the helpers of public and private key cryptography.""" - import logging -import os import sys -from pathlib import Path from typing import Optional -from aea.configurations.base import AgentConfig, PackageType -from aea.configurations.constants import ( - DEFAULT_AEA_CONFIG_FILE, - PRIVATE_KEY_PATH_SCHEMA, -) -from aea.configurations.loader import ConfigLoaders -from aea.crypto.registries import crypto_registry, make_crypto, make_faucet_api +from aea.configurations.constants import PRIVATE_KEY_PATH_SCHEMA +from aea.crypto.registries import make_crypto, make_faucet_api _default_logger = logging.getLogger(__name__) - -def verify_or_create_private_keys( - aea_project_path: Path, exit_on_error: bool = True, -) -> AgentConfig: - """ - Verify or create private keys. - - :param aea_project_path: path to an AEA project. - :param exit_on_error: whether we should exit the program on error. - :return: the agent configuration. - """ - path_to_aea_config = aea_project_path / DEFAULT_AEA_CONFIG_FILE - agent_loader = ConfigLoaders.from_package_type(PackageType.AGENT) - fp = path_to_aea_config.open(mode="r", encoding="utf-8") - aea_conf = agent_loader.load(fp) - - for identifier, _ in aea_conf.private_key_paths.read_all(): - if identifier not in crypto_registry.supported_ids: # pragma: nocover - raise ValueError( - "Unsupported identifier `{}` in private key paths. Supported identifiers: {}.".format( - identifier, sorted(crypto_registry.supported_ids) - ) - ) - - for identifier in crypto_registry.supported_ids: - config_private_key_path = aea_conf.private_key_paths.read(identifier) - if config_private_key_path is None: - private_key_path = PRIVATE_KEY_PATH_SCHEMA.format(identifier) - if identifier == aea_conf.default_ledger: # pragma: nocover - if os.path.exists(private_key_path): - raise ValueError( - "File {} for private key {} already exists. Add to aea-config.yaml.".format( - repr(config_private_key_path), identifier - ) - ) - create_private_key( - identifier, - private_key_file=str(aea_project_path / private_key_path), - ) - aea_conf.private_key_paths.update(identifier, private_key_path) - else: - try: - try_validate_private_key_path( - identifier, - str(aea_project_path / config_private_key_path), - exit_on_error=exit_on_error, - ) - except FileNotFoundError: # pragma: no cover - raise ValueError( - "File {} for private key {} not found.".format( - repr(config_private_key_path), identifier, - ) - ) - - # update aea config - fp = path_to_aea_config.open(mode="w", encoding="utf-8") - agent_loader.dump(aea_conf, fp) - return aea_conf +_ = PRIVATE_KEY_PATH_SCHEMA # some modules expect this here def try_validate_private_key_path( @@ -133,7 +68,8 @@ def create_private_key(ledger_id: str, private_key_file: str) -> None: :raises: ValueError if the identifier is invalid. """ crypto = make_crypto(ledger_id) - crypto.dump(open(private_key_file, "wb")) + with open(private_key_file, "wb") as fp: + crypto.dump(fp) def try_generate_testnet_wealth( diff --git a/aea/helpers/constants.py b/aea/helpers/constants.py new file mode 100644 index 0000000000..67abb30a12 --- /dev/null +++ b/aea/helpers/constants.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2020 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""Module with helpers constants.""" +from typing import Dict, List, Union + + +FALSE_EQUIVALENTS = ["f", "false", "False", "0"] +FROM_STRING_TO_TYPE = dict( + str=str, int=int, bool=bool, float=float, dict=dict, list=list, none=None, +) +JSON_TYPES = Union[Dict, str, List, None, int, float] diff --git a/aea/helpers/env_vars.py b/aea/helpers/env_vars.py new file mode 100644 index 0000000000..cb4a05e6bd --- /dev/null +++ b/aea/helpers/env_vars.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""Implementation of the environment variables support.""" +import json +import re +from collections.abc import Mapping as MappingType +from typing import Any, Dict, List, Mapping, Union + +from aea.helpers.constants import FALSE_EQUIVALENTS, FROM_STRING_TO_TYPE, JSON_TYPES + + +ENV_VARIABLE_RE = re.compile( + r"^\$\{(?P\w+)(:(?P\w+)(:(?P\w+))?)?\}$", re.I +) + + +def is_env_variable(value: Any) -> bool: + """Check is variable string with env variable pattern.""" + return isinstance(value, str) and bool(ENV_VARIABLE_RE.match(value)) + + +NotSet = object() + + +def replace_with_env_var( + value: str, env_variables: dict, default_value=NotSet +) -> JSON_TYPES: + """Replace env var with value.""" + result = ENV_VARIABLE_RE.match(value) + if not result: + return value + + var_name = result.groupdict()["name"] + type_str = result.groupdict()["type"] + default = result.groupdict()["default"] + + if var_name in env_variables: + var_value = env_variables[var_name] + elif default is not None: + var_value = default + elif default_value is not NotSet: + var_value = default_value + else: + raise ValueError( + f"Var name {var_name} not found in env variables and no default value set!" + ) + + if type_str is not None: + var_value = convert_value_str_to_type(var_value, type_str) + + return var_value + + +def apply_env_variables( + data: Union[Dict, List[Dict]], + env_variables: Mapping[str, Any], + default_value=NotSet, +) -> JSON_TYPES: + """Create new resulting dict with env variables applied.""" + + if isinstance(data, (list, tuple)): + result = [] + for i in data: + result.append(apply_env_variables(i, env_variables, default_value)) + return result + + if isinstance(data, MappingType): + return { + apply_env_variables(k, env_variables, default_value): apply_env_variables( + v, env_variables, default_value + ) + for k, v in data.items() + } + if is_env_variable(data): + return replace_with_env_var(data, env_variables, default_value) + + return data + + +def convert_value_str_to_type(value: str, type_str: str) -> JSON_TYPES: + """Convert value by type name to native python type.""" + try: + type_ = FROM_STRING_TO_TYPE[type_str] + if type_ == bool: + return value not in FALSE_EQUIVALENTS + if type_ is None: + return None + if type_ in (dict, list): + return json.loads(value) + return type_(value) + except (ValueError, json.decoder.JSONDecodeError): # pragma: no cover + raise ValueError(f"Cannot convert string `{value}` to type `{type_.__name__}`") # type: ignore diff --git a/aea/helpers/storage/backends/base.py b/aea/helpers/storage/backends/base.py index 5df33f47aa..e441fa1003 100644 --- a/aea/helpers/storage/backends/base.py +++ b/aea/helpers/storage/backends/base.py @@ -17,14 +17,14 @@ # # ------------------------------------------------------------------------------ """This module contains storage abstract backend class.""" - import re from abc import ABC, abstractmethod -from typing import Dict, List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Union + +from aea.helpers.constants import JSON_TYPES EQUALS_TYPE = Union[int, float, str, bool] -JSON_TYPES = Union[Dict, str, List, None, int, float] OBJECT_ID_AND_BODY = Tuple[str, JSON_TYPES] diff --git a/aea/helpers/yaml_utils.py b/aea/helpers/yaml_utils.py index 1c4922cefd..abef8f260f 100644 --- a/aea/helpers/yaml_utils.py +++ b/aea/helpers/yaml_utils.py @@ -16,12 +16,9 @@ # limitations under the License. # # ------------------------------------------------------------------------------ - """Helper functions related to YAML loading/dumping.""" -import os -import re from collections import OrderedDict -from typing import Any, Dict, List, Match, Optional, Sequence, TextIO, cast +from typing import Any, Dict, List, Optional, Sequence, TextIO import yaml from yaml import MappingNode @@ -39,9 +36,6 @@ class _AEAYamlLoader(yaml.SafeLoader): the public functions of the module 'yaml_load' and 'yaml_load_all'. """ - envvar_matcher = re.compile(r"\${([^}^{]+)\}") - envvar_key = "!envvar" - def __init__(self, *args, **kwargs): """ Initialize the AEAYamlLoader. @@ -52,15 +46,6 @@ def __init__(self, *args, **kwargs): _AEAYamlLoader.add_constructor( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, self._construct_mapping ) - _AEAYamlLoader.add_constructor(self.envvar_key, self._envvar_constructor) - self._add_implicit_resolver_if_not_present_already() - - def _add_implicit_resolver_if_not_present_already(self) -> None: - """Add implicit resolver for environment variables, if not present already.""" - if self.envvar_key not in dict(self.yaml_implicit_resolvers.get(None, [])): - _AEAYamlLoader.add_implicit_resolver( - self.envvar_key, self.envvar_matcher, None - ) @staticmethod def _construct_mapping(loader: "_AEAYamlLoader", node: MappingNode): @@ -69,27 +54,6 @@ def _construct_mapping(loader: "_AEAYamlLoader", node: MappingNode): loader.flatten_mapping(node) return object_pairs_hook(loader.construct_pairs(node)) - @staticmethod - def _envvar_constructor(_loader: "_AEAYamlLoader", node: MappingNode) -> str: - """Extract the matched value, expand env variable, and replace the match.""" - node_value = node.value - match = _AEAYamlLoader.envvar_matcher.match(node_value) - match = cast(Match[str], match) - env_var = match.group()[2:-1] - - # check for defaults - var_split = env_var.split(":") - if len(var_split) == 2: - var_name, default_value = var_split - elif len(var_split) == 1: - var_name, default_value = var_split[0], "" - else: - raise ValueError(f"Cannot resolve environment variable '{env_var}'.") - var_name = var_name.strip() - default_value = default_value.strip() - var_value = os.getenv(var_name, default_value) - return var_value + node_value[match.end() :] - class _AEAYamlDumper(yaml.SafeDumper): """ diff --git a/aea/manager/project.py b/aea/manager/project.py index 386539ea2c..59af503acd 100644 --- a/aea/manager/project.py +++ b/aea/manager/project.py @@ -25,13 +25,13 @@ from aea.aea import AEA from aea.aea_builder import AEABuilder -from aea.cli.config import AgentConfigManager from aea.cli.fetch import fetch_agent_locally from aea.cli.registry.fetch import fetch_agent from aea.cli.utils.context import Context from aea.configurations.base import AgentConfig, PublicId from aea.configurations.constants import DEFAULT_REGISTRY_NAME from aea.configurations.data_types import ComponentId +from aea.configurations.manager import AgentConfigManager from aea.crypto.helpers import create_private_key diff --git a/aea/test_tools/generic.py b/aea/test_tools/generic.py index 13a8d06910..4f8ebb1d3c 100644 --- a/aea/test_tools/generic.py +++ b/aea/test_tools/generic.py @@ -193,8 +193,8 @@ def nested_set_config( if config.package_type == PackageType.AGENT: json_data = config.ordered_json component_configurations = json_data.pop("component_configurations") - yaml_dump_all( - [json_data] + component_configurations, config_file_path.open("w") - ) + with config_file_path.open("w") as fp: + yaml_dump_all([json_data] + component_configurations, fp) else: - yaml_dump(config.ordered_json, config_file_path.open("w")) + with config_file_path.open("w") as fp: + yaml_dump(config.ordered_json, fp) diff --git a/tests/conftest.py b/tests/conftest.py index 3718c55a88..63b6afc1bd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,7 +49,7 @@ from aea.configurations.base import DEFAULT_PROTOCOL_CONFIG_FILE as PROTOCOL_YAML from aea.configurations.base import DEFAULT_SKILL_CONFIG_FILE as SKILL_YAML from aea.configurations.base import PublicId -from aea.configurations.constants import DEFAULT_LEDGER +from aea.configurations.constants import DEFAULT_LEDGER, PRIVATE_KEY_PATH_SCHEMA from aea.configurations.loader import load_component_configuration from aea.connections.base import Connection from aea.contracts.base import Contract, contract_registry @@ -63,7 +63,6 @@ from aea.crypto.ethereum import EthereumApi, EthereumCrypto, _ETHEREUM from aea.crypto.fetchai import DEFAULT_ADDRESS as FETCHAI_DEFAULT_ADDRESS from aea.crypto.fetchai import _FETCHAI -from aea.crypto.helpers import PRIVATE_KEY_PATH_SCHEMA from aea.crypto.ledger_apis import DEFAULT_LEDGER_CONFIGS from aea.crypto.registries import ledger_apis_registry, make_crypto from aea.crypto.wallet import CryptoStore diff --git a/tests/test_aea_builder.py b/tests/test_aea_builder.py index bc1d7b3992..7dd2a0e419 100644 --- a/tests/test_aea_builder.py +++ b/tests/test_aea_builder.py @@ -466,20 +466,6 @@ def test_component_add_bad_dep(): builder.add_component_instance(a_protocol) -def test_find_component_failed(): - """Test fail on compomnent not found.""" - builder = AEABuilder() - builder.set_name("aea_1") - builder.add_private_key("fetchai") - a_protocol = Protocol( - ProtocolConfig("a_protocol", "author", "0.1.0"), DefaultMessage - ) - with pytest.raises(ValueError, match=r"Package .* not found"): - builder.find_component_directory_from_component_id( - Path("/some_dir"), a_protocol.component_id - ) - - def test_set_from_config(): """Test set configuration from config loaded.""" builder = AEABuilder() diff --git a/tests/test_cli/test_registry/test_push.py b/tests/test_cli/test_registry/test_push.py index 8313140490..849cdadc17 100644 --- a/tests/test_cli/test_registry/test_push.py +++ b/tests/test_cli/test_registry/test_push.py @@ -16,11 +16,10 @@ # limitations under the License. # # ------------------------------------------------------------------------------ - """Test module for Registry push methods.""" - import os from unittest import TestCase, mock +from unittest.mock import mock_open import pytest from click import ClickException @@ -36,7 +35,7 @@ from tests.test_cli.tools_for_testing import ContextMock, PublicIdMock -@mock.patch("builtins.open", return_value="opened_file") +@mock.patch("builtins.open", mock_open(read_data="opened_file")) @mock.patch("aea.cli.registry.push.check_is_author_logged_in") @mock.patch("aea.cli.registry.utils._rm_tarfiles") @mock.patch("aea.cli.registry.push.os.getcwd", return_value="cwd") @@ -69,7 +68,6 @@ def test_push_item_positive( getcwd_mock, rm_tarfiles_mock, check_is_author_logged_in_mock, - open_mock, ): """Test for push_item positive result.""" public_id = PublicIdMock( @@ -88,7 +86,7 @@ def test_push_item_positive( "protocols": ["protocol_id"], }, is_auth=True, - files={"file": "opened_file", "readme": "opened_file"}, + files={"file": open("file.1"), "readme": open("file.2")}, ) @mock.patch("aea.cli.registry.push.os.path.exists", return_value=True) @@ -113,7 +111,7 @@ def test_push_item_positive_without_readme( "protocols": ["protocol_id"], }, is_auth=True, - files={"file": "opened_file"}, + files={"file": open("opened_file", "r")}, ) @mock.patch("aea.cli.registry.push.os.path.exists", return_value=False) @@ -126,7 +124,6 @@ def test_push_item_item_not_found( getcwd_mock, rm_tarfiles_mock, check_is_author_logged_in_mock, - open_mock, ): """Test for push_item - item not found.""" with self.assertRaises(ClickException): diff --git a/tests/test_cli/test_transfer.py b/tests/test_cli/test_transfer.py index 5ca2fe7027..509edc81ca 100644 --- a/tests/test_cli/test_transfer.py +++ b/tests/test_cli/test_transfer.py @@ -27,9 +27,9 @@ from aea.cli.transfer import wait_tx_settled from aea.cli.utils.package_utils import get_wallet_from_agent_config, try_get_balance +from aea.configurations.manager import AgentConfigManager from aea.crypto.cosmos import CosmosCrypto from aea.crypto.fetchai import FetchAICrypto -from aea.crypto.helpers import verify_or_create_private_keys from aea.helpers.base import cd from aea.test_tools.test_cases import AEATestCaseEmpty @@ -75,7 +75,9 @@ def get_address(self) -> str: def get_balance(self) -> int: """Get balance for current agent.""" with cd(self._get_cwd()): - agent_config = verify_or_create_private_keys(Path("."), False) + agent_config = AgentConfigManager.verify_or_create_private_keys( + Path("."), False, substitude_env_vars=False + ).agent_config wallet = get_wallet_from_agent_config(agent_config) return int(try_get_balance(agent_config, wallet, self.LEDGER_ID)) diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index 02d1c59cc7..31a4a7e33a 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -35,7 +35,6 @@ ConnectionConfig, ContractConfig, Dependency, - PackageConfiguration, PackageId, PackageType, PackageVersion, @@ -982,24 +981,3 @@ def test_check_public_id_consistency_negative(): with pytest.raises(ValueError, match=f"Directory {random_dir_name} is not valid."): component_configuration = ProtocolConfig("name", "author") component_configuration.check_public_id_consistency(Path(random_dir_name)) - - -def test_compare_data_pattern(): - """Test PackageConfiguration._compare_data_to_pattern.""" - errors = PackageConfiguration._compare_data_to_pattern({"a": 12}, {"a": 13}) - assert not errors - - errors = PackageConfiguration._compare_data_to_pattern({"a": 12}, {"a": "string"}) - assert errors - assert ( - errors[0] - == "For attribute `a` `str` data type is expected, but `int` was provided!" - ) - - errors = PackageConfiguration._compare_data_to_pattern({"a": 12}, {"b": 12}) - assert errors - assert errors[0] == "Attribute `a` is not allowed to be updated!" - - errors = PackageConfiguration._compare_data_to_pattern({"a": {}}, {"a": {"b": 12}}) - assert errors - assert errors[0] == "Attribute `a` is not allowed to be updated!" diff --git a/tests/test_configurations/test_manager.py b/tests/test_configurations/test_manager.py new file mode 100644 index 0000000000..ede5ecb7b6 --- /dev/null +++ b/tests/test_configurations/test_manager.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This test module contains the tests for the configurations manager module.""" + +import os +from copy import deepcopy +from unittest.mock import patch + +import pytest +import yaml + +from aea.configurations.manager import AgentConfigManager + + +agent_config_data = yaml.safe_load( + """ +agent_name: Agent0 +author: dummy_author +version: 1.0.0 +description: dummy_aea agent description +license: Apache-2.0 +aea_version: '>=0.8.0, <0.9.0' +fingerprint: {} +fingerprint_ignore_patterns: [] +connections: +- fetchai/local:0.13.0 +contracts: [] +protocols: +- fetchai/default:0.10.0 +skills: +- dummy_author/dummy:0.1.0 +default_connection: fetchai/local:0.13.0 +default_ledger: cosmos +logging_config: + disable_existing_loggers: ${DISABLE_LOGS:bool} + version: 1 +private_key_paths: + cosmos: cosmos_private_key.txt + ethereum: ethereum_private_key.txt +connection_private_key_paths: + cosmos: cosmos_private_key.txt + ethereum: ethereum_private_key.txt +registry_path: ../../packages +default_routing: {} +""" +) + + +def test_envvars_applied(): + """Test env vars replaced with values.""" + dct = deepcopy(agent_config_data) + with patch.object(AgentConfigManager, "_load_config_data", return_value=[dct]): + os.environ["DISABLE_LOGS"] = "true" + agent_config_manager = AgentConfigManager.load(".", substitude_env_vars=True) + assert ( + agent_config_manager.json["logging_config"]["disable_existing_loggers"] is True + ) + + with patch.object(AgentConfigManager, "_load_config_data", return_value=[dct]): + os.environ["DISABLE_LOGS"] = "false" + agent_config_manager = AgentConfigManager.load(".", substitude_env_vars=True) + assert ( + agent_config_manager.json["logging_config"]["disable_existing_loggers"] is False + ) + + # no env! no default value + os.environ.pop("DISABLE_LOGS") + with pytest.raises( + ValueError, + match="Var name DISABLE_LOGS not found in env variables and no default value set!", + ): + with patch.object(AgentConfigManager, "_load_config_data", return_value=[dct]): + agent_config_manager = AgentConfigManager.load( + ".", substitude_env_vars=True + ) + assert ( + agent_config_manager.json["logging_config"]["disable_existing_loggers"] + is False + ) + + # check default value specified + dct["logging_config"]["disable_existing_loggers"] = "${DISABLE_LOGS:bool:true}" + with patch.object(AgentConfigManager, "_load_config_data", return_value=[dct]): + agent_config_manager = AgentConfigManager.load(".", substitude_env_vars=True) + assert ( + agent_config_manager.json["logging_config"]["disable_existing_loggers"] + is True + ) + + # check incorrect data type + dct["logging_config"]["disable_existing_loggers"] = "${DISABLE_LOGS:int:true}" + with pytest.raises(ValueError, match="Cannot convert string `true` to type `int`"): + with patch.object(AgentConfigManager, "_load_config_data", return_value=[dct]): + agent_config_manager = AgentConfigManager.load( + ".", substitude_env_vars=True + ) + assert ( + agent_config_manager.json["logging_config"]["disable_existing_loggers"] + is True + ) + + # not applied + dct = deepcopy(agent_config_data) + with patch.object(AgentConfigManager, "_load_config_data", return_value=[dct]): + os.environ["DISABLE_LOGS"] = "true" + agent_config_manager = AgentConfigManager.load(".", substitude_env_vars=False) + assert ( + agent_config_manager.json["logging_config"]["disable_existing_loggers"] + == dct["logging_config"]["disable_existing_loggers"] + ) + + +def test_envvars_preserved(): + """Test env vars not modified on config update.""" + dct = deepcopy(agent_config_data) + new_cosmos_key_value = "cosmons_key_updated" + + with patch.object(AgentConfigManager, "_load_config_data", return_value=[dct]): + os.environ["DISABLE_LOGS"] = "true" + agent_config_manager = AgentConfigManager.load(".", substitude_env_vars=False) + + assert ( + agent_config_manager.json["logging_config"]["disable_existing_loggers"] + == dct["logging_config"]["disable_existing_loggers"] + ) + + with patch.object(AgentConfigManager, "_load_config_data", return_value=[dct]): + os.environ["DISABLE_LOGS"] = "true" + agent_config_manager = AgentConfigManager.load(".", substitude_env_vars=False) + + assert ( + agent_config_manager.json["private_key_paths"]["cosmos"] != new_cosmos_key_value + ) + agent_config_manager.update_config( + {"private_key_paths": {"cosmos": new_cosmos_key_value}} + ) + + assert ( + agent_config_manager.json["logging_config"]["disable_existing_loggers"] + == dct["logging_config"]["disable_existing_loggers"] + ) + assert ( + agent_config_manager.json["private_key_paths"]["cosmos"] == new_cosmos_key_value + ) diff --git a/tests/test_configurations/test_validation.py b/tests/test_configurations/test_validation.py new file mode 100644 index 0000000000..3f30b042bc --- /dev/null +++ b/tests/test_configurations/test_validation.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the tests for the aea.configurations.validation module.""" +from aea.configurations.validation import validate_data_with_pattern + + +def test_compare_data_pattern(): + """Test validate_data_with_pattern.""" + errors = validate_data_with_pattern({"a": 12}, {"a": 13}) + assert not errors + + errors = validate_data_with_pattern({"a": 12}, {"a": "string"}) + assert errors + assert ( + errors[0] + == "For attribute `a` `str` data type is expected, but `int` was provided!" + ) + + errors = validate_data_with_pattern({"a": 12}, {"b": 12}) + assert errors + assert errors[0] == "Attribute `a` is not allowed to be updated!" + + errors = validate_data_with_pattern({"a": {}}, {"a": {"b": 12}}) + assert errors + assert errors[0] == "Attribute `a` is not allowed to be updated!" diff --git a/tests/test_crypto/test_helpers.py b/tests/test_crypto/test_helpers.py index db4661ec33..510073bb4f 100644 --- a/tests/test_crypto/test_helpers.py +++ b/tests/test_crypto/test_helpers.py @@ -21,7 +21,6 @@ import logging import os -from pathlib import Path from unittest.mock import mock_open, patch import pytest @@ -34,7 +33,6 @@ create_private_key, try_generate_testnet_wealth, try_validate_private_key_path, - verify_or_create_private_keys, ) from tests.conftest import ( @@ -149,9 +147,3 @@ def test__create_ethereum_private_key_positive(self, *mocks): def test__create_cosmos_private_key_positive(self, *mocks): """Test _create_cosmos_private_key positive result.""" create_private_key(CosmosCrypto.identifier, COSMOS_PRIVATE_KEY_FILE) - - @patch("aea.crypto.helpers.create_private_key") - @patch("aea.crypto.helpers.try_validate_private_key_path") - def test_verify_or_create_private_keys(self, *mocks): - """Test _create_ethereum_private_key positive result.""" - verify_or_create_private_keys(Path(os.path.join(CUR_PATH, "data", "dummy_aea"))) diff --git a/tests/test_helpers/test_yaml_utils.py b/tests/test_helpers/test_yaml_utils.py index d72b9a3c8d..04ddc6aaad 100644 --- a/tests/test_helpers/test_yaml_utils.py +++ b/tests/test_helpers/test_yaml_utils.py @@ -18,13 +18,9 @@ # ------------------------------------------------------------------------------ """This module contains the tests for the yaml utils module.""" import io -import os import random import string from collections import OrderedDict -from textwrap import dedent - -import pytest from aea.helpers.yaml_utils import ( _AEAYamlLoader, @@ -67,66 +63,3 @@ def _generate_random_string(n: int = 100): return "".join( random.choice(string.ascii_uppercase + string.digits) for _ in range(n) # nosec ) - - -def test_resolve_env_variable_with_default(): - """Test that the AEAYamlLoader resolves env default variable correctly.""" - random_variable_name = _generate_random_string() - variable_value = "my_variable_default" - variable_key = "my_variable" - yaml_file = dedent( - f""" - {variable_key}: ${{{random_variable_name}:{variable_value}}} - """ - ) - stream = io.StringIO(yaml_file) - yaml_obj = yaml_load(stream) - assert yaml_obj[variable_key] == variable_value - - -def test_resolve_env_variable_without_default_positive(): - """Test that the AEAYamlLoader resolves env variable without default correctly.""" - random_variable_name = _generate_random_string() - variable_value = "my_value" - variable_key = "my_variable" - os.environ[random_variable_name] = variable_value - try: - yaml_file = dedent( - f""" - {variable_key}: ${{{random_variable_name}}} - """ - ) - stream = io.StringIO(yaml_file) - yaml_obj = yaml_load(stream) - finally: - os.environ.pop(random_variable_name) - assert yaml_obj[variable_key] == variable_value - - -def test_resolve_env_variable_without_default_negative(): - """Test that the AEAYamlLoader resolves unspecified env variable without default correctly.""" - random_variable_name = _generate_random_string() - variable_key = "my_variable" - yaml_file = dedent( - f""" - {variable_key}: ${{{random_variable_name}}} - """ - ) - stream = io.StringIO(yaml_file) - yaml_obj = yaml_load(stream) - assert yaml_obj[variable_key] == "" - - -def test_resolve_env_variable_fails(): - """Test the case when resolving the environment variable fails.""" - with pytest.raises( - ValueError, match="Cannot resolve environment variable 'some:wrong:variable'." - ): - wrong_var = "some:wrong:variable" - yaml_file = dedent( - f""" - some_variable_name: ${{{wrong_var}}} - """ - ) - stream = io.StringIO(yaml_file) - yaml_load(stream) diff --git a/tests/test_launcher.py b/tests/test_launcher.py index f3167675dd..0df908e52a 100644 --- a/tests/test_launcher.py +++ b/tests/test_launcher.py @@ -85,7 +85,8 @@ def setup_class(cls): Path(cls.t, cls.failing_agent, "skills", "exception"), ) config_path = Path(cls.t, cls.failing_agent, DEFAULT_AEA_CONFIG_FILE) - config = yaml.safe_load(open(config_path)) + with open(config_path) as fp: + config = yaml.safe_load(fp) config.setdefault("skills", []).append("fetchai/exception:0.1.0") yaml.safe_dump(config, open(config_path, "w")) os.chdir(cls.t) @@ -98,9 +99,11 @@ def set_runtime_mode_to_async(cls, agent_name: str) -> None: """Set runtime mode of the agent to async.""" with cd(agent_name): config_path = Path(cls.t, agent_name, DEFAULT_AEA_CONFIG_FILE) - config = yaml.safe_load(open(config_path)) + with open(config_path) as fp: + config = yaml.safe_load(fp) config.setdefault("runtime_mode", "async") - yaml.safe_dump(config, open(config_path, "w")) + with open(config_path, "w") as fp: + yaml.safe_dump(config, fp) @classmethod def teardown_class(cls): From 91373c87994f0cf35009cb30faec5a5f5b7498c3 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 28 Dec 2020 22:09:13 +0100 Subject: [PATCH 059/204] update tf check to skip ml skill tests --- .../fetchai/protocols/ml_trade/dialogues.py | 18 +----------------- .../fetchai/protocols/ml_trade/protocol.yaml | 2 +- packages/fetchai/skills/ml_train/behaviours.py | 8 ++------ packages/fetchai/skills/ml_train/handlers.py | 2 +- packages/fetchai/skills/ml_train/skill.yaml | 4 ++-- packages/hashes.csv | 4 ++-- .../test_skills_integration/test_ml_skills.py | 10 +++------- 7 files changed, 12 insertions(+), 36 deletions(-) diff --git a/packages/fetchai/protocols/ml_trade/dialogues.py b/packages/fetchai/protocols/ml_trade/dialogues.py index 0d093869fb..3f89804c13 100644 --- a/packages/fetchai/protocols/ml_trade/dialogues.py +++ b/packages/fetchai/protocols/ml_trade/dialogues.py @@ -25,11 +25,9 @@ """ from abc import ABC -from typing import Callable, FrozenSet, Optional, Type, cast +from typing import Callable, FrozenSet, Type, cast from aea.common import Address -from aea.exceptions import AEAEnforceError, enforce -from aea.helpers.transaction.base import Terms from aea.protocols.base import Message from aea.protocols.dialogue.base import Dialogue, DialogueLabel, Dialogues @@ -85,20 +83,6 @@ def __init__( self_address=self_address, role=role, ) - self._terms = None # type: Optional[Terms] - - @property - def terms(self) -> Terms: - """Get terms.""" - if self._terms is None: - raise AEAEnforceError("Terms not set!") - return self._terms - - @terms.setter - def terms(self, terms: Terms) -> None: - """Set terms.""" - enforce(self._terms is None, "Terms already set!") - self._terms = terms class MlTradeDialogues(Dialogues, ABC): diff --git a/packages/fetchai/protocols/ml_trade/protocol.yaml b/packages/fetchai/protocols/ml_trade/protocol.yaml index 087959bc96..95a98e39b2 100644 --- a/packages/fetchai/protocols/ml_trade/protocol.yaml +++ b/packages/fetchai/protocols/ml_trade/protocol.yaml @@ -9,7 +9,7 @@ fingerprint: README.md: QmccSUTpVzDpaAqpGPGXWY5aG9Dp1yStX4Sn2tqCjsKVEC __init__.py: QmcCS9uUQTTS2w85dTNiN5rQ14wyBhmBkr7pPPPcbLphcn custom_types.py: QmPa6mxbN8WShsniQxJACfzAPRjGzYLbUFGoVU4N9DewUw - dialogues.py: QmUUV9HXZojbFvw9mm58E2PDUETjbSfueSH8fCaysuaqPo + dialogues.py: Qmbras4sV8Fz7cnXwXMJLM5G4Y6rvSWvXakfTjDbyddbY5 message.py: QmYXwvWBCQkfPKGB8cbNPeomxS248K4CvgW8djNuXCq7Qa ml_trade.proto: QmNvSW9EUYiyu5SLhCVVaAdzLi1yjM94mnUeNSBzu2GRYH ml_trade_pb2.py: QmcrA4D7C9LFPcj6QKe8uUxYdHfHpRmZ5Dm4U5dKpT9nPh diff --git a/packages/fetchai/skills/ml_train/behaviours.py b/packages/fetchai/skills/ml_train/behaviours.py index f1e81bce0a..73bc5bee61 100644 --- a/packages/fetchai/skills/ml_train/behaviours.py +++ b/packages/fetchai/skills/ml_train/behaviours.py @@ -32,10 +32,7 @@ CONNECTION_ID as LEDGER_CONNECTION_PUBLIC_ID, ) from packages.fetchai.protocols.ledger_api import LedgerApiMessage -from packages.fetchai.protocols.ml_trade.dialogues import ( - MlTradeDialogue, - MlTradeDialogues, -) +from packages.fetchai.protocols.ml_trade.dialogues import MlTradeDialogue from packages.fetchai.skills.generic_buyer.behaviours import GenericSearchBehaviour from packages.fetchai.skills.ml_train.strategy import Strategy @@ -89,7 +86,7 @@ def act(self) -> None: def _start_processing(self) -> None: """Process the next transaction.""" - ml_trade_dialogue = self.waiting.pop(0) + ml_trade_dialogue = self.waiting.pop(0) self.context.logger.info( f"Processing transaction, {len(self.waiting)} transactions remaining" ) @@ -98,7 +95,6 @@ def _start_processing(self) -> None: terms = strategy.terms_from_proposal( proposal, ml_trade_dialogue.last_message.sender ) - ml_trade_dialogue.terms = terms ledger_api_dialogues = cast( LedgerApiDialogues, self.context.ledger_api_dialogues diff --git a/packages/fetchai/skills/ml_train/handlers.py b/packages/fetchai/skills/ml_train/handlers.py index 0f2c4250f6..978ca002cd 100644 --- a/packages/fetchai/skills/ml_train/handlers.py +++ b/packages/fetchai/skills/ml_train/handlers.py @@ -154,7 +154,7 @@ def _handle_terms( ledger_api_dialogues = cast( LedgerApiDialogues, self.context.ledger_api_dialogues ) - ledger_api_msg, ledger_api_dialogue = ledger_api_dialogues.create( + _ledger_api_msg, ledger_api_dialogue = ledger_api_dialogues.create( counterparty=LEDGER_API_ADDRESS, performative=LedgerApiMessage.Performative.GET_RAW_TRANSACTION, terms=Terms( diff --git a/packages/fetchai/skills/ml_train/skill.yaml b/packages/fetchai/skills/ml_train/skill.yaml index ef75142e4d..49846a1dee 100644 --- a/packages/fetchai/skills/ml_train/skill.yaml +++ b/packages/fetchai/skills/ml_train/skill.yaml @@ -9,9 +9,9 @@ aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmeHcYRhDJi6gqAHpK29UJz3doiyaicoqeTrPddqM3gh64 __init__.py: QmTurqhUDy13mCSrqmbd598Eqz7A5PAmDrL9UpeNg24dDr - behaviours.py: QmXiV4QSb2CfiBMNL8os6GWvtHaKj7herqYUZJ9RcHdQLL + behaviours.py: QmdD8fxpkt5ox6tgqkzST9Giyry5LmiFQGY5PsK8oyKfeV dialogues.py: QmVLXHVwXXLnK4zJzPBVhxzgu7ZmohRZ7oj3yG1LPFrmhx - handlers.py: QmYi7BaxgQcYBhSDbmEDbNcqSYPsSJTYAPQzy2SGTBVAtx + handlers.py: QmeY1m1d7XfX6zUCznjm8sRHpnPCLmo2RmNFfWRk1VsVzN ml_model.py: QmTfshn6dFnz9gKXZt7aJJczRH14bN7nk6TybwFpzkEPnk model.json: QmdV2tGrRY6VQ5VLgUa4yqAhPDG6X8tYsWecypq8nox9Td strategy.py: QmaeAuVjk4aj35qP5ypSf52rVK2uXfQqdMQFRZKH1LgufA diff --git a/packages/hashes.csv b/packages/hashes.csv index 6e087b5582..48fb985adf 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -55,7 +55,7 @@ fetchai/protocols/fipa,QmdZevSTKavH7wCVXeZyiQ5y3sDAkhEQ3SiwhaiXWYgnDJ fetchai/protocols/gym,QmW8fqeoXvsgdjZzZt6Wfj6vfCJBEweAUFdUk1dqNyBa1m fetchai/protocols/http,QmYF5MEFZbJUo2xTLJwCt6cTkADbytUqfeWvsGzvuy6hJF fetchai/protocols/ledger_api,QmWjvwNBpomTDCPEfA86dhHihr2HkCpnveCVZDWU7x5m42 -fetchai/protocols/ml_trade,QmW8pMfbErnFK7vnPAW6ikazRKt1oBvFKPTZUvHzGkW7xr +fetchai/protocols/ml_trade,QmezUacB3b2N31wcs6zxCgurFWubEFJXrfPdaT368LVhTX fetchai/protocols/oef_search,QmQXGHb6GRgeeeYGy5KVPz6bkaWxo3S6tm3fHqoNJ3Zoij fetchai/protocols/prometheus,QmR9KCLCWMMwJRkGV313WGxYwk49bLtg6SxoAavYb3C97h fetchai/protocols/register,QmNwCo2HYKjZzHbjPfmAZnYUBFVURbbFR2JZAye9Wg2WmY @@ -81,7 +81,7 @@ fetchai/skills/generic_seller,QmR7xuDhrnq8gXMe9pk65LykCydkcB8mPh3UgPGZUgvpQ6 fetchai/skills/gym,Qmb4Bhfhhiwu9UsvSyJdLworYm41agCA6WffrYGxN3Sr1x fetchai/skills/http_echo,QmRJRubhKEWPmwxqEac3Zk9AXbAR4J4oB5k56CWuj9zPFE fetchai/skills/ml_data_provider,QmbdLMEdhRuoMi7BRJM1K6K6HKBPb23s6PpYppBuz8vgiE -fetchai/skills/ml_train,QmSYF6dwth9GEpTxrc4LLmrM7LvcvFLCs7Hpim7trHLCuQ +fetchai/skills/ml_train,QmVeM97PgWAELzXA2VTbPPByQcKXsRnfJ6BLVrSTqJd33K fetchai/skills/registration_aw1,QmQiDCMeMnkQ4D3eu1mjAXoJKo51976KrAkfYWWNFKTGnA fetchai/skills/scaffold,QmezXGa1jPSteM9c6iH1hCHJVarqbLaG7mwaJmkkAJ6XUx fetchai/skills/simple_buyer,QmXLnLkkBDMq6KcFZ6XRcNW57djKCcuu7XicJGykcHZCM6 diff --git a/tests/test_packages/test_skills_integration/test_ml_skills.py b/tests/test_packages/test_skills_integration/test_ml_skills.py index e89a61b76d..42231fb22d 100644 --- a/tests/test_packages/test_skills_integration/test_ml_skills.py +++ b/tests/test_packages/test_skills_integration/test_ml_skills.py @@ -18,7 +18,7 @@ # ------------------------------------------------------------------------------ """This test module contains the integration test for the weather skills.""" - +import importlib import sys from random import uniform @@ -41,12 +41,8 @@ def _is_not_tensorflow_installed(): - try: - import tensorflow # noqa - - return False - except ImportError: - return True + tf_spec = importlib.util.find_spec("tensorflow") + return tf_spec is None @pytest.mark.integration From 8e12e6dfe538809a5d9ba059b1771bcbb1d8afc0 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 28 Dec 2020 22:17:20 +0100 Subject: [PATCH 060/204] fix lint --- packages/fetchai/skills/ml_train/behaviours.py | 8 ++++---- packages/fetchai/skills/ml_train/skill.yaml | 2 +- packages/hashes.csv | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/fetchai/skills/ml_train/behaviours.py b/packages/fetchai/skills/ml_train/behaviours.py index 73bc5bee61..57b160dfc2 100644 --- a/packages/fetchai/skills/ml_train/behaviours.py +++ b/packages/fetchai/skills/ml_train/behaviours.py @@ -23,10 +23,6 @@ from aea.protocols.dialogue.base import DialogueLabel from aea.skills.behaviours import TickerBehaviour -from ml_model_trainer.vendor.fetchai.skills.ml_train.dialogues import ( - LedgerApiDialogue, - LedgerApiDialogues, -) from packages.fetchai.connections.ledger.base import ( CONNECTION_ID as LEDGER_CONNECTION_PUBLIC_ID, @@ -34,6 +30,10 @@ from packages.fetchai.protocols.ledger_api import LedgerApiMessage from packages.fetchai.protocols.ml_trade.dialogues import MlTradeDialogue from packages.fetchai.skills.generic_buyer.behaviours import GenericSearchBehaviour +from packages.fetchai.skills.ml_train.dialogues import ( + LedgerApiDialogue, + LedgerApiDialogues, +) from packages.fetchai.skills.ml_train.strategy import Strategy diff --git a/packages/fetchai/skills/ml_train/skill.yaml b/packages/fetchai/skills/ml_train/skill.yaml index 49846a1dee..495dd9d90e 100644 --- a/packages/fetchai/skills/ml_train/skill.yaml +++ b/packages/fetchai/skills/ml_train/skill.yaml @@ -9,7 +9,7 @@ aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmeHcYRhDJi6gqAHpK29UJz3doiyaicoqeTrPddqM3gh64 __init__.py: QmTurqhUDy13mCSrqmbd598Eqz7A5PAmDrL9UpeNg24dDr - behaviours.py: QmdD8fxpkt5ox6tgqkzST9Giyry5LmiFQGY5PsK8oyKfeV + behaviours.py: QmQaDwx8X2RacUSjqwMCiswhoseJcUhUAFhSSKjt9rXJn7 dialogues.py: QmVLXHVwXXLnK4zJzPBVhxzgu7ZmohRZ7oj3yG1LPFrmhx handlers.py: QmeY1m1d7XfX6zUCznjm8sRHpnPCLmo2RmNFfWRk1VsVzN ml_model.py: QmTfshn6dFnz9gKXZt7aJJczRH14bN7nk6TybwFpzkEPnk diff --git a/packages/hashes.csv b/packages/hashes.csv index 48fb985adf..fcbb4ecede 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -81,7 +81,7 @@ fetchai/skills/generic_seller,QmR7xuDhrnq8gXMe9pk65LykCydkcB8mPh3UgPGZUgvpQ6 fetchai/skills/gym,Qmb4Bhfhhiwu9UsvSyJdLworYm41agCA6WffrYGxN3Sr1x fetchai/skills/http_echo,QmRJRubhKEWPmwxqEac3Zk9AXbAR4J4oB5k56CWuj9zPFE fetchai/skills/ml_data_provider,QmbdLMEdhRuoMi7BRJM1K6K6HKBPb23s6PpYppBuz8vgiE -fetchai/skills/ml_train,QmVeM97PgWAELzXA2VTbPPByQcKXsRnfJ6BLVrSTqJd33K +fetchai/skills/ml_train,QmZLvYHGaXiipL1rbR3KPApVnYLh1XofwjpuSJzidHGVeD fetchai/skills/registration_aw1,QmQiDCMeMnkQ4D3eu1mjAXoJKo51976KrAkfYWWNFKTGnA fetchai/skills/scaffold,QmezXGa1jPSteM9c6iH1hCHJVarqbLaG7mwaJmkkAJ6XUx fetchai/skills/simple_buyer,QmXLnLkkBDMq6KcFZ6XRcNW57djKCcuu7XicJGykcHZCM6 From b9ed04a1a70d3db216ad27f217516fdd32a6d8af Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 28 Dec 2020 22:41:15 +0100 Subject: [PATCH 061/204] fix static --- packages/fetchai/skills/ml_train/behaviours.py | 4 ++-- packages/fetchai/skills/ml_train/skill.yaml | 2 +- packages/hashes.csv | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/fetchai/skills/ml_train/behaviours.py b/packages/fetchai/skills/ml_train/behaviours.py index 57b160dfc2..0c8ae3057f 100644 --- a/packages/fetchai/skills/ml_train/behaviours.py +++ b/packages/fetchai/skills/ml_train/behaviours.py @@ -91,9 +91,9 @@ def _start_processing(self) -> None: f"Processing transaction, {len(self.waiting)} transactions remaining" ) strategy = cast(Strategy, self.context.strategy) - proposal = ml_trade_dialogue.last_message.terms + proposal = ml_trade_dialogue.last_message.terms # type: ignore terms = strategy.terms_from_proposal( - proposal, ml_trade_dialogue.last_message.sender + proposal, ml_trade_dialogue.last_message.sender # type: ignore ) ledger_api_dialogues = cast( diff --git a/packages/fetchai/skills/ml_train/skill.yaml b/packages/fetchai/skills/ml_train/skill.yaml index 495dd9d90e..7789304839 100644 --- a/packages/fetchai/skills/ml_train/skill.yaml +++ b/packages/fetchai/skills/ml_train/skill.yaml @@ -9,7 +9,7 @@ aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmeHcYRhDJi6gqAHpK29UJz3doiyaicoqeTrPddqM3gh64 __init__.py: QmTurqhUDy13mCSrqmbd598Eqz7A5PAmDrL9UpeNg24dDr - behaviours.py: QmQaDwx8X2RacUSjqwMCiswhoseJcUhUAFhSSKjt9rXJn7 + behaviours.py: QmdzzpgZiE4gp9CzMUMnzPyhf7SQNUUM45MgsTywhZT6h5 dialogues.py: QmVLXHVwXXLnK4zJzPBVhxzgu7ZmohRZ7oj3yG1LPFrmhx handlers.py: QmeY1m1d7XfX6zUCznjm8sRHpnPCLmo2RmNFfWRk1VsVzN ml_model.py: QmTfshn6dFnz9gKXZt7aJJczRH14bN7nk6TybwFpzkEPnk diff --git a/packages/hashes.csv b/packages/hashes.csv index fcbb4ecede..9b848a836f 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -81,7 +81,7 @@ fetchai/skills/generic_seller,QmR7xuDhrnq8gXMe9pk65LykCydkcB8mPh3UgPGZUgvpQ6 fetchai/skills/gym,Qmb4Bhfhhiwu9UsvSyJdLworYm41agCA6WffrYGxN3Sr1x fetchai/skills/http_echo,QmRJRubhKEWPmwxqEac3Zk9AXbAR4J4oB5k56CWuj9zPFE fetchai/skills/ml_data_provider,QmbdLMEdhRuoMi7BRJM1K6K6HKBPb23s6PpYppBuz8vgiE -fetchai/skills/ml_train,QmZLvYHGaXiipL1rbR3KPApVnYLh1XofwjpuSJzidHGVeD +fetchai/skills/ml_train,QmSSomzaXrpkfpaNdvN5rxCmQcpzMsLFAx1z4uoqudogH9 fetchai/skills/registration_aw1,QmQiDCMeMnkQ4D3eu1mjAXoJKo51976KrAkfYWWNFKTGnA fetchai/skills/scaffold,QmezXGa1jPSteM9c6iH1hCHJVarqbLaG7mwaJmkkAJ6XUx fetchai/skills/simple_buyer,QmXLnLkkBDMq6KcFZ6XRcNW57djKCcuu7XicJGykcHZCM6 From 6f858cbc6de0ac7a69e06fb0d6d73a3d092e6279 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 28 Dec 2020 22:53:52 +0100 Subject: [PATCH 062/204] update public id regex --- docs/language-agnostic-definition.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/language-agnostic-definition.md b/docs/language-agnostic-definition.md index c5ba27371a..56d3f01b1f 100644 --- a/docs/language-agnostic-definition.md +++ b/docs/language-agnostic-definition.md @@ -96,7 +96,7 @@ message DefaultMessage{ } ``` -
  • The protocol id MUST match the following regular expression: ^[a-zA-Z0-9_]*/[a-zA-Z_][a-zA-Z0-9_]*:(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$
    • +
  • The protocol id MUST match the following regular expression: ^([a-zA-Z_][a-zA-Z0-9_]{0,127})/([a-zA-Z_][a-zA-Z0-9_]{0,127})(:((any|latest|((0|[1-9]\d*))\.((0|[1-9]\d*))\.((0|[1-9]\d*))(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)))?$
  • It is recommended that it processes Envelopes asynchronously. Note, the specification regarding the processing of messages does not impose any particular implementation, and the AEA can be designed to process envelopes either synchronously and asynchronously. However, asynchronous message handling enables the agent to be more responsive and scalable in maintaining many concurrent dialogues with its peers.
  • It MUST have an identity in the form of, at a minimum, an address derived from a public key and its associated private key (where the eliptic curve must be of type SECP256k1). From 3c7fb0e35120a76911645e14e72006d40a06fc4d Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 28 Dec 2020 22:59:48 +0100 Subject: [PATCH 063/204] address PR comments --- .../test_bash_yaml/md_files/bash-car-park-skills.md | 10 ---------- .../test_bash_yaml/md_files/bash-ml-skills.md | 10 ---------- 2 files changed, 20 deletions(-) diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md index 3d25dba57c..4808d6837a 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md @@ -77,16 +77,6 @@ aea delete car_detector aea delete car_data_buyer ``` ``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml --- public_id: fetchai/p2p_libp2p:0.13.0 type: connection diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md index 465f527eb2..09aaed5d38 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md @@ -77,16 +77,6 @@ aea delete ml_data_provider aea delete ml_model_trainer ``` ``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml --- public_id: fetchai/p2p_libp2p:0.13.0 type: connection From b63c78de059513b7c3adbe00e25fea430ccddc6d Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 28 Dec 2020 23:09:26 +0100 Subject: [PATCH 064/204] remove 'yml' input as per https://github.com/codecov/codecov-action/issues/51 --- .github/workflows/workflow.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index ff0d907421..81c514e3dc 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -354,5 +354,4 @@ jobs: file: ./coverage.xml flags: unittests name: codecov-umbrella - yml: ./codecov.yml fail_ci_if_error: false From e974c0f46881a32734b6a2d73f0f92d707ebf7ab Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Tue, 29 Dec 2020 12:59:02 +0300 Subject: [PATCH 065/204] fixes --- aea/aea_builder.py | 9 +- aea/cli/utils/config.py | 186 +-------------------- aea/cli/utils/constants.py | 22 --- aea/cli/utils/package_utils.py | 10 +- aea/configurations/manager.py | 266 ++++++++++++++++++++++++------- aea/configurations/validation.py | 12 +- aea/crypto/helpers.py | 60 ++++++- aea/test_tools/generic.py | 2 +- tests/test_cli/test_config.py | 2 +- tests/test_cli/test_transfer.py | 5 +- 10 files changed, 303 insertions(+), 271 deletions(-) diff --git a/aea/aea_builder.py b/aea/aea_builder.py index 8c6f0b0957..d14ac0a6a9 100644 --- a/aea/aea_builder.py +++ b/aea/aea_builder.py @@ -75,6 +75,7 @@ find_component_directory_from_component_id, ) from aea.configurations.pypi import is_satisfiable, merge_dependencies +from aea.crypto.helpers import private_key_verify_or_create from aea.crypto.ledger_apis import DEFAULT_CURRENCY_DENOMINATIONS from aea.crypto.wallet import Wallet from aea.decision_maker.base import DecisionMakerHandler @@ -1556,12 +1557,16 @@ def from_aea_project( load_env_file(str(aea_project_path / DEFAULT_ENV_DOTFILE)) # check and create missing, do not replace env variables. updates config AgentConfigManager.verify_or_create_private_keys( - aea_project_path, substitude_env_vars=False + aea_project_path, + substitude_env_vars=False, + private_key_helper=private_key_verify_or_create, ).dump_config() # just validate agent_configuration = AgentConfigManager.verify_or_create_private_keys( - aea_project_path, substitude_env_vars=True + aea_project_path, + substitude_env_vars=True, + private_key_helper=private_key_verify_or_create, ).agent_config builder = AEABuilder(with_default_packages=False) diff --git a/aea/cli/utils/config.py b/aea/cli/utils/config.py index 8579d6a420..fd42af6007 100644 --- a/aea/cli/utils/config.py +++ b/aea/cli/utils/config.py @@ -39,34 +39,26 @@ import logging.config import os from pathlib import Path -from typing import Dict, List, Optional, Set, Tuple +from typing import Dict, Set import click import jsonschema import yaml -from aea.cli.utils.constants import ( - ALLOWED_PATH_ROOTS, - AUTHOR_KEY, - CLI_CONFIG_PATH, - RESOURCE_TYPE_TO_CONFIG_FILE, -) +from aea.cli.utils.constants import AUTHOR_KEY, CLI_CONFIG_PATH from aea.cli.utils.context import Context from aea.cli.utils.exceptions import AEAConfigException from aea.cli.utils.generic import load_yaml from aea.configurations.base import ( - AgentConfig, - ComponentId, ComponentType, PackageConfiguration, PackageType, - PublicId, _get_default_configuration_file_name_from_type, ) -from aea.configurations.constants import AGENT, AGENTS, DEFAULT_AEA_CONFIG_FILE, VENDOR +from aea.configurations.constants import DEFAULT_AEA_CONFIG_FILE from aea.configurations.loader import ConfigLoader, ConfigLoaders from aea.configurations.validation import ExtraPropertiesError -from aea.exceptions import AEAEnforceError, AEAException, enforce +from aea.exceptions import AEAEnforceError def try_to_load_agent_config( @@ -205,120 +197,6 @@ def dump_item_config( configuration_loader.dump(package_configuration, file_output) # type: ignore -def handle_dotted_path( - value: str, author: str -) -> Tuple[List[str], Path, ConfigLoader, Optional[ComponentId]]: - """Separate the path between path to resource and json path to attribute. - - Allowed values: - 'agent.an_attribute_name' - 'protocols.my_protocol.an_attribute_name' - 'connections.my_connection.an_attribute_name' - 'contracts.my_contract.an_attribute_name' - 'skills.my_skill.an_attribute_name' - 'vendor.author.[protocols|contracts|connections|skills].package_name.attribute_name - - We also return the component id to retrieve the configuration of a specific - component. Notice that at this point we don't know the version, - so we put 'latest' as version, but later we will ignore it because - we will filter with only the component prefix (i.e. the triple type, author and name). - - :param value: dotted path. - :param author: the author string. - - :return: Tuple[list of settings dict keys, filepath, config loader, component id]. - """ - parts = value.split(".") - - root = parts[0] - if root not in ALLOWED_PATH_ROOTS: - raise AEAException( - "The root of the dotted path must be one of: {}".format(ALLOWED_PATH_ROOTS) - ) - - if ( - len(parts) < 2 - or parts[0] == AGENT - and len(parts) < 2 - or parts[0] == VENDOR - and len(parts) < 5 - or parts[0] != AGENT - and len(parts) < 3 - ): - raise AEAException( - "The path is too short. Please specify a path up to an attribute name." - ) - - # if the root is 'agent', stop. - if root == AGENT: - resource_type_plural = AGENTS - path_to_resource_configuration = Path(DEFAULT_AEA_CONFIG_FILE) - json_path = parts[1:] - component_id = None - elif root == VENDOR: - # parse json path - resource_author = parts[1] - resource_type_plural = parts[2] - resource_name = parts[3] - - # extract component id - resource_type_singular = resource_type_plural[:-1] - try: - component_type = ComponentType(resource_type_singular) - except ValueError as e: - raise AEAException( - f"'{resource_type_plural}' is not a valid component type. Please use one of {ComponentType.plurals()}." - ) from e - component_id = ComponentId( - component_type, PublicId(resource_author, resource_name) - ) - - # find path to the resource directory - path_to_resource_directory = ( - Path(".") / VENDOR / resource_author / resource_type_plural / resource_name - ) - path_to_resource_configuration = ( - path_to_resource_directory - / RESOURCE_TYPE_TO_CONFIG_FILE[resource_type_plural] - ) - json_path = parts[4:] - if not path_to_resource_directory.exists(): - raise AEAException( # pragma: nocover - "Resource vendor/{}/{}/{} does not exist.".format( - resource_author, resource_type_plural, resource_name - ) - ) - else: - # navigate the resources of the agent to reach the target configuration file. - resource_type_plural = root - resource_name = parts[1] - - # extract component id - resource_type_singular = resource_type_plural[:-1] - component_type = ComponentType(resource_type_singular) - resource_author = author - component_id = ComponentId( - component_type, PublicId(resource_author, resource_name) - ) - - # find path to the resource directory - path_to_resource_directory = Path(".") / resource_type_plural / resource_name - path_to_resource_configuration = ( - path_to_resource_directory - / RESOURCE_TYPE_TO_CONFIG_FILE[resource_type_plural] - ) - json_path = parts[2:] - if not path_to_resource_directory.exists(): - raise AEAException( - "Resource {}/{} does not exist.".format( - resource_type_plural, resource_name - ) - ) - - config_loader = ConfigLoader.from_configuration_type(resource_type_plural[:-1]) - return json_path, path_to_resource_configuration, config_loader, component_id - - def update_item_config(item_type: str, package_path: Path, **kwargs) -> None: """ Update item config and item config file. @@ -364,62 +242,6 @@ def validate_item_config(item_type: str, package_path: Path) -> None: ) -def _try_get_configuration_object_from_aea_config( - agent_config: AgentConfig, component_id: ComponentId -) -> Optional[Dict]: - """ - Try to get the configuration object in the AEA config. - - The result is not guaranteed because there might not be any - - :param ctx: the CLI context. - :param component_id: the component id whose prefix points to the relevant - custom configuration in the AEA configuration file. - :return: the configuration object to get/set an attribute. - """ - if component_id is None: - # this is the case when the prefix of the json path is 'agent'. - return None # pragma: nocover - type_, author, name = ( - component_id.component_type, - component_id.author, - component_id.name, - ) - component_ids = set(agent_config.component_configurations.keys()) - true_component_id = _try_get_component_id_from_prefix( - component_ids, (type_, author, name) - ) - if true_component_id is not None: - return agent_config.component_configurations.get(true_component_id) - return None - - -def _try_get_component_id_from_prefix( - component_ids: Set[ComponentId], component_prefix: Tuple[ComponentType, str, str] -) -> Optional[ComponentId]: - """ - Find the component id matching a component prefix. - - :param component_ids: the set of component id. - :param component_prefix: the component prefix. - :return: the component id that matches the prefix. - :raises ValueError: if there are more than two components as candidate results. - """ - type_, author, name = component_prefix - results = list( - filter( - lambda x: x.component_type == type_ - and x.author == author - and x.name == name, - component_ids, - ) - ) - if len(results) == 0: - return None - enforce(len(results) == 1, f"Expected only one component, found {len(results)}.") - return results[0] - - def get_non_vendor_package_path(aea_project_path: Path) -> Set[Path]: """ Get all the paths to non-vendor packages. diff --git a/aea/cli/utils/constants.py b/aea/cli/utils/constants.py index 4818e17233..cdec68340a 100644 --- a/aea/cli/utils/constants.py +++ b/aea/cli/utils/constants.py @@ -19,18 +19,12 @@ """Module with constants of the aea cli.""" import os from pathlib import Path -from typing import Dict from aea.configurations.constants import ( - AGENT, CONNECTION, CONNECTIONS, CONTRACT, CONTRACTS, - DEFAULT_CONNECTION_CONFIG_FILE, - DEFAULT_CONTRACT_CONFIG_FILE, - DEFAULT_PROTOCOL_CONFIG_FILE, - DEFAULT_SKILL_CONFIG_FILE, PACKAGES, PROTOCOL, PROTOCOLS, @@ -61,20 +55,4 @@ CONFIG_SUPPORTED_KEY_TYPES = list(FROM_STRING_TO_TYPE.keys()) -ALLOWED_PATH_ROOTS = [ - AGENT, - CONNECTIONS, - CONTRACTS, - PROTOCOLS, - SKILLS, - VENDOR, -] -RESOURCE_TYPE_TO_CONFIG_FILE = { - SKILLS: DEFAULT_SKILL_CONFIG_FILE, - PROTOCOLS: DEFAULT_PROTOCOL_CONFIG_FILE, - CONNECTIONS: DEFAULT_CONNECTION_CONFIG_FILE, - CONTRACTS: DEFAULT_CONTRACT_CONFIG_FILE, -} # type: Dict[str, str] - - REQUIREMENTS = "requirements.txt" diff --git a/aea/cli/utils/package_utils.py b/aea/cli/utils/package_utils.py index 5179bfa5cb..b59a0690c5 100644 --- a/aea/cli/utils/package_utils.py +++ b/aea/cli/utils/package_utils.py @@ -20,6 +20,7 @@ import os import re import shutil +import sys from pathlib import Path from typing import Dict, Optional, Set, Tuple @@ -62,6 +63,7 @@ from aea.configurations.loader import ConfigLoader from aea.configurations.manager import AgentConfigManager from aea.configurations.utils import replace_component_ids +from aea.crypto.helpers import private_key_verify_or_create from aea.crypto.ledger_apis import DEFAULT_LEDGER_CONFIGS, LedgerApis from aea.crypto.wallet import Wallet from aea.exceptions import AEAEnforceError @@ -83,14 +85,18 @@ def verify_or_create_private_keys_ctx( """ try: AgentConfigManager.verify_or_create_private_keys( - aea_project_path, exit_on_error, substitude_env_vars=False + aea_project_path, + private_key_helper=private_key_verify_or_create, + substitude_env_vars=False, ).dump_config() agent_config = AgentConfigManager.verify_or_create_private_keys( - aea_project_path, exit_on_error + aea_project_path, private_key_helper=private_key_verify_or_create ).agent_config if ctx is not None: ctx.agent_config = agent_config except ValueError as e: # pragma: nocover + if exit_on_error: + sys.exit(1) click.ClickException(str(e)) diff --git a/aea/configurations/manager.py b/aea/configurations/manager.py index ae7461c002..cfbc5e9739 100644 --- a/aea/configurations/manager.py +++ b/aea/configurations/manager.py @@ -16,18 +16,13 @@ # limitations under the License. # # ------------------------------------------------------------------------------ -"""Implementation of the 'aea config' subcommand.""" +"""Implementation of the AgentConfigManager.""" import json import os from copy import deepcopy from pathlib import Path -from typing import Dict, List, NewType, Optional, Tuple, Union, cast +from typing import Callable, Dict, List, NewType, Optional, Set, Tuple, Union, cast -from aea.cli.utils.config import ( - _try_get_component_id_from_prefix, - _try_get_configuration_object_from_aea_config, - handle_dotted_path, -) from aea.configurations.base import ( AgentConfig, ComponentConfiguration, @@ -35,15 +30,45 @@ DEFAULT_AEA_CONFIG_FILE, PackageType, ) -from aea.configurations.constants import PRIVATE_KEY_PATH_SCHEMA, VENDOR +from aea.configurations.constants import ( + AGENT, + AGENTS, + CONNECTIONS, + CONTRACTS, + DEFAULT_CONNECTION_CONFIG_FILE, + DEFAULT_CONTRACT_CONFIG_FILE, + DEFAULT_PROTOCOL_CONFIG_FILE, + DEFAULT_SKILL_CONFIG_FILE, + PROTOCOLS, + SKILLS, + VENDOR, +) +from aea.configurations.data_types import ComponentType, PublicId from aea.configurations.loader import ConfigLoader, load_component_configuration -from aea.crypto.helpers import create_private_key, try_validate_private_key_path -from aea.crypto.registries import crypto_registry -from aea.helpers.env_vars import apply_env_variables, is_env_variable +from aea.exceptions import AEAException, enforce +from aea.helpers.env_vars import apply_env_variables from aea.helpers.storage.backends.base import JSON_TYPES from aea.helpers.yaml_utils import yaml_load_all +ALLOWED_PATH_ROOTS = [ + AGENT, + CONNECTIONS, + CONTRACTS, + PROTOCOLS, + SKILLS, + VENDOR, +] + + +RESOURCE_TYPE_TO_CONFIG_FILE = { + SKILLS: DEFAULT_SKILL_CONFIG_FILE, + PROTOCOLS: DEFAULT_PROTOCOL_CONFIG_FILE, + CONNECTIONS: DEFAULT_CONNECTION_CONFIG_FILE, + CONTRACTS: DEFAULT_CONTRACT_CONFIG_FILE, +} # type: Dict[str, str] + + JsonPath = List[str] VariablePath = Union[str, JsonPath] @@ -56,6 +81,176 @@ class VariableDoesNotExist(ValueError): NotExists = NotExistsType(None) +def _try_get_configuration_object_from_aea_config( + agent_config: AgentConfig, component_id: ComponentId +) -> Optional[Dict]: + """ + Try to get the configuration object in the AEA config. + + The result is not guaranteed because there might not be any + + :param ctx: the CLI context. + :param component_id: the component id whose prefix points to the relevant + custom configuration in the AEA configuration file. + :return: the configuration object to get/set an attribute. + """ + if component_id is None: + # this is the case when the prefix of the json path is 'agent'. + return None # pragma: nocover + type_, author, name = ( + component_id.component_type, + component_id.author, + component_id.name, + ) + component_ids = set(agent_config.component_configurations.keys()) + true_component_id = _try_get_component_id_from_prefix( + component_ids, (type_, author, name) + ) + if true_component_id is not None: + return agent_config.component_configurations.get(true_component_id) + return None + + +def _try_get_component_id_from_prefix( + component_ids: Set[ComponentId], component_prefix: Tuple[ComponentType, str, str] +) -> Optional[ComponentId]: + """ + Find the component id matching a component prefix. + + :param component_ids: the set of component id. + :param component_prefix: the component prefix. + :return: the component id that matches the prefix. + :raises ValueError: if there are more than two components as candidate results. + """ + type_, author, name = component_prefix + results = list( + filter( + lambda x: x.component_type == type_ + and x.author == author + and x.name == name, + component_ids, + ) + ) + if len(results) == 0: + return None + enforce(len(results) == 1, f"Expected only one component, found {len(results)}.") + return results[0] + + +def handle_dotted_path( + value: str, author: str +) -> Tuple[List[str], Path, ConfigLoader, Optional[ComponentId]]: + """Separate the path between path to resource and json path to attribute. + + Allowed values: + 'agent.an_attribute_name' + 'protocols.my_protocol.an_attribute_name' + 'connections.my_connection.an_attribute_name' + 'contracts.my_contract.an_attribute_name' + 'skills.my_skill.an_attribute_name' + 'vendor.author.[protocols|contracts|connections|skills].package_name.attribute_name + + We also return the component id to retrieve the configuration of a specific + component. Notice that at this point we don't know the version, + so we put 'latest' as version, but later we will ignore it because + we will filter with only the component prefix (i.e. the triple type, author and name). + + :param value: dotted path. + :param author: the author string. + + :return: Tuple[list of settings dict keys, filepath, config loader, component id]. + """ + parts = value.split(".") + + root = parts[0] + if root not in ALLOWED_PATH_ROOTS: + raise AEAException( + "The root of the dotted path must be one of: {}".format(ALLOWED_PATH_ROOTS) + ) + + if ( + len(parts) < 2 + or parts[0] == AGENT + and len(parts) < 2 + or parts[0] == VENDOR + and len(parts) < 5 + or parts[0] != AGENT + and len(parts) < 3 + ): + raise AEAException( + "The path is too short. Please specify a path up to an attribute name." + ) + + # if the root is 'agent', stop. + if root == AGENT: + resource_type_plural = AGENTS + path_to_resource_configuration = Path(DEFAULT_AEA_CONFIG_FILE) + json_path = parts[1:] + component_id = None + elif root == VENDOR: + # parse json path + resource_author = parts[1] + resource_type_plural = parts[2] + resource_name = parts[3] + + # extract component id + resource_type_singular = resource_type_plural[:-1] + try: + component_type = ComponentType(resource_type_singular) + except ValueError as e: + raise AEAException( + f"'{resource_type_plural}' is not a valid component type. Please use one of {ComponentType.plurals()}." + ) from e + component_id = ComponentId( + component_type, PublicId(resource_author, resource_name) + ) + + # find path to the resource directory + path_to_resource_directory = ( + Path(".") / VENDOR / resource_author / resource_type_plural / resource_name + ) + path_to_resource_configuration = ( + path_to_resource_directory + / RESOURCE_TYPE_TO_CONFIG_FILE[resource_type_plural] + ) + json_path = parts[4:] + if not path_to_resource_directory.exists(): + raise AEAException( # pragma: nocover + "Resource vendor/{}/{}/{} does not exist.".format( + resource_author, resource_type_plural, resource_name + ) + ) + else: + # navigate the resources of the agent to reach the target configuration file. + resource_type_plural = root + resource_name = parts[1] + + # extract component id + resource_type_singular = resource_type_plural[:-1] + component_type = ComponentType(resource_type_singular) + resource_author = author + component_id = ComponentId( + component_type, PublicId(resource_author, resource_name) + ) + + # find path to the resource directory + path_to_resource_directory = Path(".") / resource_type_plural / resource_name + path_to_resource_configuration = ( + path_to_resource_directory + / RESOURCE_TYPE_TO_CONFIG_FILE[resource_type_plural] + ) + json_path = parts[2:] + if not path_to_resource_directory.exists(): + raise AEAException( + "Resource {}/{} does not exist.".format( + resource_type_plural, resource_name + ) + ) + + config_loader = ConfigLoader.from_configuration_type(resource_type_plural[:-1]) + return json_path, path_to_resource_configuration, config_loader, component_id + + def find_component_directory_from_component_id( aea_project_directory: Path, component_id: ComponentId ) -> Path: @@ -324,7 +519,7 @@ def dump_config(self) -> None: def verify_or_create_private_keys( cls, aea_project_path: Union[Path, str], - exit_on_error: bool = True, + private_key_helper: Callable[[AgentConfig, Path], None], substitude_env_vars: bool = False, ) -> "AgentConfigManager": """ @@ -333,7 +528,7 @@ def verify_or_create_private_keys( Does not saves the config! Use AgentConfigManager.dump_config() :param aea_project_path: path to an AEA project. - :param exit_on_error: whether we should exit the program on error. + :param private_key_helper: private_key_helper is a function that use agent config to check the keys :param substitude_env_vars: replace env vars with values, does not dump config :return: the agent configuration manager. @@ -343,50 +538,7 @@ def verify_or_create_private_keys( aea_project_path, substitude_env_vars=substitude_env_vars ) aea_conf = agent_config_manager.agent_config - - for identifier, _ in aea_conf.private_key_paths.read_all(): - if identifier not in crypto_registry.supported_ids: # pragma: nocover - raise ValueError( - "Unsupported identifier `{}` in private key paths. Supported identifiers: {}.".format( - identifier, sorted(crypto_registry.supported_ids) - ) - ) - - for identifier in crypto_registry.supported_ids: - config_private_key_path = aea_conf.private_key_paths.read(identifier) - - if is_env_variable(config_private_key_path): - # skip env var definition - continue - - if config_private_key_path is None: - private_key_path = PRIVATE_KEY_PATH_SCHEMA.format(identifier) - if identifier == aea_conf.default_ledger: # pragma: nocover - if os.path.exists(private_key_path): - raise ValueError( - "File {} for private key {} already exists. Add to aea-config.yaml.".format( - repr(config_private_key_path), identifier - ) - ) - create_private_key( - identifier, - private_key_file=str(aea_project_path / private_key_path), - ) - aea_conf.private_key_paths.update(identifier, private_key_path) - else: - try: - try_validate_private_key_path( - identifier, - str(aea_project_path / config_private_key_path), - exit_on_error=exit_on_error, - ) - except FileNotFoundError: # pragma: no cover - raise ValueError( - "File {} for private key {} not found.".format( - repr(config_private_key_path), identifier, - ) - ) - + private_key_helper(aea_conf, Path(aea_project_path)) return agent_config_manager def get_overridables(self) -> Tuple[Dict, Dict[ComponentId, Dict]]: diff --git a/aea/configurations/validation.py b/aea/configurations/validation.py index 49ee51fa47..e64d9e66c6 100644 --- a/aea/configurations/validation.py +++ b/aea/configurations/validation.py @@ -183,6 +183,8 @@ def validate_component_configuration( :param component_id: the component id. :param configuration: the configuration dictionary. + :param env_vars_friendly: bool, if set True, will not raise errors over the env variable definitions. + :return: None :raises ValueError: if the configuration is not valid. """ @@ -256,7 +258,15 @@ def validate_data_with_pattern( excludes: Optional[List[Tuple[str]]] = None, skip_env_vars: bool = False, ) -> List[str]: - """Validate data dict with pattern dict for attributes present and type match.""" + """ + Validate data dict with pattern dict for attributes present and type match. + + :param pattern: dict with pattern to check over + :param excludes: list of tuples of str of paths to be skipped during the check + :param skip_env_vars: is set True will not check data type over env variables. + + :return: list of str with error descriptions + """ excludes = excludes or [] pattern_path_value = { tuple(path): value for path, value in dict_to_path_value(pattern) diff --git a/aea/crypto/helpers.py b/aea/crypto/helpers.py index 16d01353e0..4bd8e2b626 100644 --- a/aea/crypto/helpers.py +++ b/aea/crypto/helpers.py @@ -16,14 +16,17 @@ # limitations under the License. # # ------------------------------------------------------------------------------ - """Module wrapping the helpers of public and private key cryptography.""" import logging +import os import sys +from pathlib import Path from typing import Optional +from aea.configurations.base import AgentConfig from aea.configurations.constants import PRIVATE_KEY_PATH_SCHEMA -from aea.crypto.registries import make_crypto, make_faucet_api +from aea.crypto.registries import crypto_registry, make_crypto, make_faucet_api +from aea.helpers.env_vars import is_env_variable _default_logger = logging.getLogger(__name__) @@ -87,3 +90,56 @@ def try_generate_testnet_wealth( faucet_api = make_faucet_api(identifier) if faucet_api is not None: faucet_api.get_wealth(address, url) + + +def private_key_verify_or_create(aea_conf: AgentConfig, aea_project_path: Path) -> None: + """ + Check key or create if none present. + + :param aea_conf: AgentConfig + :param aea_project_path: Path, where project placed. + + :return: None + """ + for identifier, _ in aea_conf.private_key_paths.read_all(): + if identifier not in crypto_registry.supported_ids: # pragma: nocover + raise ValueError( + "Unsupported identifier `{}` in private key paths. Supported identifiers: {}.".format( + identifier, sorted(crypto_registry.supported_ids) + ) + ) + + for identifier in crypto_registry.supported_ids: + config_private_key_path = aea_conf.private_key_paths.read(identifier) + + if is_env_variable(config_private_key_path): + # config_private_key_path is env vaariable to be used, skip it. check will be performed after substitution + continue + + if config_private_key_path is None: + private_key_path = PRIVATE_KEY_PATH_SCHEMA.format(identifier) + if identifier == aea_conf.default_ledger: # pragma: nocover + if os.path.exists(private_key_path): + raise ValueError( + "File {} for private key {} already exists. Add to aea-config.yaml.".format( + repr(config_private_key_path), identifier + ) + ) + create_private_key( + identifier, + private_key_file=str(aea_project_path / private_key_path), + ) + aea_conf.private_key_paths.update(identifier, private_key_path) + else: + try: + try_validate_private_key_path( + identifier, + str(aea_project_path / config_private_key_path), + exit_on_error=False, # do not exit process + ) + except FileNotFoundError: # pragma: no cover + raise ValueError( + "File {} for private key {} not found.".format( + repr(config_private_key_path), identifier, + ) + ) diff --git a/aea/test_tools/generic.py b/aea/test_tools/generic.py index 4f8ebb1d3c..188e2517a8 100644 --- a/aea/test_tools/generic.py +++ b/aea/test_tools/generic.py @@ -22,7 +22,6 @@ from pathlib import Path from typing import Any, Dict, List, cast -from aea.cli.utils.config import handle_dotted_path from aea.configurations.base import ( CRUDCollection, ComponentConfiguration, @@ -32,6 +31,7 @@ SkillConfig, dependencies_from_json, ) +from aea.configurations.manager import handle_dotted_path from aea.exceptions import enforce from aea.helpers.file_io import write_envelope from aea.helpers.yaml_utils import yaml_dump, yaml_dump_all diff --git a/tests/test_cli/test_config.py b/tests/test_cli/test_config.py index 27fdf2d656..31c59f7713 100644 --- a/tests/test_cli/test_config.py +++ b/tests/test_cli/test_config.py @@ -28,9 +28,9 @@ from aea.aea_builder import AEABuilder from aea.cli import cli from aea.cli.config import AgentConfigManager -from aea.cli.utils.constants import ALLOWED_PATH_ROOTS from aea.configurations.base import AgentConfig, DEFAULT_AEA_CONFIG_FILE, PackageType from aea.configurations.loader import ConfigLoader +from aea.configurations.manager import ALLOWED_PATH_ROOTS from aea.helpers.yaml_utils import yaml_load from tests.conftest import CLI_LOG_OPTION, CUR_PATH, CliRunner, ROOT_DIR diff --git a/tests/test_cli/test_transfer.py b/tests/test_cli/test_transfer.py index 509edc81ca..c076be39e3 100644 --- a/tests/test_cli/test_transfer.py +++ b/tests/test_cli/test_transfer.py @@ -30,6 +30,7 @@ from aea.configurations.manager import AgentConfigManager from aea.crypto.cosmos import CosmosCrypto from aea.crypto.fetchai import FetchAICrypto +from aea.crypto.helpers import private_key_verify_or_create from aea.helpers.base import cd from aea.test_tools.test_cases import AEATestCaseEmpty @@ -76,7 +77,9 @@ def get_balance(self) -> int: """Get balance for current agent.""" with cd(self._get_cwd()): agent_config = AgentConfigManager.verify_or_create_private_keys( - Path("."), False, substitude_env_vars=False + Path("."), + substitude_env_vars=False, + private_key_helper=private_key_verify_or_create, ).agent_config wallet = get_wallet_from_agent_config(agent_config) return int(try_get_balance(agent_config, wallet, self.LEDGER_ID)) From 0e56964a7780babb1c536f23d8659fca94bfe797 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 29 Dec 2020 11:07:01 +0100 Subject: [PATCH 066/204] add docs tests for language agnostic definition docs --- .../bash-language-agnostic-definition.md | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 tests/test_docs/test_bash_yaml/md_files/bash-language-agnostic-definition.md diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-language-agnostic-definition.md b/tests/test_docs/test_bash_yaml/md_files/bash-language-agnostic-definition.md new file mode 100644 index 0000000000..aace3deee7 --- /dev/null +++ b/tests/test_docs/test_bash_yaml/md_files/bash-language-agnostic-definition.md @@ -0,0 +1,72 @@ +``` proto +syntax = "proto3"; + +package aea; + +message Envelope{ + string to = 1; + string sender = 2; + string protocol_id = 3; + bytes message = 4; + string uri = 5; +} +``` +``` proto +import "google/protobuf/struct.proto"; + +message DialogueMessage { + int32 message_id = 1; + string dialogue_starter_reference = 2; + string dialogue_responder_reference = 3; + int32 target = 4; + bytes content = 5; +} + +message Message { + oneof message { + google.protobuf.Struct body = 1; + DialogueMessage dialogue_message = 2; + } +} +``` +``` proto +syntax = "proto3"; + +package aea.fetchai.default; + +message DefaultMessage{ + + // Custom Types + message ErrorCode{ + enum ErrorCodeEnum { + UNSUPPORTED_PROTOCOL = 0; + DECODING_ERROR = 1; + INVALID_MESSAGE = 2; + UNSUPPORTED_SKILL = 3; + INVALID_DIALOGUE = 4; + } + ErrorCodeEnum error_code = 1; + } + + + // Performatives and contents + message Bytes_Performative{ + bytes content = 1; + } + + message Error_Performative{ + ErrorCode error_code = 1; + string error_msg = 2; + map error_data = 3; + } + + message End_Performative{} + + + oneof performative{ + Bytes_Performative bytes = 5; + End_Performative end = 6; + Error_Performative error = 7; + } +} +``` \ No newline at end of file From ed6bf0bfabe76d79411d7b7cbc0650ab7f57056b Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 29 Dec 2020 11:34:35 +0100 Subject: [PATCH 067/204] add more consistency tests --- docs/language-agnostic-definition.md | 1 + .../test_language_agnostic_definition.py | 97 +++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 tests/test_docs/test_language_agnostic_definition.py diff --git a/docs/language-agnostic-definition.md b/docs/language-agnostic-definition.md index 56d3f01b1f..2309c554f8 100644 --- a/docs/language-agnostic-definition.md +++ b/docs/language-agnostic-definition.md @@ -32,6 +32,7 @@ The format for the above fields, except message, is specified below ``` proto import "google/protobuf/struct.proto"; + message DialogueMessage { int32 message_id = 1; string dialogue_starter_reference = 2; diff --git a/tests/test_docs/test_language_agnostic_definition.py b/tests/test_docs/test_language_agnostic_definition.py new file mode 100644 index 0000000000..aadf480c24 --- /dev/null +++ b/tests/test_docs/test_language_agnostic_definition.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the tests for the content of language-agnostic-definition.md file.""" +from pathlib import Path +from typing import Dict + +import mistune + +from aea import AEA_DIR +from aea.configurations.data_types import PublicId + +from tests.conftest import ROOT_DIR + + +MAIL_BASE_PROTO = Path(AEA_DIR) / "mail" / "base.proto" +DEFAULT_MESSAGE_PROTO = ( + Path(ROOT_DIR) / "packages" / "fetchai" / "protocols" / "default" / "default.proto" +) + + +class TestLanguageAgnosticDocs: + """Test the integrity of the language agnostic definitions in docs.""" + + @classmethod + def _proto_snippet_selector(cls, block: Dict) -> bool: + return block["type"] == "block_code" and block["info"].strip() == "proto" + + @classmethod + def setup_class(cls): + """Set up the test.""" + markdown_parser = mistune.create_markdown(renderer=mistune.AstRenderer()) + + cls.spec_doc_file = Path(ROOT_DIR, "docs", "language-agnostic-definition.md") + cls.spec_doc_content = cls.spec_doc_file.read_text() + doc = markdown_parser(cls.spec_doc_content) + # get only code blocks + cls.code_blocks = list(filter(cls._proto_snippet_selector, doc)) + + cls.actual_mail_base_file_content = MAIL_BASE_PROTO.read_text() + cls.actual_default_message_file_content = DEFAULT_MESSAGE_PROTO.read_text() + + def test_envelope_code_snippet(self): + """ + Test the Envelope protobuf code snippet. + + It requires treating the preamble lines separately, + because aea/mail/base.proto doesn't follow + the same order of language-agnostic-definition.md file. + """ + block = self.code_blocks[0] + assert block["info"].strip() == "proto" + + lines = block["text"].splitlines() + first_part, second_part = "\n".join(lines[:3]), "\n".join(lines[3:]) + + assert first_part in self.actual_mail_base_file_content + assert second_part in self.actual_mail_base_file_content + + def test_dialogue_message_code_snippet(self): + """ + Test the DialogueMessage protobuf code snippet. + + It requires treating the preamble lines separately, + because aea/mail/base.proto doesn't follow + the same order of language-agnostic-definition.md file. + """ + block = self.code_blocks[1] + assert block["info"].strip() == "proto" + assert block["text"] in self.actual_mail_base_file_content + + def test_public_id_regular_expression(self): + """Test public id regular expression is the same.""" + expected_regex = PublicId.PUBLIC_ID_REGEX + assert expected_regex in self.spec_doc_content + + def test_default_message_code_snippet(self): + """Test DefaultMessage protobuf code snippet.""" + block = self.code_blocks[2] + assert block["info"].strip() == "proto" + assert block["text"] in self.actual_default_message_file_content From 019d4dfe3e0b7b097881fcdb86595be7a3585f70 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 29 Dec 2020 11:49:24 +0100 Subject: [PATCH 068/204] fix aea builder changes --- aea/aea_builder.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/aea/aea_builder.py b/aea/aea_builder.py index 65fbd4402f..0cf3ee4067 100644 --- a/aea/aea_builder.py +++ b/aea/aea_builder.py @@ -1580,23 +1580,16 @@ def from_aea_project( :return: an AEABuilder. """ aea_project_path = Path(aea_project_path) - - cls._try_to_load_agent_configuration_file(aea_project_path) - if verify_or_create_keys: - verify_or_create_private_keys( - aea_project_path=aea_project_path, exit_on_error=False - ) - builder = AEABuilder(with_default_packages=False) - load_env_file(str(aea_project_path / DEFAULT_ENV_DOTFILE)) # load to verify agent_configuration = cls.try_to_load_agent_configuration_file(aea_project_path) logging.config.dictConfig(agent_configuration.logging_config) # type: ignore - verify_or_create_private_keys( - aea_project_path=aea_project_path, exit_on_error=False - ) + if verify_or_create_keys: + verify_or_create_private_keys( + aea_project_path=aea_project_path, exit_on_error=False + ) # load agent configuration file agent_configuration = cls.try_to_load_agent_configuration_file(aea_project_path) From ce95663c8f940b5b5dc4d49bc679027ae2d836e3 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 29 Dec 2020 12:26:23 +0100 Subject: [PATCH 069/204] update ml skill to use tensorflow~=2.4.0 --- .../skills/ml_data_provider/skill.yaml | 2 +- packages/fetchai/skills/ml_train/ml_model.py | 33 +++++++++---------- packages/fetchai/skills/ml_train/skill.yaml | 2 +- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/packages/fetchai/skills/ml_data_provider/skill.yaml b/packages/fetchai/skills/ml_data_provider/skill.yaml index a8442a9382..9fea51b684 100644 --- a/packages/fetchai/skills/ml_data_provider/skill.yaml +++ b/packages/fetchai/skills/ml_data_provider/skill.yaml @@ -74,5 +74,5 @@ models: dependencies: numpy: {} tensorflow: - version: ==1.14.0 + version: ~=2.4.0 is_abstract: false diff --git a/packages/fetchai/skills/ml_train/ml_model.py b/packages/fetchai/skills/ml_train/ml_model.py index 24b4e56af6..4e5c8da27f 100644 --- a/packages/fetchai/skills/ml_train/ml_model.py +++ b/packages/fetchai/skills/ml_train/ml_model.py @@ -23,7 +23,6 @@ from queue import Queue import tensorflow as tf -from tensorflow import keras from aea.skills.base import Model @@ -48,7 +47,6 @@ def __init__(self, **kwargs): self._lock = threading.RLock() self._weights = None - self.graph = tf.get_default_graph() self.data_queue = Queue() self.training_thread = threading.Thread(target=self.training_loop) @@ -66,28 +64,27 @@ def training_loop(self): :return: None """ - with self.graph.as_default(): - model = self._make_model() + model = self._make_model() + self._set_weights(model.get_weights()) + while True: + data = self.data_queue.get() + if data is None: + break + + X, y, kwargs = data + model.fit(X, y, **kwargs) + loss, acc = model.evaluate(X, y, verbose=2) + self.context.logger.info("Loss: {}, Acc: {}".format(loss, acc)) self._set_weights(model.get_weights()) - while True: - data = self.data_queue.get() - if data is None: - break - - X, y, kwargs = data - model.fit(X, y, **kwargs) - loss, acc = model.evaluate(X, y, verbose=2) - self.context.logger.info("Loss: {}, Acc: {}".format(loss, acc)) - self._set_weights(model.get_weights()) @staticmethod def _make_model(): """Make the model.""" - model = keras.Sequential( + model = tf.keras.Sequential( [ - keras.layers.Flatten(input_shape=(28, 28)), - keras.layers.Dense(128, activation="relu"), - keras.layers.Dense(10, activation="softmax"), + tf.keras.layers.Flatten(input_shape=(28, 28)), + tf.keras.layers.Dense(128, activation="relu"), + tf.keras.layers.Dense(10, activation="softmax"), ] ) model.compile( diff --git a/packages/fetchai/skills/ml_train/skill.yaml b/packages/fetchai/skills/ml_train/skill.yaml index 7789304839..8775cb9cb3 100644 --- a/packages/fetchai/skills/ml_train/skill.yaml +++ b/packages/fetchai/skills/ml_train/skill.yaml @@ -91,5 +91,5 @@ models: dependencies: numpy: {} tensorflow: - version: ==1.14.0 + version: ~=2.4.0 is_abstract: false From 52572c0bf16a8305b6c921f4a3535741badc4937 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 29 Dec 2020 12:28:00 +0100 Subject: [PATCH 070/204] update hashes --- packages/fetchai/skills/ml_train/skill.yaml | 2 +- packages/hashes.csv | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/fetchai/skills/ml_train/skill.yaml b/packages/fetchai/skills/ml_train/skill.yaml index 8775cb9cb3..495ed3e24d 100644 --- a/packages/fetchai/skills/ml_train/skill.yaml +++ b/packages/fetchai/skills/ml_train/skill.yaml @@ -12,7 +12,7 @@ fingerprint: behaviours.py: QmdzzpgZiE4gp9CzMUMnzPyhf7SQNUUM45MgsTywhZT6h5 dialogues.py: QmVLXHVwXXLnK4zJzPBVhxzgu7ZmohRZ7oj3yG1LPFrmhx handlers.py: QmeY1m1d7XfX6zUCznjm8sRHpnPCLmo2RmNFfWRk1VsVzN - ml_model.py: QmTfshn6dFnz9gKXZt7aJJczRH14bN7nk6TybwFpzkEPnk + ml_model.py: QmYy293Zq3gKdJaraNNGSVsVMjf82DkAUHnz5SdAFmySMQ model.json: QmdV2tGrRY6VQ5VLgUa4yqAhPDG6X8tYsWecypq8nox9Td strategy.py: QmaeAuVjk4aj35qP5ypSf52rVK2uXfQqdMQFRZKH1LgufA tasks.py: QmahJRCf6V61FsqrKgMMUyJ8F7PRd6C2bjunZg2XtM9fpF diff --git a/packages/hashes.csv b/packages/hashes.csv index 9b848a836f..5c18db2a6e 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -80,8 +80,8 @@ fetchai/skills/generic_buyer,QmNRP6rWnTNMwDYygzzN6BY7kUxh2DMYrjNaQVd19M4Nv4 fetchai/skills/generic_seller,QmR7xuDhrnq8gXMe9pk65LykCydkcB8mPh3UgPGZUgvpQ6 fetchai/skills/gym,Qmb4Bhfhhiwu9UsvSyJdLworYm41agCA6WffrYGxN3Sr1x fetchai/skills/http_echo,QmRJRubhKEWPmwxqEac3Zk9AXbAR4J4oB5k56CWuj9zPFE -fetchai/skills/ml_data_provider,QmbdLMEdhRuoMi7BRJM1K6K6HKBPb23s6PpYppBuz8vgiE -fetchai/skills/ml_train,QmSSomzaXrpkfpaNdvN5rxCmQcpzMsLFAx1z4uoqudogH9 +fetchai/skills/ml_data_provider,QmYy8PD87WjiFiXT76weLgksRBqZuV4DJDXjsYWboRuFDe +fetchai/skills/ml_train,QmQzvFUXsJgQTS4XkZmVMGRdfgtyL1V5FjLEFe87NHbfwp fetchai/skills/registration_aw1,QmQiDCMeMnkQ4D3eu1mjAXoJKo51976KrAkfYWWNFKTGnA fetchai/skills/scaffold,QmezXGa1jPSteM9c6iH1hCHJVarqbLaG7mwaJmkkAJ6XUx fetchai/skills/simple_buyer,QmXLnLkkBDMq6KcFZ6XRcNW57djKCcuu7XicJGykcHZCM6 From 6f335d6f58977d1482b0342d6583b770ef44ecc9 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 29 Dec 2020 12:36:20 +0100 Subject: [PATCH 071/204] fix pylint checks --- packages/fetchai/skills/ml_train/ml_model.py | 18 ++++++++---------- packages/fetchai/skills/ml_train/skill.yaml | 2 +- packages/hashes.csv | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/fetchai/skills/ml_train/ml_model.py b/packages/fetchai/skills/ml_train/ml_model.py index 4e5c8da27f..384a70fc71 100644 --- a/packages/fetchai/skills/ml_train/ml_model.py +++ b/packages/fetchai/skills/ml_train/ml_model.py @@ -107,20 +107,18 @@ def _set_weights(self, weights): def predict(self, *args, **kwargs): """Predict.""" with self._lock: - with self.graph.as_default(): - model = self._make_model() - weights = self._get_weights() - model.set_weights(weights) - return model.predict(*args, **kwargs) + model = self._make_model() + weights = self._get_weights() + model.set_weights(weights) + return model.predict(*args, **kwargs) def evaluate(self, *args, **kwargs): """Predict.""" with self._lock: - with self.graph.as_default(): - model = self._make_model() - weights = self._get_weights() - model.set_weights(weights) - return model.evaluate(*args, **kwargs) + model = self._make_model() + weights = self._get_weights() + model.set_weights(weights) + return model.evaluate(*args, **kwargs) def save(self): """Save the model weights.""" diff --git a/packages/fetchai/skills/ml_train/skill.yaml b/packages/fetchai/skills/ml_train/skill.yaml index 495ed3e24d..3aa75aecb5 100644 --- a/packages/fetchai/skills/ml_train/skill.yaml +++ b/packages/fetchai/skills/ml_train/skill.yaml @@ -12,7 +12,7 @@ fingerprint: behaviours.py: QmdzzpgZiE4gp9CzMUMnzPyhf7SQNUUM45MgsTywhZT6h5 dialogues.py: QmVLXHVwXXLnK4zJzPBVhxzgu7ZmohRZ7oj3yG1LPFrmhx handlers.py: QmeY1m1d7XfX6zUCznjm8sRHpnPCLmo2RmNFfWRk1VsVzN - ml_model.py: QmYy293Zq3gKdJaraNNGSVsVMjf82DkAUHnz5SdAFmySMQ + ml_model.py: QmYDhwerUrJBMXxTkkZcG1fD1tcZvraPueJNNnNiVdBYEJ model.json: QmdV2tGrRY6VQ5VLgUa4yqAhPDG6X8tYsWecypq8nox9Td strategy.py: QmaeAuVjk4aj35qP5ypSf52rVK2uXfQqdMQFRZKH1LgufA tasks.py: QmahJRCf6V61FsqrKgMMUyJ8F7PRd6C2bjunZg2XtM9fpF diff --git a/packages/hashes.csv b/packages/hashes.csv index 5c18db2a6e..22628f8bdc 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -81,7 +81,7 @@ fetchai/skills/generic_seller,QmR7xuDhrnq8gXMe9pk65LykCydkcB8mPh3UgPGZUgvpQ6 fetchai/skills/gym,Qmb4Bhfhhiwu9UsvSyJdLworYm41agCA6WffrYGxN3Sr1x fetchai/skills/http_echo,QmRJRubhKEWPmwxqEac3Zk9AXbAR4J4oB5k56CWuj9zPFE fetchai/skills/ml_data_provider,QmYy8PD87WjiFiXT76weLgksRBqZuV4DJDXjsYWboRuFDe -fetchai/skills/ml_train,QmQzvFUXsJgQTS4XkZmVMGRdfgtyL1V5FjLEFe87NHbfwp +fetchai/skills/ml_train,QmXHKNeerdYnoCDKCEmkHH2j6qiyMSbWNPnqyWb6vRU2d5 fetchai/skills/registration_aw1,QmQiDCMeMnkQ4D3eu1mjAXoJKo51976KrAkfYWWNFKTGnA fetchai/skills/scaffold,QmezXGa1jPSteM9c6iH1hCHJVarqbLaG7mwaJmkkAJ6XUx fetchai/skills/simple_buyer,QmXLnLkkBDMq6KcFZ6XRcNW57djKCcuu7XicJGykcHZCM6 From cce80044efa6b3a7492c320c309ae28536b3a6be Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Tue, 29 Dec 2020 11:40:38 +0000 Subject: [PATCH 072/204] fix ml skills --- aea/exceptions.py | 2 + .../skills/ml_data_provider/skill.yaml | 3 +- .../skills/ml_data_provider/strategy.py | 23 +++- .../fetchai/skills/ml_train/behaviours.py | 10 +- packages/fetchai/skills/ml_train/dialogues.py | 46 ++++++- packages/fetchai/skills/ml_train/handlers.py | 120 +++++++++++++----- packages/fetchai/skills/ml_train/skill.yaml | 13 +- packages/fetchai/skills/ml_train/strategy.py | 15 ++- packages/hashes.csv | 4 +- 9 files changed, 175 insertions(+), 61 deletions(-) diff --git a/aea/exceptions.py b/aea/exceptions.py index 046faada72..303a854168 100644 --- a/aea/exceptions.py +++ b/aea/exceptions.py @@ -106,6 +106,8 @@ def parse_exception(exception: Exception, limit=-1) -> str: :param limit: the limit :return: exception as string """ + if isinstance(exception, AEAEnforceError): + limit = -2 msgs = traceback.format_exception( type(exception), exception, exception.__traceback__, limit=limit ) diff --git a/packages/fetchai/skills/ml_data_provider/skill.yaml b/packages/fetchai/skills/ml_data_provider/skill.yaml index a8442a9382..8a99118be5 100644 --- a/packages/fetchai/skills/ml_data_provider/skill.yaml +++ b/packages/fetchai/skills/ml_data_provider/skill.yaml @@ -12,7 +12,7 @@ fingerprint: behaviours.py: QmWgXU9qgahXwMKNqLLfDiGNYJozSXv2SVMkoPDQncC7ok dialogues.py: QmVwLCVudrwmJ33t5npXFDfRp8UYK7QFH4djanZgzziPTw handlers.py: QmWCVSnLMSLLPB9koJVc7g12YsfW3eCgkMET8TTpwDDW69 - strategy.py: QmRro9Lztj4AwUPcBFWsQV7CWVjUmZ12Xp4Q4rsvPfbyKL + strategy.py: QmYBoMmcmfkV1jvFsVwBaUNostXWouApgAL1GXhQAN5nAX fingerprint_ignore_patterns: [] connections: [] contracts: [] @@ -70,6 +70,7 @@ models: service_data: key: dataset_id value: fmnist + service_id: data_service class_name: Strategy dependencies: numpy: {} diff --git a/packages/fetchai/skills/ml_data_provider/strategy.py b/packages/fetchai/skills/ml_data_provider/strategy.py index 75901b6c08..cd91a290ac 100644 --- a/packages/fetchai/skills/ml_data_provider/strategy.py +++ b/packages/fetchai/skills/ml_data_provider/strategy.py @@ -19,6 +19,8 @@ """This module contains the strategy class.""" +import uuid + import numpy as np from tensorflow import keras @@ -38,6 +40,7 @@ DEFAULT_BATCH_SIZE = 32 DEFAULT_SELLER_TX_FEE = 0 DEFAULT_BUYER_TX_FEE = 0 +DEFAULT_SERVICE_ID = "data_service" DEFAULT_LOCATION = {"longitude": 0.1270, "latitude": 51.5194} DEFAULT_PERSONALITY_DATA = {"piece": "genus", "value": "data"} @@ -59,6 +62,7 @@ def __init__(self, **kwargs) -> None: currency_id = kwargs.pop("currency_id", None) ledger_id = kwargs.pop("ledger_id", None) self._is_ledger_tx = kwargs.pop("is_ledger_tx", False) + self._service_id = kwargs.pop("service_id", DEFAULT_SERVICE_ID) location = kwargs.pop("location", DEFAULT_LOCATION) self._agent_location = { @@ -226,6 +230,8 @@ def generate_terms(self) -> Description: "currency_id": self._currency_id, "ledger_id": self.ledger_id, "address": address, + "service_id": self._service_id, + "nonce": uuid.uuid4().hex, } ) return proposal @@ -237,4 +243,19 @@ def is_valid_terms(self, terms: Description) -> bool: :param terms: the terms :return: boolean """ - return terms == self.generate_terms() + generated_terms = self.generate_terms() + return all( + [ + terms.values[key] == generated_terms.values[key] + for key in [ + "batch_size", + "price", + "seller_tx_fee", + "buyer_tx_fee", + "currency_id", + "ledger_id", + "address", + "service_id", + ] + ] + ) diff --git a/packages/fetchai/skills/ml_train/behaviours.py b/packages/fetchai/skills/ml_train/behaviours.py index 0c8ae3057f..c8276a3751 100644 --- a/packages/fetchai/skills/ml_train/behaviours.py +++ b/packages/fetchai/skills/ml_train/behaviours.py @@ -28,13 +28,12 @@ CONNECTION_ID as LEDGER_CONNECTION_PUBLIC_ID, ) from packages.fetchai.protocols.ledger_api import LedgerApiMessage -from packages.fetchai.protocols.ml_trade.dialogues import MlTradeDialogue from packages.fetchai.skills.generic_buyer.behaviours import GenericSearchBehaviour from packages.fetchai.skills.ml_train.dialogues import ( LedgerApiDialogue, LedgerApiDialogues, + MlTradeDialogue, ) -from packages.fetchai.skills.ml_train.strategy import Strategy DEFAULT_MAX_PROCESSING = 120 @@ -90,11 +89,6 @@ def _start_processing(self) -> None: self.context.logger.info( f"Processing transaction, {len(self.waiting)} transactions remaining" ) - strategy = cast(Strategy, self.context.strategy) - proposal = ml_trade_dialogue.last_message.terms # type: ignore - terms = strategy.terms_from_proposal( - proposal, ml_trade_dialogue.last_message.sender # type: ignore - ) ledger_api_dialogues = cast( LedgerApiDialogues, self.context.ledger_api_dialogues @@ -102,7 +96,7 @@ def _start_processing(self) -> None: ledger_api_msg, ledger_api_dialogue = ledger_api_dialogues.create( counterparty=LEDGER_API_ADDRESS, performative=LedgerApiMessage.Performative.GET_RAW_TRANSACTION, - terms=terms, + terms=ml_trade_dialogue.terms, ) ledger_api_dialogue = cast(LedgerApiDialogue, ledger_api_dialogue) ledger_api_dialogue.associated_ml_trade_dialogue = ml_trade_dialogue diff --git a/packages/fetchai/skills/ml_train/dialogues.py b/packages/fetchai/skills/ml_train/dialogues.py index 59dd7bd81e..a934bf3ba2 100644 --- a/packages/fetchai/skills/ml_train/dialogues.py +++ b/packages/fetchai/skills/ml_train/dialogues.py @@ -28,10 +28,11 @@ - MlTradeDialogues: The dialogues class keeps track of all dialogues of type ml_trade. """ -from typing import Optional, Type +from typing import Dict, Optional, Type from aea.common import Address from aea.exceptions import AEAEnforceError, enforce +from aea.helpers.transaction.base import Terms from aea.protocols.base import Message from aea.protocols.dialogue.base import Dialogue as BaseDialogue from aea.protocols.dialogue.base import DialogueLabel as BaseDialogueLabel @@ -56,6 +57,7 @@ from packages.fetchai.protocols.ml_trade.dialogues import ( MlTradeDialogues as BaseMlTradeDialogues, ) +from packages.fetchai.protocols.ml_trade.message import MlTradeMessage from packages.fetchai.protocols.oef_search.dialogues import ( OefSearchDialogue as BaseOefSearchDialogue, ) @@ -103,7 +105,47 @@ def role_from_first_message( # pylint: disable=unused-argument ) -MlTradeDialogue = BaseMlTradeDialogue +class MlTradeDialogue(BaseMlTradeDialogue): + """The dialogue class maintains state of a dialogue and manages it.""" + + def __init__( + self, + dialogue_label: BaseDialogueLabel, + self_address: Address, + role: BaseDialogue.Role, + message_class: Type[MlTradeMessage] = MlTradeMessage, + ) -> None: + """ + Initialize a dialogue. + + :param dialogue_label: the identifier of the dialogue + :param self_address: the address of the entity for whom this dialogue is maintained + :param role: the role of the agent this dialogue is maintained for + + :return: None + """ + BaseMlTradeDialogue.__init__( + self, + dialogue_label=dialogue_label, + self_address=self_address, + role=role, + message_class=message_class, + ) + self.data_for_sale = None # type: Optional[Dict[str, str]] + self._terms = None # type: Optional[Terms] + + @property + def terms(self) -> Terms: + """Get terms.""" + if self._terms is None: + raise AEAEnforceError("Terms not set!") + return self._terms + + @terms.setter + def terms(self, terms: Terms) -> None: + """Set terms.""" + enforce(self._terms is None, "Terms already set!") + self._terms = terms class MlTradeDialogues(Model, BaseMlTradeDialogues): diff --git a/packages/fetchai/skills/ml_train/handlers.py b/packages/fetchai/skills/ml_train/handlers.py index 978ca002cd..6219314540 100644 --- a/packages/fetchai/skills/ml_train/handlers.py +++ b/packages/fetchai/skills/ml_train/handlers.py @@ -20,11 +20,10 @@ """This module contains the handler for the 'ml_train' skill.""" import pickle # nosec -import uuid from typing import Optional, cast from aea.configurations.base import PublicId -from aea.helpers.transaction.base import Terms +from aea.crypto.ledger_apis import LedgerApis from aea.protocols.base import Message from aea.skills.base import Handler @@ -154,23 +153,12 @@ def _handle_terms( ledger_api_dialogues = cast( LedgerApiDialogues, self.context.ledger_api_dialogues ) + terms_ = strategy.terms_from_proposal(ml_trade_msg.terms) + ml_trade_dialogue.terms = terms_ _ledger_api_msg, ledger_api_dialogue = ledger_api_dialogues.create( counterparty=LEDGER_API_ADDRESS, performative=LedgerApiMessage.Performative.GET_RAW_TRANSACTION, - terms=Terms( - ledger_id=terms.values["ledger_id"], - sender_address=self.context.agent_addresses[ - terms.values["ledger_id"] - ], - counterparty_address=terms.values["address"], - amount_by_currency_id={ - terms.values["currency_id"]: -terms.values["price"] - }, - is_sender_payable_tx_fee=True, - quantities_by_good_id={"ml_training_data": 1}, - nonce=uuid.uuid4().hex, - fee_by_currency_id={terms.values["currency_id"]: 1}, - ), + terms=terms_, ) ledger_api_dialogue = cast(LedgerApiDialogue, ledger_api_dialogue) ledger_api_dialogue.associated_ml_trade_dialogue = ml_trade_dialogue @@ -401,6 +389,11 @@ def handle(self, message: Message) -> None: == LedgerApiMessage.Performative.TRANSACTION_DIGEST ): self._handle_transaction_digest(ledger_api_msg, ledger_api_dialogue) + elif ( + ledger_api_msg.performative + == LedgerApiMessage.Performative.TRANSACTION_RECEIPT + ): + self._handle_transaction_receipt(ledger_api_msg, ledger_api_dialogue) elif ledger_api_msg.performative == LedgerApiMessage.Performative.ERROR: self._handle_error(ledger_api_msg, ledger_api_dialogue) else: @@ -484,31 +477,67 @@ def _handle_transaction_digest( :param ledger_api_message: the ledger api message :param ledger_api_dialogue: the ledger api dialogue """ - ml_trade_dialogue = ledger_api_dialogue.associated_ml_trade_dialogue self.context.logger.info( "transaction was successfully submitted. Transaction digest={}".format( ledger_api_msg.transaction_digest ) ) - ml_trade_msg = cast( - Optional[MlTradeMessage], ml_trade_dialogue.last_incoming_message + ledger_api_msg_ = ledger_api_dialogue.reply( + performative=LedgerApiMessage.Performative.GET_TRANSACTION_RECEIPT, + target_message=ledger_api_msg, + transaction_digest=ledger_api_msg.transaction_digest, ) - tx_behaviour = cast(TransactionBehaviour, self.context.behaviours.transaction) - tx_behaviour.finish_processing(ledger_api_dialogue) # type: ignore - if ml_trade_msg is None: - raise ValueError("Could not retrieve ml_trade message") - ml_accept = ml_trade_dialogue.reply( - performative=MlTradeMessage.Performative.ACCEPT, - target_message=ml_trade_msg, - tx_digest=ledger_api_msg.transaction_digest.body, - terms=ml_trade_msg.terms, + self.context.logger.info("checking transaction is settled.") + self.context.outbox.put_message(message=ledger_api_msg_) + + def _handle_transaction_receipt( + self, ledger_api_msg: LedgerApiMessage, ledger_api_dialogue: LedgerApiDialogue + ) -> None: + """ + Handle a message of balance performative. + + :param ledger_api_message: the ledger api message + :param ledger_api_dialogue: the ledger api dialogue + """ + ml_trade_dialogue = ledger_api_dialogue.associated_ml_trade_dialogue + is_settled = LedgerApis.is_transaction_settled( + ml_trade_dialogue.terms.ledger_id, + ledger_api_msg.transaction_receipt.receipt, ) - self.context.outbox.put_message(message=ml_accept) - self.context.logger.info( - "informing counterparty={} of transaction digest={}.".format( - ml_trade_msg.sender[-5:], ledger_api_msg.transaction_digest, + tx_behaviour = cast(TransactionBehaviour, self.context.behaviours.transaction) + if is_settled: + tx_behaviour.finish_processing(ledger_api_dialogue) + ledger_api_msg_ = cast( + Optional[LedgerApiMessage], ledger_api_dialogue.last_outgoing_message + ) + if ledger_api_msg_ is None: + raise ValueError( # pragma: nocover + "Could not retrieve last ledger_api message" + ) + ml_trade_msg = cast( + Optional[MlTradeMessage], ml_trade_dialogue.last_incoming_message + ) + if ml_trade_msg is None: + raise ValueError("Could not retrieve last ml_trade message") + ml_accept = ml_trade_dialogue.reply( + performative=MlTradeMessage.Performative.ACCEPT, + target_message=ml_trade_msg, + tx_digest=ledger_api_msg_.transaction_digest.body, + terms=ml_trade_msg.terms, + ) + self.context.outbox.put_message(message=ml_accept) + self.context.logger.info( + "informing counterparty={} of transaction digest={}.".format( + ml_trade_msg.sender[-5:], ledger_api_msg_.transaction_digest, + ) + ) + else: + tx_behaviour.failed_processing(ledger_api_dialogue) + self.context.logger.info( + "transaction_receipt={} not settled or not valid, aborting".format( + ledger_api_msg.transaction_receipt + ) ) - ) def _handle_error( self, ledger_api_msg: LedgerApiMessage, ledger_api_dialogue: LedgerApiDialogue @@ -524,6 +553,18 @@ def _handle_error( ledger_api_msg, ledger_api_dialogue ) ) + ledger_api_msg_ = cast( + Optional[LedgerApiMessage], ledger_api_dialogue.last_outgoing_message + ) + if ( + ledger_api_msg_ is not None + and ledger_api_msg_.performative + != LedgerApiMessage.Performative.GET_BALANCE + ): + tx_behaviour = cast( + TransactionBehaviour, self.context.behaviours.transaction + ) + tx_behaviour.failed_processing(ledger_api_dialogue) def _handle_invalid( self, ledger_api_msg: LedgerApiMessage, ledger_api_dialogue: LedgerApiDialogue @@ -636,6 +677,19 @@ def _handle_error( signing_msg.error_code, signing_dialogue ) ) + signing_msg_ = cast( + Optional[SigningMessage], signing_dialogue.last_outgoing_message + ) + if ( + signing_msg_ is not None + and signing_msg_.performative + == SigningMessage.Performative.SIGN_TRANSACTION + ): + tx_behaviour = cast( + TransactionBehaviour, self.context.behaviours.transaction + ) + ledger_api_dialogue = signing_dialogue.associated_ledger_api_dialogue + tx_behaviour.failed_processing(ledger_api_dialogue) def _handle_invalid( self, signing_msg: SigningMessage, signing_dialogue: SigningDialogue diff --git a/packages/fetchai/skills/ml_train/skill.yaml b/packages/fetchai/skills/ml_train/skill.yaml index 7789304839..2d076b3434 100644 --- a/packages/fetchai/skills/ml_train/skill.yaml +++ b/packages/fetchai/skills/ml_train/skill.yaml @@ -9,12 +9,12 @@ aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmeHcYRhDJi6gqAHpK29UJz3doiyaicoqeTrPddqM3gh64 __init__.py: QmTurqhUDy13mCSrqmbd598Eqz7A5PAmDrL9UpeNg24dDr - behaviours.py: QmdzzpgZiE4gp9CzMUMnzPyhf7SQNUUM45MgsTywhZT6h5 - dialogues.py: QmVLXHVwXXLnK4zJzPBVhxzgu7ZmohRZ7oj3yG1LPFrmhx - handlers.py: QmeY1m1d7XfX6zUCznjm8sRHpnPCLmo2RmNFfWRk1VsVzN + behaviours.py: QmSLQf6neRYG27KKWVYQET497LxcDAAS9Wpu9Wo64qbtcF + dialogues.py: QmZCXn2do9uUjSD58zkhQh2pAJikoDHGhHHV2ert9fCPcb + handlers.py: QmaWisdbWNSRjsUCk3jUoVddq8W2hdJeWNEimHwroRUxvM ml_model.py: QmTfshn6dFnz9gKXZt7aJJczRH14bN7nk6TybwFpzkEPnk model.json: QmdV2tGrRY6VQ5VLgUa4yqAhPDG6X8tYsWecypq8nox9Td - strategy.py: QmaeAuVjk4aj35qP5ypSf52rVK2uXfQqdMQFRZKH1LgufA + strategy.py: QmUtEbBfKqNoHrNMHvQ9j6mCiEcn8cNCtkTTBpyx74mF5L tasks.py: QmahJRCf6V61FsqrKgMMUyJ8F7PRd6C2bjunZg2XtM9fpF fingerprint_ignore_patterns: [] connections: @@ -78,15 +78,14 @@ models: latitude: 51.5194 longitude: 0.127 max_buyer_tx_fee: 20 - max_negotiations: 1 - max_quantity: 100 + max_negotiations: 2 max_unit_price: 70 - min_quantity: 1 search_query: constraint_type: == search_key: dataset_id search_value: fmnist search_radius: 5.0 + service_id: data_service class_name: Strategy dependencies: numpy: {} diff --git a/packages/fetchai/skills/ml_train/strategy.py b/packages/fetchai/skills/ml_train/strategy.py index 0f004556e5..3e5bf10cbf 100644 --- a/packages/fetchai/skills/ml_train/strategy.py +++ b/packages/fetchai/skills/ml_train/strategy.py @@ -28,13 +28,14 @@ Location, Query, ) -from aea.helpers.transaction.base import Address, Terms +from aea.helpers.transaction.base import Terms from aea.skills.base import Model DEFAULT_MAX_ROW_PRICE = 5 DEFAULT_MAX_TX_FEE = 2 DEFAULT_MAX_NEGOTIATIONS = 1 +DEFAULT_SERVICE_ID = "service_data" DEFAULT_LOCATION = {"longitude": 0.1270, "latitude": 51.5194} DEFAULT_SEARCH_QUERY = { @@ -58,6 +59,7 @@ def __init__(self, **kwargs) -> None: self._max_negotiations = kwargs.pop( "max_negotiations", DEFAULT_MAX_NEGOTIATIONS ) + self._service_id = kwargs.pop("service_id", DEFAULT_SERVICE_ID) self._search_query = kwargs.pop("search_query", DEFAULT_SEARCH_QUERY) location = kwargs.pop("location", DEFAULT_LOCATION) @@ -177,6 +179,7 @@ def is_acceptable_terms(self, terms: Description) -> bool: and (terms.values["buyer_tx_fee"] <= self._max_buyer_tx_fee) and (terms.values["currency_id"] == self._currency_id) and (terms.values["ledger_id"] == self._ledger_id) + and (terms.values["service_id"] == self._service_id) ) return result @@ -198,9 +201,7 @@ def is_affordable_terms(self, terms: Description) -> bool: result = True return result - def terms_from_proposal( - self, proposal: Description, counterparty_address: Address - ) -> Terms: + def terms_from_proposal(self, proposal: Description) -> Terms: """ Get the terms from a proposal. @@ -211,15 +212,15 @@ def terms_from_proposal( terms = Terms( ledger_id=proposal.values["ledger_id"], sender_address=buyer_address, - counterparty_address=counterparty_address, + counterparty_address=proposal.values["address"], amount_by_currency_id={ proposal.values["currency_id"]: -proposal.values["price"] }, quantities_by_good_id={ - proposal.values["service_id"]: proposal.values["quantity"] + proposal.values["service_id"]: proposal.values["batch_size"] }, is_sender_payable_tx_fee=True, - nonce=proposal.values["tx_nonce"], + nonce=proposal.values["nonce"], fee_by_currency_id={proposal.values["currency_id"]: self._max_buyer_tx_fee}, ) return terms diff --git a/packages/hashes.csv b/packages/hashes.csv index 9b848a836f..0d60dc7c4a 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -80,8 +80,8 @@ fetchai/skills/generic_buyer,QmNRP6rWnTNMwDYygzzN6BY7kUxh2DMYrjNaQVd19M4Nv4 fetchai/skills/generic_seller,QmR7xuDhrnq8gXMe9pk65LykCydkcB8mPh3UgPGZUgvpQ6 fetchai/skills/gym,Qmb4Bhfhhiwu9UsvSyJdLworYm41agCA6WffrYGxN3Sr1x fetchai/skills/http_echo,QmRJRubhKEWPmwxqEac3Zk9AXbAR4J4oB5k56CWuj9zPFE -fetchai/skills/ml_data_provider,QmbdLMEdhRuoMi7BRJM1K6K6HKBPb23s6PpYppBuz8vgiE -fetchai/skills/ml_train,QmSSomzaXrpkfpaNdvN5rxCmQcpzMsLFAx1z4uoqudogH9 +fetchai/skills/ml_data_provider,QmP2GmJGz8XuXfWwatJn7RgXXLE5eXs4xMFuxXZqA5JkWP +fetchai/skills/ml_train,QmXJ7cV7JCvHGLbzmix9ecFepDHYunUBe2jtG1vxCGbdKu fetchai/skills/registration_aw1,QmQiDCMeMnkQ4D3eu1mjAXoJKo51976KrAkfYWWNFKTGnA fetchai/skills/scaffold,QmezXGa1jPSteM9c6iH1hCHJVarqbLaG7mwaJmkkAJ6XUx fetchai/skills/simple_buyer,QmXLnLkkBDMq6KcFZ6XRcNW57djKCcuu7XicJGykcHZCM6 From 371c6ba7d8b6f160a5b3f6b10e5b610dd8595e5c Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 29 Dec 2020 12:49:14 +0100 Subject: [PATCH 073/204] remove 3.8 check on ML skill + ledger --- .../test_packages/test_skills_integration/test_ml_skills.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_packages/test_skills_integration/test_ml_skills.py b/tests/test_packages/test_skills_integration/test_ml_skills.py index 42231fb22d..00e29a5bd1 100644 --- a/tests/test_packages/test_skills_integration/test_ml_skills.py +++ b/tests/test_packages/test_skills_integration/test_ml_skills.py @@ -19,15 +19,12 @@ """This test module contains the integration test for the weather skills.""" import importlib -import sys from random import uniform import pytest from aea.test_tools.test_cases import AEATestCaseMany - from packages.fetchai.connections.p2p_libp2p.connection import LIBP2P_SUCCESS_MESSAGE - from tests.conftest import ( COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, @@ -227,8 +224,7 @@ class TestMLSkillsFetchaiLedger(AEATestCaseMany): reruns=MAX_FLAKY_RERUNS_INTEGRATION ) # cause possible network issues @pytest.mark.skipif( - sys.version_info >= (3, 8), - reason="cannot run on 3.8 as tensorflow not installable", + _is_not_tensorflow_installed(), reason="This test requires Tensorflow.", ) def test_ml_skills(self, pytestconfig): """Run the ml skills sequence.""" From dbcf7a785698153bf29a290fe67eab0bf5dc6601 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 29 Dec 2020 12:56:07 +0100 Subject: [PATCH 074/204] fix isort checks --- tests/test_packages/test_skills_integration/test_ml_skills.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_packages/test_skills_integration/test_ml_skills.py b/tests/test_packages/test_skills_integration/test_ml_skills.py index 00e29a5bd1..40ece8d0a3 100644 --- a/tests/test_packages/test_skills_integration/test_ml_skills.py +++ b/tests/test_packages/test_skills_integration/test_ml_skills.py @@ -24,7 +24,9 @@ import pytest from aea.test_tools.test_cases import AEATestCaseMany + from packages.fetchai.connections.p2p_libp2p.connection import LIBP2P_SUCCESS_MESSAGE + from tests.conftest import ( COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, From 24861a3bcf9de2d5bdd693b7108142c6fdea24ec Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Tue, 29 Dec 2020 17:17:45 +0300 Subject: [PATCH 075/204] coverage fixes --- aea/cli/utils/package_utils.py | 4 +- aea/configurations/manager.py | 20 ++- .../connection-custom_config.json | 3 + aea/helpers/env_vars.py | 3 +- tests/test_cli/test_generate_wealth.py | 3 +- tests/test_cli/tools_for_testing.py | 5 + tests/test_configurations/test_base.py | 11 ++ tests/test_configurations/test_manager.py | 133 +++++++++++++++++- tests/test_configurations/test_validation.py | 7 + tests/test_crypto/test_helpers.py | 25 +++- tests/test_helpers/test_base.py | 8 ++ tests/test_helpers/test_env_vars.py | 70 +++++++++ 12 files changed, 276 insertions(+), 16 deletions(-) create mode 100644 tests/test_helpers/test_env_vars.py diff --git a/aea/cli/utils/package_utils.py b/aea/cli/utils/package_utils.py index b59a0690c5..d6f95484b7 100644 --- a/aea/cli/utils/package_utils.py +++ b/aea/cli/utils/package_utils.py @@ -76,7 +76,7 @@ def verify_or_create_private_keys_ctx( - ctx: Context, aea_project_path: Path = ROOT, exit_on_error: bool = True, + ctx: Context, aea_project_path: Path = ROOT, exit_on_error: bool = False, ) -> None: """ Verify or create private keys with ctx provided. @@ -97,7 +97,7 @@ def verify_or_create_private_keys_ctx( except ValueError as e: # pragma: nocover if exit_on_error: sys.exit(1) - click.ClickException(str(e)) + raise click.ClickException(str(e)) def validate_package_name(package_name: str): diff --git a/aea/configurations/manager.py b/aea/configurations/manager.py index cfbc5e9739..0e25eb09cd 100644 --- a/aea/configurations/manager.py +++ b/aea/configurations/manager.py @@ -138,7 +138,7 @@ def _try_get_component_id_from_prefix( def handle_dotted_path( - value: str, author: str + value: str, author: str, aea_project_path: Union[str, Path] = ".", ) -> Tuple[List[str], Path, ConfigLoader, Optional[ComponentId]]: """Separate the path between path to resource and json path to attribute. @@ -157,10 +157,12 @@ def handle_dotted_path( :param value: dotted path. :param author: the author string. + :param aea_project_path: project path :return: Tuple[list of settings dict keys, filepath, config loader, component id]. """ parts = value.split(".") + aea_project_path = Path(aea_project_path) root = parts[0] if root not in ALLOWED_PATH_ROOTS: @@ -207,7 +209,11 @@ def handle_dotted_path( # find path to the resource directory path_to_resource_directory = ( - Path(".") / VENDOR / resource_author / resource_type_plural / resource_name + aea_project_path + / VENDOR + / resource_author + / resource_type_plural + / resource_name ) path_to_resource_configuration = ( path_to_resource_directory @@ -432,11 +438,13 @@ def _get_value_for_json_path( :return: one of the json values of NotExists if value not presents in data dict. """ - value = json.loads(json.dumps(data)) # in case or ordered dict + value = json.loads(json.dumps(data)) # in case of ordered dict doing copy prev_key = "" for key in json_path: if not isinstance(value, dict): - raise ValueError(f"Attribute '{prev_key}' is not a dictionary.") + raise ValueError( + f"Attribute '{prev_key}' is not a dictionary." + ) # pragma: nocover if key not in value: return NotExists @@ -454,7 +462,9 @@ def _parse_path(self, path: VariablePath) -> Tuple[Optional[ComponentId], JsonPa """ if isinstance(path, str): json_path, *_, component_id = handle_dotted_path( - path, self.agent_config.author + path, + self.agent_config.author, + aea_project_path=self.aea_project_directory, ) else: # pragma: nocover if isinstance(path[0], ComponentId): diff --git a/aea/configurations/schemas/configurable_parts/connection-custom_config.json b/aea/configurations/schemas/configurable_parts/connection-custom_config.json index 8e1dd0f835..6f62cd8828 100644 --- a/aea/configurations/schemas/configurable_parts/connection-custom_config.json +++ b/aea/configurations/schemas/configurable_parts/connection-custom_config.json @@ -15,6 +15,9 @@ }, "config": { "type": "object" + }, + "is_abstract": { + "$ref": "skill-config_schema.json#/properties/is_abstract" } } } diff --git a/aea/helpers/env_vars.py b/aea/helpers/env_vars.py index cb4a05e6bd..963b0ecc3d 100644 --- a/aea/helpers/env_vars.py +++ b/aea/helpers/env_vars.py @@ -44,6 +44,7 @@ def replace_with_env_var( ) -> JSON_TYPES: """Replace env var with value.""" result = ENV_VARIABLE_RE.match(value) + if not result: return value @@ -59,7 +60,7 @@ def replace_with_env_var( var_value = default_value else: raise ValueError( - f"Var name {var_name} not found in env variables and no default value set!" + f"Var name `{var_name}` not found in env variables and no default value set!" ) if type_str is not None: diff --git a/tests/test_cli/test_generate_wealth.py b/tests/test_cli/test_generate_wealth.py index 9afcbd1659..39c294b07a 100644 --- a/tests/test_cli/test_generate_wealth.py +++ b/tests/test_cli/test_generate_wealth.py @@ -23,7 +23,6 @@ from aea.cli import cli from aea.cli.generate_wealth import _try_generate_wealth -from aea.exceptions import AEAException from aea.test_tools.test_cases import AEATestCaseMany from tests.conftest import ( @@ -108,6 +107,6 @@ def test_wealth_commands_negative(self): settings = {"unsupported_crypto": "path"} self.nested_set_config("agent.private_key_paths", settings) with pytest.raises( - AEAException, match="Item not registered with id 'unsupported_crypto'." + Exception, match="Unsupported identifier `unsupported_crypto`" ): self.generate_wealth() diff --git a/tests/test_cli/tools_for_testing.py b/tests/test_cli/tools_for_testing.py index 9560cee147..37c0d2e2c5 100644 --- a/tests/test_cli/tools_for_testing.py +++ b/tests/test_cli/tools_for_testing.py @@ -23,6 +23,7 @@ from click import ClickException from aea.configurations.base import PackageVersion +from aea.configurations.constants import DEFAULT_LEDGER from tests.conftest import AUTHOR, COSMOS, ETHEREUM from tests.test_cli.constants import DEFAULT_TESTING_VERSION @@ -49,6 +50,9 @@ def __init__(self, *args, **kwargs): private_key_paths = kwargs.get("private_key_paths", []) self.private_key_paths = Mock() self.private_key_paths.read_all = Mock(return_value=private_key_paths) + self.private_key_paths.read = Mock( + return_value=private_key_paths[0][1] if private_key_paths else None + ) connection_private_key_paths = kwargs.get("connection_private_key_paths", []) self.connection_private_key_paths = Mock() self.connection_private_key_paths.read_all = Mock( @@ -58,6 +62,7 @@ def __init__(self, *args, **kwargs): self.component_configurations = {} self.package_dependencies = set() self.config: dict = {} + self.default_ledger = DEFAULT_LEDGER registry_path = "registry" name = "name" diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index 31a4a7e33a..eade52496b 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -981,3 +981,14 @@ def test_check_public_id_consistency_negative(): with pytest.raises(ValueError, match=f"Directory {random_dir_name} is not valid."): component_configuration = ProtocolConfig("name", "author") component_configuration.check_public_id_consistency(Path(random_dir_name)) + + +def test_component_id_from_json(): + """Test ComponentId.from_json.""" + json_data = { + "type": "connection", + "author": "author", + "name": "name", + "version": "1.0.0", + } + assert ComponentId.from_json(json_data).json == json_data diff --git a/tests/test_configurations/test_manager.py b/tests/test_configurations/test_manager.py index ede5ecb7b6..32da864fdc 100644 --- a/tests/test_configurations/test_manager.py +++ b/tests/test_configurations/test_manager.py @@ -17,16 +17,27 @@ # # ------------------------------------------------------------------------------ """This test module contains the tests for the configurations manager module.""" - import os from copy import deepcopy -from unittest.mock import patch +from pathlib import Path +from unittest.mock import mock_open, patch import pytest import yaml -from aea.configurations.manager import AgentConfigManager +from aea.configurations.constants import CONNECTION +from aea.configurations.data_types import ComponentId, PublicId +from aea.configurations.manager import ( + AgentConfigManager, + find_component_directory_from_component_id, + handle_dotted_path, +) +from aea.exceptions import AEAException + +from tests.conftest import ROOT_DIR + +DUMMY_AEA = Path(ROOT_DIR) / "tests" / "data" / "dummy_aea" agent_config_data = yaml.safe_load( """ @@ -45,6 +56,7 @@ - fetchai/default:0.10.0 skills: - dummy_author/dummy:0.1.0 +- fetchai/error:0.10.0 default_connection: fetchai/local:0.13.0 default_ledger: cosmos logging_config: @@ -83,7 +95,7 @@ def test_envvars_applied(): os.environ.pop("DISABLE_LOGS") with pytest.raises( ValueError, - match="Var name DISABLE_LOGS not found in env variables and no default value set!", + match="Var name `DISABLE_LOGS` not found in env variables and no default value set!", ): with patch.object(AgentConfigManager, "_load_config_data", return_value=[dct]): agent_config_manager = AgentConfigManager.load( @@ -158,3 +170,116 @@ def test_envvars_preserved(): assert ( agent_config_manager.json["private_key_paths"]["cosmos"] == new_cosmos_key_value ) + + +def test_agent_attribute_get_set(): + """Test agent config manager get set variables.""" + dct = deepcopy(agent_config_data) + with patch.object(AgentConfigManager, "_load_config_data", return_value=[dct]): + os.environ["DISABLE_LOGS"] = "true" + agent_config_manager = AgentConfigManager.load( + DUMMY_AEA, substitude_env_vars=False + ) + assert ( + agent_config_manager.get_variable("agent.default_ledger") + == dct["default_ledger"] + ) + assert ( + agent_config_manager.get_variable("vendor.fetchai.skills.error.name") + == "error" + ) + + assert ( + agent_config_manager.get_variable( + "vendor.fetchai.connections.local.is_abstract" + ) + is False + ) + agent_config_manager.set_variable( + "vendor.fetchai.connections.local.is_abstract", True + ) + assert ( + agent_config_manager.get_variable( + "vendor.fetchai.connections.local.is_abstract" + ) + is True + ) + + agent_config_manager.set_variable("agent.default_ledger", "fetchai") + assert agent_config_manager.get_variable("agent.default_ledger") == "fetchai" + + assert ( + agent_config_manager.json["component_configurations"][0]["is_abstract"] + is True + ) + + agent_config_manager = AgentConfigManager.load(DUMMY_AEA, substitude_env_vars=False) + agent_config_manager.set_variable( + "vendor.fetchai.connections.p2p_libp2p.config.delegate_uri", "some_url" + ) + assert ( + agent_config_manager.get_variable( + "vendor.fetchai.connections.p2p_libp2p.config.delegate_uri" + ) + == "some_url" + ) + + with pytest.raises( + ValueError, match="Attribute `does_not_exist` for AgentConfig does not exist" + ): + agent_config_manager.get_variable("agent.does_not_exist") + + agent_config_manager.validate_current_config() + agent_config_manager.verify_or_create_private_keys(DUMMY_AEA, lambda x, y: None) + + +def test_agent_attribute_get_overridables(): + """Test AgentConfigManager.get_overridables.""" + agent_config_manager = AgentConfigManager.load(DUMMY_AEA, substitude_env_vars=False) + agent_overrides, component_overrides = agent_config_manager.get_overridables() + assert "default_ledger" in agent_overrides + assert "is_abstract" in list(component_overrides.values())[0] + + +def test_dump_config(): + """Test AgentConfigManager.dump_config.""" + agent_config_manager = AgentConfigManager.load(DUMMY_AEA, substitude_env_vars=False) + with patch("aea.configurations.manager.open", mock_open()), patch( + "aea.configurations.loader.ConfigLoader.dump" + ) as dump_mock: + agent_config_manager.dump_config() + + dump_mock.assert_called_once() + + +def test_handle_dotted_path(): + """Test handle_dotted_path.""" + with pytest.raises( + AEAException, match=r"The root of the dotted path must be one of:" + ): + handle_dotted_path("something", author="fetchai") + + with pytest.raises( + AEAException, + match=r"The path is too short. Please specify a path up to an attribute name.", + ): + handle_dotted_path("skills", author="fetchai") + + with pytest.raises( + AEAException, match=r"is not a valid component type. Please use one of" + ): + handle_dotted_path("vendor.fetchai.notskills.dummy.name", author="fetchai") + + with pytest.raises(AEAException, match=r"Resource .* does not exist."): + handle_dotted_path("skills.notdummy.name", author="fetchai") + + +def test_find_component_directory_from_component_id(): + """Test find_component_directory_from_component_id.""" + with pytest.raises(ValueError, match=r"Package .* not found."): + find_component_directory_from_component_id( + Path("."), + ComponentId( + component_type=CONNECTION, public_id=PublicId("test", "test", "1.0.1") + ), + ) diff --git a/tests/test_configurations/test_validation.py b/tests/test_configurations/test_validation.py index 3f30b042bc..9a41a5ea37 100644 --- a/tests/test_configurations/test_validation.py +++ b/tests/test_configurations/test_validation.py @@ -32,6 +32,13 @@ def test_compare_data_pattern(): == "For attribute `a` `str` data type is expected, but `int` was provided!" ) + assert not validate_data_with_pattern( + {"a": 12}, {"a": "${var}"}, skip_env_vars=True + ) + assert not validate_data_with_pattern( + {"a": "${var}"}, {"a": "string"}, skip_env_vars=True + ) + errors = validate_data_with_pattern({"a": 12}, {"b": 12}) assert errors assert errors[0] == "Attribute `a` is not allowed to be updated!" diff --git a/tests/test_crypto/test_helpers.py b/tests/test_crypto/test_helpers.py index 510073bb4f..70cb90f931 100644 --- a/tests/test_crypto/test_helpers.py +++ b/tests/test_crypto/test_helpers.py @@ -16,11 +16,10 @@ # limitations under the License. # # ------------------------------------------------------------------------------ - """This module contains the tests for the crypto/helpers module.""" - import logging import os +from pathlib import Path from unittest.mock import mock_open, patch import pytest @@ -31,6 +30,7 @@ from aea.crypto.fetchai import FetchAICrypto from aea.crypto.helpers import ( create_private_key, + private_key_verify_or_create, try_generate_testnet_wealth, try_validate_private_key_path, ) @@ -42,6 +42,7 @@ ETHEREUM_PRIVATE_KEY_PATH, FETCHAI_PRIVATE_KEY_PATH, ) +from tests.test_cli.tools_for_testing import AgentConfigMock logger = logging.getLogger(__name__) @@ -147,3 +148,23 @@ def test__create_ethereum_private_key_positive(self, *mocks): def test__create_cosmos_private_key_positive(self, *mocks): """Test _create_cosmos_private_key positive result.""" create_private_key(CosmosCrypto.identifier, COSMOS_PRIVATE_KEY_FILE) + + +def test_private_key_verify_or_create(): + """Test private_key_verify_or_create.""" + agent_conf = AgentConfigMock() + with patch("aea.crypto.helpers.create_private_key") as mock_create: + private_key_verify_or_create(agent_conf, Path(".")) + mock_create.assert_called() + + agent_conf = AgentConfigMock(private_key_paths=[("fetchai", "test")]) + with patch("aea.crypto.helpers.try_validate_private_key_path") as mock_validate: + private_key_verify_or_create(agent_conf, Path(".")) + mock_validate.assert_called() + + agent_conf = AgentConfigMock(private_key_paths=[("fetchai", "${var}")]) + with patch("aea.crypto.helpers.try_validate_private_key_path") as mock_validate: + with patch("aea.crypto.helpers.create_private_key") as mock_create: + private_key_verify_or_create(agent_conf, Path(".")) + mock_validate.assert_not_called() + mock_create.assert_not_called() diff --git a/tests/test_helpers/test_base.py b/tests/test_helpers/test_base.py index 5b3d719771..e55d0f1f66 100644 --- a/tests/test_helpers/test_base.py +++ b/tests/test_helpers/test_base.py @@ -537,6 +537,14 @@ def test_instantiation(self): ) assert self.cert_request.not_after == expected_not_after + assert self.cert_request.not_before_string == expected_not_before.strftime( + "%Y-%m-%d" + ) + assert self.cert_request.not_after_string == expected_not_after.strftime( + "%Y-%m-%d" + ) + assert self.cert_request.get_message("some_key") + assert self.cert_request.save_path == Path(self.expected_path) def test_from_to_json(self): diff --git a/tests/test_helpers/test_env_vars.py b/tests/test_helpers/test_env_vars.py new file mode 100644 index 0000000000..78e56f9ea1 --- /dev/null +++ b/tests/test_helpers/test_env_vars.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the tests for the helper module.""" +import pytest + +from aea.helpers.env_vars import ( + apply_env_variables, + convert_value_str_to_type, + is_env_variable, + replace_with_env_var, +) + + +def test_is_env_variable(): + """Test is_env_variable.""" + assert is_env_variable("${test}") + assert is_env_variable("${test:int}") + assert is_env_variable("${test:int:12}") + + assert not is_env_variable("sdfsdf") + + +def test_apply_env_variables(): + """Test apply_env_variables""" + assert apply_env_variables("${var}", {"var": "test"}) == "test" + assert apply_env_variables("var", {"var": "test"}) == "var" + assert apply_env_variables(["${var}"], {"var": "test"}) == ["test"] + assert apply_env_variables({"${var}": "${var}"}, {"var": "test"}) == { + "test": "test" + } + + +def test_replace_with_env_var(): + """Test replace_with_env_var.""" + assert replace_with_env_var("${var:int:12}", {"var": "10"}) == 10 + assert replace_with_env_var("${var:int:12}", {}) == 12 + assert replace_with_env_var("var", {}) == "var" + assert replace_with_env_var("${var}", {}, default_value=100) == 100 + + with pytest.raises( + ValueError, + match=r"Var name `var` not found in env variables and no default value set!", + ): + replace_with_env_var("${var}", {}) + + +def test_convert_value_str_to_type(): + """Test convert_value_str_to_type.""" + assert convert_value_str_to_type("false", "bool") is False + assert convert_value_str_to_type("True", "bool") is True + assert convert_value_str_to_type("12", "int") == 12 + assert convert_value_str_to_type("1.1", "float") == 1.1 + assert convert_value_str_to_type("1sdfsdf2", "none") is None + assert convert_value_str_to_type('{"a": 12}', "dict") == {"a": 12} From 1080029586c705b5ccd4a1dc02f21a772196985c Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 29 Dec 2020 15:41:10 +0100 Subject: [PATCH 076/204] update tac-skills.md --- docs/tac-skills.md | 83 ++++++++++++------- .../md_files/bash-tac-skills.md | 63 +++++++++----- 2 files changed, 98 insertions(+), 48 deletions(-) diff --git a/docs/tac-skills.md b/docs/tac-skills.md index e7e38f4a61..3953db6295 100644 --- a/docs/tac-skills.md +++ b/docs/tac-skills.md @@ -117,16 +117,14 @@ aea add connection fetchai/p2p_libp2p:0.13.0 aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_control:0.13.0 -aea install -aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger fetchai -``` - -In `tac_controller/aea-config.yaml` add -``` yaml -default_routing: - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' +aea install +aea build ```

    @@ -163,17 +161,15 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_participation:0.14.0 aea add skill fetchai/tac_negotiation:0.16.0 -aea install -aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger fetchai -``` - -In `tac_participant_one/aea-config.yaml` add -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' +aea install +aea build ``` Then, build participant two: @@ -184,17 +180,15 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_participation:0.14.0 aea add skill fetchai/tac_negotiation:0.16.0 -aea install -aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger fetchai -``` - -In `tac_participant_two/aea-config.yaml` add -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' +aea install +aea build ```

    @@ -218,6 +212,11 @@ aea config get vendor.fetchai.skills.tac_control.models.parameters.args.registra aea config set vendor.fetchai.skills.tac_control.models.parameters.args.registration_start_time '01 01 2020 00:01' ``` +To set the registration time, you may find handy the following command: +``` bash +aea config set vendor.fetchai.skills.tac_control.models.parameters.args.registration_start_time "$(date -d "2 minutes" +'%d %m %Y %H:%M')" +``` + ### Update the connection params Briefly run the controller AEA: @@ -228,7 +227,7 @@ aea run Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) -Then, update the configuration of the weather client AEA's p2p connection (in `aea-config.yaml`) add the following: + + +Then, in the participant one, run this command (replace `SOME_ADDRESS` with the correct value as described above): +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` + +Do the same in participant two (beware of the different port numbers): +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11002", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9002", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9002" +}' +``` + +This allows the TAC participants to connect to the same local agent communication network as the TAC controller. + ### Run the AEAs diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md index 8b5705d136..7e0348d2d9 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md @@ -11,10 +11,14 @@ aea add connection fetchai/p2p_libp2p:0.13.0 aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_control:0.13.0 -aea install -aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger fetchai +aea config set --type dict agent.default_routing \ +'{ + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' +aea install +aea build ``` ``` bash aea fetch fetchai/tac_participant:0.18.0 --alias tac_participant_one @@ -37,10 +41,15 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_participation:0.14.0 aea add skill fetchai/tac_negotiation:0.16.0 -aea install -aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger fetchai +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' +aea install +aea build ``` ``` bash cd tac_participant_two @@ -49,10 +58,15 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_participation:0.14.0 aea add skill fetchai/tac_negotiation:0.16.0 -aea install -aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger fetchai +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' +aea install +aea build ``` ``` bash aea generate-key fetchai @@ -64,9 +78,32 @@ aea config get vendor.fetchai.skills.tac_control.models.parameters.args.registra aea config set vendor.fetchai.skills.tac_control.models.parameters.args.registration_start_time '01 01 2020 00:01' ``` ``` bash +aea config set vendor.fetchai.skills.tac_control.models.parameters.args.registration_start_time "$(date -d "2 minutes" +'%d %m %Y %H:%M')" +``` +``` bash aea run ``` ``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11002", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9002", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9002" +}' +``` +``` bash aea launch tac_controller tac_participant_one tac_participant_two ``` ``` bash @@ -75,20 +112,6 @@ aea delete tac_participant_one aea delete tac_participant_two ``` ``` yaml -default_routing: - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml --- public_id: fetchai/p2p_libp2p:0.13.0 type: connection From 250fffd8d174fad193e1a965614f202c747049ef Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 29 Dec 2020 16:55:31 +0100 Subject: [PATCH 077/204] fix soef connection token fetching in aea launch The issue was discovered when trying 'aea launch' on TAC demo. At SOEF connection initialization time, the path was relative to the AEA directory of the project the connection belongs to. When running the agents, the path was relative to the working directory of `aea launch`, which caused the path to the token to be wrong. This commit works around this issue by storing the absolute path in the SOEF connection object, instead of the relative one. --- packages/fetchai/connections/soef/connection.py | 2 ++ packages/fetchai/connections/soef/connection.yaml | 2 +- packages/hashes.csv | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/fetchai/connections/soef/connection.py b/packages/fetchai/connections/soef/connection.py index 4bbb3aac64..7f27f58105 100644 --- a/packages/fetchai/connections/soef/connection.py +++ b/packages/fetchai/connections/soef/connection.py @@ -21,6 +21,7 @@ import asyncio import copy import logging +import os import re import urllib from asyncio import CancelledError @@ -258,6 +259,7 @@ def __init__( self._token_storage_path = token_storage_path if self._token_storage_path is not None: + self._token_storage_path = os.path.abspath(self._token_storage_path) Path(self._token_storage_path).touch() self.declared_name = uuid4().hex self._unique_page_address = None # type: Optional[str] diff --git a/packages/fetchai/connections/soef/connection.yaml b/packages/fetchai/connections/soef/connection.yaml index a0f5b2fc93..ae6d7245f7 100644 --- a/packages/fetchai/connections/soef/connection.yaml +++ b/packages/fetchai/connections/soef/connection.yaml @@ -8,7 +8,7 @@ aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmSunLbHEkiiMMnQsSgeG4sQ43dCszoWTFFFJictf1dqDb __init__.py: Qmd5VBGFJHXFe1H45XoUh5mMSYBwvLSViJuGFeMgbPdQts - connection.py: QmP6vGJ5eKVjrPbxLsgGrZa3xjmCESLdB9CGt9mcktAxvR + connection.py: Qmf8z8d6sg62fem8vfsW5uURLtUe9L1YR5x6kDPgQYHQWA fingerprint_ignore_patterns: [] connections: [] protocols: diff --git a/packages/hashes.csv b/packages/hashes.csv index e1fcefef21..ed9c086945 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -39,7 +39,7 @@ fetchai/connections/p2p_libp2p_client,QmQndwAPhGHTpe1J8c6Q6jabV2KfkND7jsYiEF9Ysm fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T fetchai/connections/scaffold,QmaCupER4vedakPPLzn2Rw9YHPmqSPaZh5xnqi6xVxQHvn -fetchai/connections/soef,QmcsCsU9cYKozwR2Ag2V38xaCLJwTVfWNSmHmQTLqZki9j +fetchai/connections/soef,QmaF6dDLAdWm64osCmZXUncU76xYVyWcK89qt4o2sStsfN fetchai/connections/stub,QmUviVXuCuBu1c6PPioNg3htJ9aatpATDNDJ8mapVriQM8 fetchai/connections/tcp,QmcAEg3XkYFZ6oKqYSWpJodiakxDAVYadwcczpqeUjo7op fetchai/connections/webhook,QmXc9wa7iQ1qhRnPpjKqkQX2XxjpWw8toMC2wD7PpKBVLk From b36a3af73297624151c68b4f098960b3afa58761 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 29 Dec 2020 18:40:53 +0100 Subject: [PATCH 078/204] update tac skill contract demo docs --- docs/tac-skills-contract.md | 110 ++++++++++++--- docs/tac-skills.md | 9 +- .../md_files/bash-tac-skills-contract.md | 132 +++++++++--------- .../md_files/bash-tac-skills.md | 5 +- 4 files changed, 172 insertions(+), 84 deletions(-) diff --git a/docs/tac-skills-contract.md b/docs/tac-skills-contract.md index 95cd24be01..84ae0de9d4 100644 --- a/docs/tac-skills-contract.md +++ b/docs/tac-skills-contract.md @@ -118,10 +118,18 @@ aea add connection fetchai/p2p_libp2p:0.13.0 aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_control_contract:0.15.0 -aea install -aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger ethereum +aea config set vendor.fetchai.connections.soef.config.chain_identifier ethereum +aea config set --type bool vendor.fetchai.skills.tac_control.is_abstract true +aea config set --type dict agent.default_routing \ +'{ + "fetchai/contract_api:0.9.0": "fetchai/ledger:0.11.0", + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' +aea install +aea build ```

    @@ -139,17 +147,16 @@ In a separate terminal, in the root directory, fetch at least two participants: ``` bash aea fetch fetchai/tac_participant:0.18.0 --alias tac_participant_one cd tac_participant_one -aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using_contract 'True' --type bool -aea config set vendor.fetchai.skills.tac_negotiation.models.strategy.args.is_contract_tx 'True' --type bool +aea generate-key ethereum +aea add-key ethereum ethereum_private_key.txt aea install aea build cd .. aea fetch fetchai/tac_participant:0.18.0 --alias tac_participant_two cd tac_participant_two -aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using_contract 'True' --type bool -aea config set vendor.fetchai.skills.tac_negotiation.models.strategy.args.is_contract_tx 'True' --type bool aea generate-key ethereum aea add-key ethereum ethereum_private_key.txt +aea install aea build ``` @@ -170,14 +177,21 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_participation:0.14.0 aea add skill fetchai/tac_negotiation:0.16.0 -aea generate-key ethereum -aea add-key ethereum ethereum_private_key.txt -aea install -aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger ethereum +aea config set vendor.fetchai.connections.soef.config.chain_identifier ethereum aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using_contract 'True' --type bool aea config set vendor.fetchai.skills.tac_negotiation.models.strategy.args.is_contract_tx 'True' --type bool +aea config set --type dict agent.default_routing \ +'{ + "fetchai/contract_api:0.9.0": "fetchai/ledger:0.11.0", + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' +aea generate-key ethereum +aea add-key ethereum ethereum_private_key.txt +aea install +aea build ``` Then, build participant two: @@ -188,22 +202,33 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_participation:0.14.0 aea add skill fetchai/tac_negotiation:0.16.0 -aea generate-key ethereum -aea add-key ethereum ethereum_private_key.txt -aea install -aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger ethereum +aea config set vendor.fetchai.connections.soef.config.chain_identifier ethereum aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using_contract 'True' --type bool aea config set vendor.fetchai.skills.tac_negotiation.models.strategy.args.is_contract_tx 'True' --type bool +aea config set --type dict agent.default_routing \ +'{ + "fetchai/contract_api:0.9.0": "fetchai/ledger:0.11.0", + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' +aea generate-key ethereum +aea add-key ethereum ethereum_private_key.txt +aea install +aea build ```

    • -### Fund both tac participants +### Add keys for all AEAs -Similar to how you funded the controller, fund both tac participants. +Create the private key for the AEA for Fetch.ai `AgentLand`: +``` bash +aea generate-key fetchai +aea add-key fetchai fetchai_private_key.txt --connection +``` ### Update the game parameters in the controller @@ -214,6 +239,50 @@ aea config get vendor.fetchai.skills.tac_control_contract.models.parameters.args aea config set vendor.fetchai.skills.tac_control_contract.models.parameters.args.registration_start_time '01 01 2020 00:01' ``` +To set the registration time, you may find handy the following command: +``` bash +aea config set vendor.fetchai.skills.tac_control_contract.models.parameters.args.registration_start_time "$(date -d "2 minutes" +'%d %m %Y %H:%M')" +``` + + +### Update the connection params + +Briefly run the controller AEA: + +``` bash +aea run +``` + +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) + + +Then, in the participant one, run this command (replace `SOME_ADDRESS` with the correct value as described above): +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` + +Do the same in participant two (beware of the different port numbers): +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11002", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9002", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9002" +}' +``` + +This allows the TAC participants to connect to the same local agent communication network as the TAC controller. + + ## Run Ganache Run the following command @@ -231,13 +300,18 @@ You should get `1000000000000000000000`. ### Run the AEAs +First, launch the `tac_contract_controller`: +``` bash +aea run tac_contract_controller +``` + The CLI tool supports the launch of several agents at once. For example, assuming you followed the tutorial, you -can launch all the TAC agents as follows from the root directory: +can launch both the TAC agents as follows from the root directory: ``` bash -aea launch tac_controller_contract tac_participant_one tac_participant_two +aea launch tac_participant_one tac_participant_two ``` You may want to try `--multithreaded` diff --git a/docs/tac-skills.md b/docs/tac-skills.md index 3953db6295..364f7b53bd 100644 --- a/docs/tac-skills.md +++ b/docs/tac-skills.md @@ -285,13 +285,18 @@ This allows the TAC participants to connect to the same local agent communicatio ### Run the AEAs +First, launch the `tac_controller`: +``` bash +aea run +``` + The CLI tool supports the launch of several agents at once. For example, assuming you followed the tutorial, you -can launch all the TAC agents as follows from the root directory: +can launch both the TAC agents as follows from the root directory: ``` bash -aea launch tac_controller tac_participant_one tac_participant_two +aea launch tac_participant_one tac_participant_two ``` You may want to try `--multithreaded` diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills-contract.md b/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills-contract.md index f021d06840..f4a1a40807 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills-contract.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills-contract.md @@ -11,10 +11,18 @@ aea add connection fetchai/p2p_libp2p:0.13.0 aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_control_contract:0.15.0 -aea install -aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger ethereum +aea config set vendor.fetchai.connections.soef.config.chain_identifier ethereum +aea config set --type bool vendor.fetchai.skills.tac_control.is_abstract true +aea config set --type dict agent.default_routing \ +'{ + "fetchai/contract_api:0.9.0": "fetchai/ledger:0.11.0", + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' +aea install +aea build ``` ``` bash aea generate-key ethereum @@ -23,17 +31,16 @@ aea add-key ethereum ethereum_private_key.txt ``` bash aea fetch fetchai/tac_participant:0.18.0 --alias tac_participant_one cd tac_participant_one -aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using_contract 'True' --type bool -aea config set vendor.fetchai.skills.tac_negotiation.models.strategy.args.is_contract_tx 'True' --type bool +aea generate-key ethereum +aea add-key ethereum ethereum_private_key.txt aea install aea build cd .. aea fetch fetchai/tac_participant:0.18.0 --alias tac_participant_two cd tac_participant_two -aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using_contract 'True' --type bool -aea config set vendor.fetchai.skills.tac_negotiation.models.strategy.args.is_contract_tx 'True' --type bool aea generate-key ethereum aea add-key ethereum ethereum_private_key.txt +aea install aea build ``` ``` bash @@ -47,14 +54,21 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_participation:0.14.0 aea add skill fetchai/tac_negotiation:0.16.0 -aea generate-key ethereum -aea add-key ethereum ethereum_private_key.txt -aea install -aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger ethereum +aea config set vendor.fetchai.connections.soef.config.chain_identifier ethereum aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using_contract 'True' --type bool aea config set vendor.fetchai.skills.tac_negotiation.models.strategy.args.is_contract_tx 'True' --type bool +aea config set --type dict agent.default_routing \ +'{ + "fetchai/contract_api:0.9.0": "fetchai/ledger:0.11.0", + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' +aea generate-key ethereum +aea add-key ethereum ethereum_private_key.txt +aea install +aea build ``` ``` bash cd tac_participant_two @@ -63,78 +77,70 @@ aea add connection fetchai/soef:0.14.0 aea add connection fetchai/ledger:0.11.0 aea add skill fetchai/tac_participation:0.14.0 aea add skill fetchai/tac_negotiation:0.16.0 -aea generate-key ethereum -aea add-key ethereum ethereum_private_key.txt -aea install -aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_ledger ethereum +aea config set vendor.fetchai.connections.soef.config.chain_identifier ethereum aea config set vendor.fetchai.skills.tac_participation.models.game.args.is_using_contract 'True' --type bool aea config set vendor.fetchai.skills.tac_negotiation.models.strategy.args.is_contract_tx 'True' --type bool +aea config set --type dict agent.default_routing \ +'{ + "fetchai/contract_api:0.9.0": "fetchai/ledger:0.11.0", + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' +aea generate-key ethereum +aea add-key ethereum ethereum_private_key.txt +aea install +aea build +``` +``` bash +aea generate-key fetchai +aea add-key fetchai fetchai_private_key.txt --connection ``` ``` bash aea config get vendor.fetchai.skills.tac_control_contract.models.parameters.args.registration_start_time aea config set vendor.fetchai.skills.tac_control_contract.models.parameters.args.registration_start_time '01 01 2020 00:01' ``` ``` bash +aea config set vendor.fetchai.skills.tac_control_contract.models.parameters.args.registration_start_time "$(date -d "2 minutes" +'%d %m %Y %H:%M')" +``` +``` bash +aea run +``` +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11002", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9002", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9002" +}' +``` +``` bash docker run -p 8545:8545 trufflesuite/ganache-cli:latest --verbose --gasPrice=0 --gasLimit=0x1fffffffffffff --account="$(cat tac_controller_contract/ethereum_private_key.txt),1000000000000000000000" --account="$(cat tac_participant_one/ethereum_private_key.txt),1000000000000000000000" --account="$(cat tac_participant_two/ethereum_private_key.txt),1000000000000000000000" ``` ``` bash aea get-wealth ethereum ``` ``` bash -aea launch tac_controller_contract tac_participant_one tac_participant_two +aea run tac_contract_controller +``` +``` bash +aea launch tac_participant_one tac_participant_two ``` ``` bash aea delete tac_controller_contract aea delete tac_participant_one aea delete tac_participant_two -``` -``` yaml -name: tac_negotiation -authors: fetchai -version: 0.1.0 -license: Apache-2.0 -description: "The tac negotiation skill implements the logic for an AEA to do fipa negotiation in the TAC." -behaviours: - behaviour: - class_name: GoodsRegisterAndSearchBehaviour - args: - services_interval: 5 - clean_up: - class_name: TransactionCleanUpBehaviour - args: - tick_interval: 5.0 -handlers: - fipa: - class_name: FIPANegotiationHandler - args: {} - transaction: - class_name: TransactionHandler - args: {} - oef: - class_name: OEFSearchHandler - args: {} -models: - search: - class_name: Search - args: - search_interval: 5 - registration: - class_name: Registration - args: - update_interval: 5 - strategy: - class_name: Strategy - args: - register_as: both - search_for: both - dialogues: - class_name: Dialogues - args: {} - transactions: - class_name: Transactions - args: - pending_transaction_timeout: 30 -protocols: ['fetchai/oef_search:0.11.0', 'fetchai/fipa:0.11.0'] ``` \ No newline at end of file diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md index 7e0348d2d9..00e5c7f1fa 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md @@ -104,7 +104,10 @@ aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ }' ``` ``` bash -aea launch tac_controller tac_participant_one tac_participant_two +aea run +``` +``` bash +aea launch tac_participant_one tac_participant_two ``` ``` bash aea delete tac_controller From 1cbda45f66e24a2c44b96ee19066818094b9a79b Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Wed, 30 Dec 2020 11:07:53 +0300 Subject: [PATCH 079/204] fixes --- aea/helpers/env_vars.py | 2 +- tests/test_configurations/test_manager.py | 2 +- tests/test_helpers/test_env_vars.py | 3 +-- tests/test_helpers/test_search/test_models.py | 9 +++++++++ tests/test_manager/__init__.py | 20 +++++++++++++++++++ tests/{ => test_manager}/test_manager.py | 0 6 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 tests/test_manager/__init__.py rename tests/{ => test_manager}/test_manager.py (100%) diff --git a/aea/helpers/env_vars.py b/aea/helpers/env_vars.py index 963b0ecc3d..78c0841a5c 100644 --- a/aea/helpers/env_vars.py +++ b/aea/helpers/env_vars.py @@ -60,7 +60,7 @@ def replace_with_env_var( var_value = default_value else: raise ValueError( - f"Var name `{var_name}` not found in env variables and no default value set!" + f"`{var_name}` not found in env variables and no default value set! Please ensure a .env file is provided." ) if type_str is not None: diff --git a/tests/test_configurations/test_manager.py b/tests/test_configurations/test_manager.py index 32da864fdc..95d41c8805 100644 --- a/tests/test_configurations/test_manager.py +++ b/tests/test_configurations/test_manager.py @@ -95,7 +95,7 @@ def test_envvars_applied(): os.environ.pop("DISABLE_LOGS") with pytest.raises( ValueError, - match="Var name `DISABLE_LOGS` not found in env variables and no default value set!", + match="`DISABLE_LOGS` not found in env variables and no default value set!", ): with patch.object(AgentConfigManager, "_load_config_data", return_value=[dct]): agent_config_manager = AgentConfigManager.load( diff --git a/tests/test_helpers/test_env_vars.py b/tests/test_helpers/test_env_vars.py index 78e56f9ea1..5e1f404ff3 100644 --- a/tests/test_helpers/test_env_vars.py +++ b/tests/test_helpers/test_env_vars.py @@ -54,8 +54,7 @@ def test_replace_with_env_var(): assert replace_with_env_var("${var}", {}, default_value=100) == 100 with pytest.raises( - ValueError, - match=r"Var name `var` not found in env variables and no default value set!", + ValueError, match=r"`var` not found in env variables and no default value set!", ): replace_with_env_var("${var}", {}) diff --git a/tests/test_helpers/test_search/test_models.py b/tests/test_helpers/test_search/test_models.py index 7226832547..c6fc83b070 100644 --- a/tests/test_helpers/test_search/test_models.py +++ b/tests/test_helpers/test_search/test_models.py @@ -167,6 +167,8 @@ def test_description(): mock.description_bytes = None Description.encode(mock, description) assert mock.description_bytes is not None + description = Description.decode(mock) + assert "test" in description.values def test_constraint_type(): @@ -613,3 +615,10 @@ def test_query(): query_pb = query._encode() actual_query = Query._decode(query_pb) assert actual_query == query + + query_pb = MagicMock() + query_pb.query_bytes = None + Query.encode(query_pb, query) + assert query_pb.query_bytes is not None + query = Query.decode(query_pb) + assert "author" in query.model.attributes_by_name diff --git a/tests/test_manager/__init__.py b/tests/test_manager/__init__.py new file mode 100644 index 0000000000..8c03efbc34 --- /dev/null +++ b/tests/test_manager/__init__.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains a test for aea.manager.""" diff --git a/tests/test_manager.py b/tests/test_manager/test_manager.py similarity index 100% rename from tests/test_manager.py rename to tests/test_manager/test_manager.py From 4a10cb5ac8c944dc3641172862e093106b333c14 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Wed, 30 Dec 2020 10:34:10 +0000 Subject: [PATCH 080/204] revert aea launcher change --- aea/launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aea/launcher.py b/aea/launcher.py index 73840dad67..f06938f317 100644 --- a/aea/launcher.py +++ b/aea/launcher.py @@ -56,7 +56,7 @@ def load_agent(agent_dir: Union[PathLike, str]) -> AEA: :return: AEA instance """ with cd(agent_dir): - return AEABuilder.from_aea_project(".", verify_or_create_keys=False).build() + return AEABuilder.from_aea_project(".").build() def _set_logger( From 726796aa2216439b13fb532ba0b5f440a4a51169 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Wed, 30 Dec 2020 14:00:19 +0300 Subject: [PATCH 081/204] fingerprint by-path command --- aea/cli/fingerprint.py | 120 +++++++++++++++++++++++------ scripts/whitelist.py | 1 + tests/test_cli/test_fingerprint.py | 45 ++++++++++- 3 files changed, 143 insertions(+), 23 deletions(-) diff --git a/aea/cli/fingerprint.py b/aea/cli/fingerprint.py index ad05d65427..8eab79cf02 100644 --- a/aea/cli/fingerprint.py +++ b/aea/cli/fingerprint.py @@ -16,10 +16,10 @@ # limitations under the License. # # ------------------------------------------------------------------------------ - """Implementation of the 'aea add' subcommand.""" +import os from pathlib import Path -from typing import Dict +from typing import Dict, Union import click @@ -35,14 +35,24 @@ CONNECTION, CONTRACT, DEFAULT_CONNECTION_CONFIG_FILE, + DEFAULT_CONTRACT_CONFIG_FILE, DEFAULT_PROTOCOL_CONFIG_FILE, DEFAULT_SKILL_CONFIG_FILE, PROTOCOL, SKILL, ) +from aea.configurations.data_types import PackageType from aea.configurations.loader import ConfigLoader +CONFIG_FILE_TO_PACKAGE_TYPE = { + DEFAULT_SKILL_CONFIG_FILE: PackageType.SKILL, + DEFAULT_PROTOCOL_CONFIG_FILE: PackageType.PROTOCOL, + DEFAULT_CONNECTION_CONFIG_FILE: PackageType.CONNECTION, + DEFAULT_CONTRACT_CONFIG_FILE: PackageType.CONTRACT, +} # type: Dict[str, PackageType] + + @click.group() @click.pass_context def fingerprint(click_context: click.core.Context): # pylint: disable=unused-argument @@ -81,6 +91,19 @@ def skill(ctx: Context, skill_public_id: PublicId): fingerprint_item(ctx, SKILL, skill_public_id) +@fingerprint.command() +@click.argument("path", type=str, required=True) +@pass_ctx +def by_path(ctx: Context, path: str): + """Fingerprint a package by its path.""" + try: + click.echo("Fingerprinting component in '{}' ...".format(path)) + full_path = Path(ctx.cwd) / Path(path) + fingerprint_package_by_path(full_path) + except Exception as e: + raise click.ClickException(str(e)) + + def fingerprint_item(ctx: Context, item_type: str, item_public_id: PublicId) -> None: """ Fingerprint components of an item. @@ -99,25 +122,78 @@ def fingerprint_item(ctx: Context, item_type: str, item_public_id: PublicId) -> # create fingerprints package_dir = Path(ctx.cwd, item_type_plural, item_public_id.name) try: - default_config_file_name = _get_default_configuration_file_name_from_type( - item_type - ) - config_loader = ConfigLoader.from_configuration_type(item_type) - config_file_path = Path(package_dir, default_config_file_name) - config = config_loader.load(config_file_path.open()) - - if not package_dir.exists(): - # we only permit non-vendorized packages to be fingerprinted - raise click.ClickException( - "Package not found at path {}".format(package_dir) - ) - - fingerprints_dict = _compute_fingerprint( - package_dir, ignore_patterns=config.fingerprint_ignore_patterns - ) # type: Dict[str, str] - - # Load item specification yaml file and add fingerprints - config.fingerprint = fingerprints_dict - config_loader.dump(config, open(config_file_path, "w")) + fingerprint_package(package_dir, item_type) except Exception as e: raise click.ClickException(str(e)) + + +def fingerprint_package_by_path(package_dir: Path) -> None: + """ + Fingerprint package placed in package_dir. + + :param package_dir: directory of the package + + :return: None + """ + package_type = determine_package_type_for_directory(package_dir) + fingerprint_package(package_dir, package_type) + + +def determine_package_type_for_directory(package_dir: Path) -> PackageType: + """ + Find package type for the package directory by checking config file names. + + :param package_dir: package dir to determine pcakge type: + + :return: PackageType + """ + config_files = list( + set(os.listdir(str(package_dir))).intersection( + set(CONFIG_FILE_TO_PACKAGE_TYPE.keys()) + ) + ) + + if len(config_files) > 1: + raise ValueError( + f"Too many config files in the directory, only one has to present!: {', '.join(config_files)}" + ) + if len(config_files) == 0: + raise ValueError( + f"No package config file found in `{str(package_dir)}`. Incorrect directory?" + ) + + config_file = config_files[0] + package_type = CONFIG_FILE_TO_PACKAGE_TYPE[config_file] + + return package_type + + +def fingerprint_package( + package_dir: Path, package_type: Union[str, PackageType] +) -> None: + """ + Fingerprint components of an item. + + :param ctx: the context. + :param item_type: the item type. + :param item_public_id: the item public id. + :return: None + """ + package_type = PackageType(package_type) + item_type = str(package_type) + default_config_file_name = _get_default_configuration_file_name_from_type(item_type) + config_loader = ConfigLoader.from_configuration_type(item_type) + config_file_path = Path(package_dir, default_config_file_name) + config = config_loader.load(config_file_path.open()) + + if not package_dir.exists(): + # we only permit non-vendorized packages to be fingerprinted + raise ValueError("Package not found at path {}".format(package_dir)) + + fingerprints_dict = _compute_fingerprint( + package_dir, ignore_patterns=config.fingerprint_ignore_patterns + ) # type: Dict[str, str] + + # Load item specification yaml file and add fingerprints + config.fingerprint = fingerprints_dict + config_loader.dump(config, open(config_file_path, "w")) diff --git a/scripts/whitelist.py b/scripts/whitelist.py index 11e2ade7f0..121e35c9d8 100644 --- a/scripts/whitelist.py +++ b/scripts/whitelist.py @@ -239,3 +239,4 @@ ensure_dir # unused function (aea/helpers/base.py:561) _.get_agent_overridables # unused method (aea/manager/manager.py:419) _.set_agent_overrides # unused method (aea/manager/manager.py:432) +by_path # unused function (aea/cli/fingerprint.py:94) diff --git a/tests/test_cli/test_fingerprint.py b/tests/test_cli/test_fingerprint.py index 6179c190c1..325db9bd29 100644 --- a/tests/test_cli/test_fingerprint.py +++ b/tests/test_cli/test_fingerprint.py @@ -20,16 +20,21 @@ from unittest import TestCase, mock +import pytest from click import ClickException from aea.cli import cli from aea.cli.fingerprint import fingerprint_item +from aea.configurations.constants import ( + DEFAULT_CONNECTION_CONFIG_FILE, + DEFAULT_SKILL_CONFIG_FILE, +) from tests.conftest import CLI_LOG_OPTION, CliRunner from tests.test_cli.tools_for_testing import ConfigLoaderMock, ContextMock, PublicIdMock -@mock.patch("aea.cli.fingerprint.fingerprint_item") +@mock.patch("aea.cli.fingerprint.fingerprint_package") class FingerprintCommandTestCase(TestCase): """Test case for CLI fingerprint command.""" @@ -68,6 +73,44 @@ def test_fingerprint_positive(self, *mocks): ) self.assertEqual(result.exit_code, 0) + def _run_fingerprint_by_path(self): + """Call fingerprint by-path cli command.""" + result = self.runner.invoke( + cli, + [*CLI_LOG_OPTION, "fingerprint", "by-path", "some_dir"], + standalone_mode=False, + catch_exceptions=False, + ) + self.assertEqual(result.exit_code, 0, result.exception) + + def test_by_path_ok(self, fingerprint_mock): + """Test fingerprint by_path works ok.""" + with mock.patch("os.listdir", return_value=[DEFAULT_CONNECTION_CONFIG_FILE]): + self._run_fingerprint_by_path() + fingerprint_mock.assert_called() + + def test_by_path_exceptions(self, *mocks): + """Test fingerprint by_path works raises exceptions.""" + with pytest.raises( + ClickException, + match=f"No package config file found in `.*`. Incorrect directory?", + ): + with mock.patch("os.listdir", return_value=[]): + self._run_fingerprint_by_path() + + with pytest.raises( + ClickException, + match=f"Too many config files in the directory, only one has to present!", + ): + with mock.patch( + "os.listdir", + return_value=[ + DEFAULT_CONNECTION_CONFIG_FILE, + DEFAULT_SKILL_CONFIG_FILE, + ], + ): + self._run_fingerprint_by_path() + def _raise_exception(*args, **kwargs): raise Exception() From 8f3d0a7f607776b20b6db77748bfe3222fc57175 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Wed, 30 Dec 2020 12:09:40 +0100 Subject: [PATCH 082/204] minor refactoring of test_demo_docs.py --- tests/test_docs/helper.py | 7 ---- .../test_bash_yaml/test_demo_docs.py | 39 ++++++++++--------- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/tests/test_docs/helper.py b/tests/test_docs/helper.py index ac9e6046ee..34cd06a9e9 100644 --- a/tests/test_docs/helper.py +++ b/tests/test_docs/helper.py @@ -57,13 +57,6 @@ def extract_python_code(filepath): return python_str -def read_md_file(filepath): - """Reads an md file and returns the string.""" - with open(filepath, "r", encoding="utf-8") as md_file: - md_file_str = md_file.read() - return md_file_str - - def compile_and_exec(code: str, locals_dict: Dict = None) -> Dict: """ Compile and exec the code. diff --git a/tests/test_docs/test_bash_yaml/test_demo_docs.py b/tests/test_docs/test_bash_yaml/test_demo_docs.py index 85b1c5e31b..dcf0cc39af 100644 --- a/tests/test_docs/test_bash_yaml/test_demo_docs.py +++ b/tests/test_docs/test_bash_yaml/test_demo_docs.py @@ -24,7 +24,7 @@ from pathlib import Path from tests.conftest import ROOT_DIR -from tests.test_docs.helper import extract_code_blocks, read_md_file +from tests.test_docs.helper import extract_code_blocks logger = logging.getLogger(__name__) @@ -33,25 +33,26 @@ class TestDemoDocs: """This class contains the tests for the bash/yaml-blocks in *.md file.""" + BASH_DIR_PATH = Path(ROOT_DIR, "tests", "test_docs", "test_bash_yaml", "md_files") + + def _test_blocks(self, filename: str, filter_: str): + """Test code blocks of a certain type determined by 'filter_' param.""" + bash_file = Path(self.BASH_DIR_PATH, filename).read_text() + md_path = os.path.join(ROOT_DIR, "docs", filename.replace("bash-", "")) + + bash_code_blocks = extract_code_blocks(filepath=md_path, filter=filter_) + for blocks in bash_code_blocks: + assert blocks in bash_file, "[{}]: FAILED. Code must be identical".format( + filename + ) + logger.info("[{}]: PASSED".format(filename)) + def test_code_blocks_exist(self): """Test that all the code-blocks exist in the python file.""" - path = Path(ROOT_DIR, "tests", "test_docs", "test_bash_yaml", "md_files") - logger.info(os.listdir(path)) - for file in os.listdir(path): + logger.info(os.listdir(self.BASH_DIR_PATH)) + for file in os.listdir(self.BASH_DIR_PATH): if not file.endswith(".md"): continue - bash_file = read_md_file(filepath=Path(path, file)) - md_path = os.path.join(ROOT_DIR, "docs", file.replace("bash-", "")) - bash_code_blocks = extract_code_blocks(filepath=md_path, filter="bash") - for blocks in bash_code_blocks: - assert ( - blocks in bash_file - ), "[{}]: FAILED. Code must be identical".format(file) - logger.info("[{}]: PASSED".format(file)) - - yaml_code_blocks = extract_code_blocks(filepath=md_path, filter="yaml") - for blocks in yaml_code_blocks: - assert ( - blocks in bash_file - ), "[{}]: FAILED. Code must be identical".format(file) - logger.info("[{}]: PASSED".format(file)) + # in the future, we might add other filters like "python" + self._test_blocks(file, "bash") + self._test_blocks(file, "yaml") From 1e48555edec5c54e1a0ecf29e8cab83207e70e1c Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Wed, 30 Dec 2020 11:35:05 +0000 Subject: [PATCH 083/204] fix skill guide and build command no-key generation --- aea/aea_builder.py | 19 +++++++++-------- aea/cli/build.py | 4 +--- aea/configurations/manager.py | 5 +++-- aea/crypto/helpers.py | 15 +++++++------ docs/gym-skill.md | 2 +- docs/skill-guide.md | 40 +++++++++++++++++------------------ 6 files changed, 43 insertions(+), 42 deletions(-) diff --git a/aea/aea_builder.py b/aea/aea_builder.py index 237f095a59..c57ab1b042 100644 --- a/aea/aea_builder.py +++ b/aea/aea_builder.py @@ -1538,7 +1538,7 @@ def from_aea_project( cls, aea_project_path: PathLike, skip_consistency_check: bool = False, - verify_or_create_keys: bool = True, + create_keys: bool = True, ) -> "AEABuilder": """ Construct the builder from an AEA project. @@ -1552,7 +1552,7 @@ def from_aea_project( :param aea_project_path: path to the AEA project. :param skip_consistency_check: if True, the consistency check are skipped. - :param verify_or_create_keys: if True, verify_or_create_keys + :param create_keys: if True, verify_or_create_keys :return: an AEABuilder. """ aea_project_path = Path(aea_project_path) @@ -1560,19 +1560,20 @@ def from_aea_project( load_env_file(str(aea_project_path / DEFAULT_ENV_DOTFILE)) - if verify_or_create_keys: - # check and create missing, do not replace env variables. updates config - AgentConfigManager.verify_or_create_private_keys( - aea_project_path, - substitude_env_vars=False, - private_key_helper=private_key_verify_or_create, - ).dump_config() + # check and create missing, do not replace env variables. updates config + AgentConfigManager.verify_or_create_private_keys( + aea_project_path, + substitude_env_vars=False, + private_key_helper=private_key_verify_or_create, + create_keys=create_keys, + ).dump_config() # just validate agent_configuration = AgentConfigManager.verify_or_create_private_keys( aea_project_path, substitude_env_vars=True, private_key_helper=private_key_verify_or_create, + create_keys=create_keys, ).agent_config builder = AEABuilder(with_default_packages=False) diff --git a/aea/cli/build.py b/aea/cli/build.py index 0ad0cbb55d..b01a61c180 100644 --- a/aea/cli/build.py +++ b/aea/cli/build.py @@ -48,9 +48,7 @@ def build_aea(skip_consistency_check: bool) -> None: """ try: builder = AEABuilder.from_aea_project( - Path("."), - skip_consistency_check=skip_consistency_check, - verify_or_create_keys=False, + Path("."), skip_consistency_check=skip_consistency_check, create_keys=False, ) builder.call_all_build_entrypoints() except Exception as e: diff --git a/aea/configurations/manager.py b/aea/configurations/manager.py index 0e25eb09cd..7169197c8d 100644 --- a/aea/configurations/manager.py +++ b/aea/configurations/manager.py @@ -529,8 +529,9 @@ def dump_config(self) -> None: def verify_or_create_private_keys( cls, aea_project_path: Union[Path, str], - private_key_helper: Callable[[AgentConfig, Path], None], + private_key_helper: Callable[[AgentConfig, Path, bool], None], substitude_env_vars: bool = False, + create_keys: bool = True, ) -> "AgentConfigManager": """ Verify or create private keys. @@ -548,7 +549,7 @@ def verify_or_create_private_keys( aea_project_path, substitude_env_vars=substitude_env_vars ) aea_conf = agent_config_manager.agent_config - private_key_helper(aea_conf, Path(aea_project_path)) + private_key_helper(aea_conf, Path(aea_project_path), create_keys) return agent_config_manager def get_overridables(self) -> Tuple[Dict, Dict[ComponentId, Dict]]: diff --git a/aea/crypto/helpers.py b/aea/crypto/helpers.py index 4bd8e2b626..a3186939c3 100644 --- a/aea/crypto/helpers.py +++ b/aea/crypto/helpers.py @@ -92,7 +92,9 @@ def try_generate_testnet_wealth( faucet_api.get_wealth(address, url) -def private_key_verify_or_create(aea_conf: AgentConfig, aea_project_path: Path) -> None: +def private_key_verify_or_create( + aea_conf: AgentConfig, aea_project_path: Path, create_keys: bool = True +) -> None: """ Check key or create if none present. @@ -125,11 +127,12 @@ def private_key_verify_or_create(aea_conf: AgentConfig, aea_project_path: Path) repr(config_private_key_path), identifier ) ) - create_private_key( - identifier, - private_key_file=str(aea_project_path / private_key_path), - ) - aea_conf.private_key_paths.update(identifier, private_key_path) + if create_keys: + create_private_key( + identifier, + private_key_file=str(aea_project_path / private_key_path), + ) + aea_conf.private_key_paths.update(identifier, private_key_path) else: try: try_validate_private_key_path( diff --git a/docs/gym-skill.md b/docs/gym-skill.md index 473d6c6758..c54626b617 100644 --- a/docs/gym-skill.md +++ b/docs/gym-skill.md @@ -119,7 +119,7 @@ The `GymTask` is responsible for training the RL agent. In particular, `MyRLAgen In this particular skill, which chiefly serves for demonstration purposes, we implement a very basic RL agent. The agent trains a model of price of `n` goods: it aims to discover the most likely price of each good. To this end, the agent randomly selects one of the `n` goods on each training step and then chooses as an `action` the price which it deems is most likely accepted. Each good is represented by an id and the possible price range `[1,100]` divided into 100 integer bins. For each price bin, a `PriceBandit` is created which models the likelihood of this price. In particular, a price bandit maintains a beta distribution. The beta distribution is initialized to the uniform distribution. Each time the price associated with a given `PriceBandit` is accepted or rejected the distribution maintained by the `PriceBandit` is updated. For each good, the agent can therefore over time learn which price is most likely. -Gym skill illustration +Gym skill illustration The illustration shows how the RL agent only interacts with the proxy environment by sending it `action (A)` and receiving `observation (O)`, `reward (R)`, `done (D)` and `info (I)`. diff --git a/docs/skill-guide.md b/docs/skill-guide.md index 955fc1e912..b95f032272 100644 --- a/docs/skill-guide.md +++ b/docs/skill-guide.md @@ -410,28 +410,26 @@ aea add protocol fetchai/oef_search:0.11.0 This adds the protocol to our AEA and makes it available on the path `packages.fetchai.protocols...`. -We also need to add the soef and p2p connections and install the AEA's dependencies: +We also need to add the soef and p2p connections and install the AEA's dependencies as well as configure the AEA: ``` bash aea add connection fetchai/soef:0.14.0 aea add connection fetchai/p2p_libp2p:0.13.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` -Finally, in the `aea-config.yaml` add the following lines: -``` yaml -default_routing: - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` - -This will ensure that search requests are processed by the correct connection. +The last command will ensure that search requests are processed by the correct connection. ## Step 8: Run a service provider AEA In order to be able to find another AEA when searching, from a different terminal window, we fetch another finished AEA and install its Python dependencies: ``` bash -aea fetch fetchai/simple_service_registration:0.18.0 && cd simple_service_registration && aea install +aea fetch fetchai/simple_service_registration:0.18.0 && cd simple_service_registration && aea install && aea build ``` This AEA will simply register a location service on the SOEF search node so we can search for it. @@ -448,7 +446,7 @@ Then we run the aea: aea run ``` -Once you see a message of the form `To join its network use multiaddr: ['SOME_ADDRESS']` take note of the address. +Once you see a message of the form `To join its network use multiaddr: ['SOME_ADDRESS']` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) This is the entry peer address for the local agent communication network created by the simple_service_registration AEA.
    Click here to see full code

    @@ -874,18 +872,18 @@ aea add-key fetchai fetchai_private_key.txt aea add-key fetchai fetchai_private_key.txt --connection ``` -Then, update the configuration of the search AEA's p2p connection (in `vendor/fetchai/connections/p2p_libp2p/connection.yaml`) replace the following: - -``` yaml -config: - delegate_uri: 127.0.0.1:11001 - entry_peers: ['SOME_ADDRESS'] - local_uri: 127.0.0.1:9001 - log_file: libp2p_node.log - public_uri: 127.0.0.1:9001 +Then, in the search AEA, run this command (replace `SOME_ADDRESS` with the correct value as described above): +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["/dns4/127.0.0.1/tcp/9000/p2p/16Uiu2HAm1uJpFsqSgHStJdtTBPpDme1fo8uFEvvY182D2y89jQuj"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' ``` - -where `SOME_ADDRESS` is replaced accordingly. +This allows the search AEA to connect to the same local agent communication network as the service registration AEA. We can then launch our AEA. From bf4ce6eab42e02e82f885f4d1e0f4af25bfffbdb Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Wed, 30 Dec 2020 12:38:27 +0100 Subject: [PATCH 084/204] do further refactoring to test_demo_docs.py --- tests/test_docs/helper.py | 16 ++++++++++++++++ .../test_language_agnostic_definition.py | 19 +++++++------------ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/test_docs/helper.py b/tests/test_docs/helper.py index 34cd06a9e9..1b64808239 100644 --- a/tests/test_docs/helper.py +++ b/tests/test_docs/helper.py @@ -20,8 +20,10 @@ """This module contains helper function to extract code from the .md files.""" import re import traceback +from pathlib import Path from typing import Dict +import mistune import pytest @@ -93,3 +95,17 @@ def compare_enum_classes(expected_enum_class, actual_enum_class): expected_enum_class, actual_enum_class ) ) + + +class BaseTestMarkdownDocs: + """Base test class for testing Markdown documents.""" + + DOC_PATH: Path + + @classmethod + def setup_class(cls): + """Set up the test.""" + markdown_parser = mistune.create_markdown(renderer=mistune.AstRenderer()) + cls.doc_path = cls.DOC_PATH + cls.doc_content = cls.doc_path.read_text() + cls.blocks = markdown_parser(cls.doc_content) diff --git a/tests/test_docs/test_language_agnostic_definition.py b/tests/test_docs/test_language_agnostic_definition.py index aadf480c24..0b9ccf6e6e 100644 --- a/tests/test_docs/test_language_agnostic_definition.py +++ b/tests/test_docs/test_language_agnostic_definition.py @@ -21,12 +21,11 @@ from pathlib import Path from typing import Dict -import mistune - from aea import AEA_DIR from aea.configurations.data_types import PublicId from tests.conftest import ROOT_DIR +from tests.test_docs.helper import BaseTestMarkdownDocs MAIL_BASE_PROTO = Path(AEA_DIR) / "mail" / "base.proto" @@ -35,9 +34,11 @@ ) -class TestLanguageAgnosticDocs: +class TestLanguageAgnosticDocs(BaseTestMarkdownDocs): """Test the integrity of the language agnostic definitions in docs.""" + DOC_PATH = Path(ROOT_DIR, "docs", "language-agnostic-definition.md") + @classmethod def _proto_snippet_selector(cls, block: Dict) -> bool: return block["type"] == "block_code" and block["info"].strip() == "proto" @@ -45,14 +46,8 @@ def _proto_snippet_selector(cls, block: Dict) -> bool: @classmethod def setup_class(cls): """Set up the test.""" - markdown_parser = mistune.create_markdown(renderer=mistune.AstRenderer()) - - cls.spec_doc_file = Path(ROOT_DIR, "docs", "language-agnostic-definition.md") - cls.spec_doc_content = cls.spec_doc_file.read_text() - doc = markdown_parser(cls.spec_doc_content) - # get only code blocks - cls.code_blocks = list(filter(cls._proto_snippet_selector, doc)) - + super().setup_class() + cls.code_blocks = list(filter(cls._proto_snippet_selector, cls.blocks)) cls.actual_mail_base_file_content = MAIL_BASE_PROTO.read_text() cls.actual_default_message_file_content = DEFAULT_MESSAGE_PROTO.read_text() @@ -88,7 +83,7 @@ def test_dialogue_message_code_snippet(self): def test_public_id_regular_expression(self): """Test public id regular expression is the same.""" expected_regex = PublicId.PUBLIC_ID_REGEX - assert expected_regex in self.spec_doc_content + assert expected_regex in self.doc_content def test_default_message_code_snippet(self): """Test DefaultMessage protobuf code snippet.""" From 52c9e5f193cb8f0f56a98abfd66bcd8ee4fccb60 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Wed, 30 Dec 2020 11:46:12 +0000 Subject: [PATCH 085/204] builder docstring fix --- aea/aea_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aea/aea_builder.py b/aea/aea_builder.py index c57ab1b042..b37597dcad 100644 --- a/aea/aea_builder.py +++ b/aea/aea_builder.py @@ -1552,7 +1552,7 @@ def from_aea_project( :param aea_project_path: path to the AEA project. :param skip_consistency_check: if True, the consistency check are skipped. - :param create_keys: if True, verify_or_create_keys + :param create_keys: if True, create keys, otherwise just verify :return: an AEABuilder. """ aea_project_path = Path(aea_project_path) From fe1948da6790e314b0cb97711025ba4fd7ec3ae8 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Wed, 30 Dec 2020 12:12:38 +0000 Subject: [PATCH 086/204] fix tests skill guide --- tests/test_configurations/test_manager.py | 2 +- .../md_files/bash-skill-guide.md | 28 ++++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/test_configurations/test_manager.py b/tests/test_configurations/test_manager.py index 95d41c8805..946cc21756 100644 --- a/tests/test_configurations/test_manager.py +++ b/tests/test_configurations/test_manager.py @@ -230,7 +230,7 @@ def test_agent_attribute_get_set(): agent_config_manager.get_variable("agent.does_not_exist") agent_config_manager.validate_current_config() - agent_config_manager.verify_or_create_private_keys(DUMMY_AEA, lambda x, y: None) + agent_config_manager.verify_or_create_private_keys(DUMMY_AEA, lambda x, y, z: None) def test_agent_attribute_get_overridables(): diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-skill-guide.md b/tests/test_docs/test_bash_yaml/md_files/bash-skill-guide.md index c1bca3c3a2..c4e6c7c3dc 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-skill-guide.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-skill-guide.md @@ -14,9 +14,13 @@ aea add connection fetchai/p2p_libp2p:0.13.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` ``` bash -aea fetch fetchai/simple_service_registration:0.18.0 && cd simple_service_registration && aea install +aea fetch fetchai/simple_service_registration:0.18.0 && cd simple_service_registration && aea install && aea build ``` ``` bash aea generate-key fetchai @@ -32,6 +36,16 @@ aea add-key fetchai fetchai_private_key.txt aea add-key fetchai fetchai_private_key.txt --connection ``` ``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["/dns4/127.0.0.1/tcp/9000/p2p/16Uiu2HAm1uJpFsqSgHStJdtTBPpDme1fo8uFEvvY182D2y89jQuj"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +``` bash aea run ``` ``` yaml @@ -72,10 +86,6 @@ models: dependencies: {} ``` ``` yaml -default_routing: - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml name: simple_service_registration author: fetchai version: 0.4.0 @@ -117,12 +127,4 @@ models: value: generic_service class_name: Strategy dependencies: {} -``` -``` yaml -config: - delegate_uri: 127.0.0.1:11001 - entry_peers: ['SOME_ADDRESS'] - local_uri: 127.0.0.1:9001 - log_file: libp2p_node.log - public_uri: 127.0.0.1:9001 ``` \ No newline at end of file From c81d68738476c38acddf034e1a01e50debf11776 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Wed, 30 Dec 2020 16:28:53 +0300 Subject: [PATCH 087/204] fixes --- aea/cli/fingerprint.py | 15 ++------------- aea/configurations/constants.py | 9 ++++++++- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/aea/cli/fingerprint.py b/aea/cli/fingerprint.py index 8eab79cf02..6bf185cde8 100644 --- a/aea/cli/fingerprint.py +++ b/aea/cli/fingerprint.py @@ -32,12 +32,9 @@ _get_default_configuration_file_name_from_type, ) from aea.configurations.constants import ( # noqa: F401 # pylint: disable=unused-import + CONFIG_FILE_TO_PACKAGE_TYPE, CONNECTION, CONTRACT, - DEFAULT_CONNECTION_CONFIG_FILE, - DEFAULT_CONTRACT_CONFIG_FILE, - DEFAULT_PROTOCOL_CONFIG_FILE, - DEFAULT_SKILL_CONFIG_FILE, PROTOCOL, SKILL, ) @@ -45,14 +42,6 @@ from aea.configurations.loader import ConfigLoader -CONFIG_FILE_TO_PACKAGE_TYPE = { - DEFAULT_SKILL_CONFIG_FILE: PackageType.SKILL, - DEFAULT_PROTOCOL_CONFIG_FILE: PackageType.PROTOCOL, - DEFAULT_CONNECTION_CONFIG_FILE: PackageType.CONNECTION, - DEFAULT_CONTRACT_CONFIG_FILE: PackageType.CONTRACT, -} # type: Dict[str, PackageType] - - @click.group() @click.pass_context def fingerprint(click_context: click.core.Context): # pylint: disable=unused-argument @@ -163,7 +152,7 @@ def determine_package_type_for_directory(package_dir: Path) -> PackageType: ) config_file = config_files[0] - package_type = CONFIG_FILE_TO_PACKAGE_TYPE[config_file] + package_type = PackageType(CONFIG_FILE_TO_PACKAGE_TYPE[config_file]) return package_type diff --git a/aea/configurations/constants.py b/aea/configurations/constants.py index d7ed3d0547..b2577b0e1f 100644 --- a/aea/configurations/constants.py +++ b/aea/configurations/constants.py @@ -19,7 +19,7 @@ """Module to declare constants.""" from pathlib import Path -from typing import List +from typing import Dict, List FETCHAI = "fetchai" @@ -79,3 +79,10 @@ DEFAULT_ENV_DOTFILE = ".env" DOTTED_PATH_MODULE_ELEMENT_SEPARATOR = ":" LIBPROTOC_VERSION = "libprotoc 3.11.4" + +CONFIG_FILE_TO_PACKAGE_TYPE = { + DEFAULT_SKILL_CONFIG_FILE: SKILL, + DEFAULT_PROTOCOL_CONFIG_FILE: PROTOCOL, + DEFAULT_CONNECTION_CONFIG_FILE: CONNECTION, + DEFAULT_CONTRACT_CONFIG_FILE: CONTRACT, +} # type: Dict[str, str] From 530f94d8a5b92b5a7def5ac2bc76895d9199dba7 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Wed, 30 Dec 2020 13:36:46 +0000 Subject: [PATCH 088/204] modify navigation in docs --- docs/diagram.md | 8 ++++---- mkdocs.yml | 19 ++++++++++--------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/diagram.md b/docs/diagram.md index a4b9d784ea..277d68d05b 100644 --- a/docs/diagram.md +++ b/docs/diagram.md @@ -5,10 +5,10 @@ The framework has two distinctive parts. Currently, the framework supports four types of packages which can be added to the core as modules: -- Skills -- Protocols -- Connections -- Contracts +- Skills are the core focus of the framework's extensibility as they implement business logic to deliver economic value for the AEA. +- Protocols define agent-to-agent as well as component-to-component interactions (messages and dialogues) within agents. +- Connections wrap SDKs or APIs and provide an interface to network, ledgers and other services. +- Contracts wrap smart contracts for Fetch.ai and third-party decentralized ledgers. The following figure illustrates the framework's architecture: diff --git a/mkdocs.yml b/mkdocs.yml index 09ffeecfc0..2a9b3c6433 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -23,7 +23,7 @@ nav: - Application areas: 'app-areas.md' - Relation to OEF and Ledger: 'oef-ledger.md' - Identity: 'identity.md' - # - Trust issues: 'trust.md' + - Trust issues: 'trust.md' - Demos: - Demos: 'demos.md' - Aries Cloud Agents Demo: 'aries-cloud-agent-demo.md' @@ -37,14 +37,15 @@ nav: - TAC external app: 'tac.md' - Thermometer skills: 'thermometer-skills.md' - Weather skills: 'weather-skills.md' - - Development: - - Getting started: - - AEA quick start: 'quickstart.md' - - Core components - Part 1: 'core-components-1.md' - - AEA and web frameworks: 'aea-vs-mvc.md' - - Build a skill for an AEA: 'skill-guide.md' - - Core components - Part 2: 'core-components-2.md' - - Trade between two AEAs: 'generic-skills-step-by-step.md' + - Development - Beginner: + - AEA quick start: 'quickstart.md' + - Core components - Part 1: 'core-components-1.md' + - AEA and web frameworks: 'aea-vs-mvc.md' + - Build a skill for an AEA: 'skill-guide.md' + - Core components - Part 2: 'core-components-2.md' + - Interaction protocols: 'interaction-protocol.md' + - Trade between two AEAs: 'generic-skills-step-by-step.md' + - Development - Advanced: - Topic guides: - Ways to build an AEA: 'step-one.md' - Development setup: 'development-setup.md' From 4eb7f67c154d98d85a5295b0cb20b94184e2b580 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Wed, 30 Dec 2020 14:59:47 +0100 Subject: [PATCH 089/204] add tests for 'defining-data-models.md' --- docs/defining-data-models.md | 6 ++--- tests/test_docs/helper.py | 26 ++++++++++++++++++ tests/test_docs/test_data_models.py | 42 +++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 tests/test_docs/test_data_models.py diff --git a/docs/defining-data-models.md b/docs/defining-data-models.md index c8aba07eb4..f09c2f72c5 100644 --- a/docs/defining-data-models.md +++ b/docs/defining-data-models.md @@ -37,7 +37,7 @@ Finally, every attribute might have a _description_ that explains the purpose of For each of this fields, we can define an attribute by using `Attribute`: ``` python -from aea.helpers.search import Attribute, Location +from aea.helpers.search.models import Attribute, Location attr_title = Attribute("title", str, True, "The title of the book.") attr_author = Attribute("author", str, True, "The author of the book.") attr_genre = Attribute("genre", str, True, "The genre of the book.") @@ -64,7 +64,7 @@ A _data model_ is just a set of _attributes_. The class that implements the data in the same structure. We can do it in the following way: ``` python -from aea.helpers.search import DataModel +from aea.helpers.search.models import DataModel book_model = DataModel("book", [ attr_title, @@ -93,7 +93,7 @@ The class that implements the description is `Description`. **Example**: now we have all we need to create a little catalogue about our books: ``` python -from oef.schema import Description +from aea.helpers.search.models import Description It = Description({ "title" : "It", diff --git a/tests/test_docs/helper.py b/tests/test_docs/helper.py index 1b64808239..4548eb1bd7 100644 --- a/tests/test_docs/helper.py +++ b/tests/test_docs/helper.py @@ -20,6 +20,8 @@ """This module contains helper function to extract code from the .md files.""" import re import traceback +from abc import abstractmethod +from copy import copy from pathlib import Path from typing import Dict @@ -109,3 +111,27 @@ def setup_class(cls): cls.doc_path = cls.DOC_PATH cls.doc_content = cls.doc_path.read_text() cls.blocks = markdown_parser(cls.doc_content) + + +class BasePythonMarkdownDocs(BaseTestMarkdownDocs): + """Test Markdown documentation by running Python snippets in sequence.""" + + @classmethod + def _python_selector(cls, block: Dict) -> bool: + return block["type"] == "block_code" and block["info"].strip() == "python" + + @abstractmethod + def _assert(self, **locals_): + """Do assertions after Python code execution.""" + + def test_python_blocks(self): + """Run Python code block in sequence.""" + python_blocks = list(filter(self._python_selector, self.blocks)) + + for python_block in python_blocks: + python_code = python_block["text"] + exec(python_code) + + locals_ = copy(locals()) + locals_.pop("self") + self._assert(**locals_) diff --git a/tests/test_docs/test_data_models.py b/tests/test_docs/test_data_models.py new file mode 100644 index 0000000000..ec22b684b3 --- /dev/null +++ b/tests/test_docs/test_data_models.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the tests for the content of data-models.md file.""" +from pathlib import Path + +from aea.helpers.search.models import Attribute, DataModel, Description + +from tests.conftest import ROOT_DIR +from tests.test_docs.helper import BasePythonMarkdownDocs + + +class TestDataModel(BasePythonMarkdownDocs): + """Test the integrity of the language agnostic definitions in docs.""" + + DOC_PATH = Path(ROOT_DIR, "docs", "defining-data-models.md") + + def _assert(self, **locals_): + attribute = locals_["attr_title"] + assert isinstance(attribute, Attribute) + + data_model = locals_["book_model"] + assert isinstance(data_model, DataModel) + + description = locals_["It"] + assert isinstance(description, Description) From eb6afd228792e13e72f53d3e2d37abd2e4b25deb Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Wed, 30 Dec 2020 15:10:57 +0100 Subject: [PATCH 090/204] fix security check --- tests/test_docs/helper.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_docs/helper.py b/tests/test_docs/helper.py index 4548eb1bd7..d3b7e4c289 100644 --- a/tests/test_docs/helper.py +++ b/tests/test_docs/helper.py @@ -21,7 +21,6 @@ import re import traceback from abc import abstractmethod -from copy import copy from pathlib import Path from typing import Dict @@ -128,10 +127,8 @@ def test_python_blocks(self): """Run Python code block in sequence.""" python_blocks = list(filter(self._python_selector, self.blocks)) + globals_, locals_ = {}, {} for python_block in python_blocks: python_code = python_block["text"] - exec(python_code) - - locals_ = copy(locals()) - locals_.pop("self") + exec(python_code, globals_, locals_) # nosec self._assert(**locals_) From f0d0327fa512026dbdef2ee5f576d7255722b609 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Wed, 30 Dec 2020 17:11:07 +0100 Subject: [PATCH 091/204] add consistency docs tests for 'generic-storage.md' --- docs/generic-storage.md | 69 ++++++++++++------------- tests/test_docs/helper.py | 9 ++++ tests/test_docs/test_generic_storage.py | 64 +++++++++++++++++++++++ 3 files changed, 106 insertions(+), 36 deletions(-) create mode 100644 tests/test_docs/test_generic_storage.py diff --git a/docs/generic-storage.md b/docs/generic-storage.md index 21b9eccd7b..a2645197d8 100644 --- a/docs/generic-storage.md +++ b/docs/generic-storage.md @@ -63,52 +63,49 @@ To get/put specific object collection instance should be used. Collection instance provide set of methods to handle data objects. List of collection methods: ``` -def put(self, object_id: str, object_body: JSON_TYPES) -> None: - """ - Put object into collection. + def put(self, object_id: str, object_body: JSON_TYPES) -> None: + """ + Put object into collection. - :param object_id: str object id - :param object_body: python dict, json compatible. - :return: None - """ + :param object_id: str object id + :param object_body: python dict, json compatible. + :return: None + """ -def get(self, object_id: str) -> Optional[JSON_TYPES]: - """ - Get object from the collection. + def get(self, object_id: str) -> Optional[JSON_TYPES]: + """ + Get object from the collection. - :param object_id: str object id + :param object_id: str object id - :return: dict if object exists in collection otherwise None - """ + :return: dict if object exists in collection otherwise None + """ + def remove(self, object_id: str) -> None: + """ + Remove object from the collection. -def remove(self, object_id: str) -> None: - """ - Remove object from the collection. + :param object_id: str object id - :param object_id: str object id + :return: None + """ - :return: None - """ + def find(self, field: str, equals: EQUALS_TYPE) -> List[OBJECT_ID_AND_BODY]: + """ + Get objects from the collection by filtering by field value. + :param field: field name to search: example "parent.field" + :param equals: value field should be equal to -def find(self, field: str, equals: EQUALS_TYPE) -> List[OBJECT_ID_AND_BODY]: - """ - Get objects from the collection by filtering by field value. + :return: List of object bodies + """ - :param field: field name to search: example "parent.field" - :param equals: value field should be equal to + def list(self) -> List[OBJECT_ID_AND_BODY]: + """ + List all objects with keys from the collection. - :return: List of object bodies - """ - - -def list(self) -> List[OBJECT_ID_AND_BODY]: - """ - List all objects with keys from the collection. - - :return: Tuple of objects keys, bodies. - """ + :return: Tuple of objects keys, bodies. + """ ``` @@ -130,8 +127,8 @@ class TestBehaviour(TickerBehaviour): collection = self.context.storage.get_sync_collection('my_collection') first_call_datetime = collection.get("first_call_ts") if not first_call_ts: - # there is no object with "first_call_ts" id. - first_call_datetime = str(datetime.datetime.now()) + # there is no object with "first_call_ts" id. + first_call_datetime = str(datetime.datetime.now()) col.put(first_call_ts, first_call_datetime) print("Act was called for the first time on:", first_call_datetime) ``` diff --git a/tests/test_docs/helper.py b/tests/test_docs/helper.py index d3b7e4c289..6ff0dd720b 100644 --- a/tests/test_docs/helper.py +++ b/tests/test_docs/helper.py @@ -28,6 +28,9 @@ import pytest +MISTUNE_BLOCK_CODE_ID = "block_code" + + def extract_code_blocks(filepath, filter=None): """Extract code blocks from .md files.""" code_blocks = [] @@ -103,6 +106,11 @@ class BaseTestMarkdownDocs: DOC_PATH: Path + @classmethod + def _block_code_filter(cls, b: Dict) -> bool: + """Check Mistune block is a code block.""" + return b["type"] == MISTUNE_BLOCK_CODE_ID + @classmethod def setup_class(cls): """Set up the test.""" @@ -110,6 +118,7 @@ def setup_class(cls): cls.doc_path = cls.DOC_PATH cls.doc_content = cls.doc_path.read_text() cls.blocks = markdown_parser(cls.doc_content) + cls.code_blocks = list(filter(cls._block_code_filter, cls.blocks)) class BasePythonMarkdownDocs(BaseTestMarkdownDocs): diff --git a/tests/test_docs/test_generic_storage.py b/tests/test_docs/test_generic_storage.py new file mode 100644 index 0000000000..3cdfb36f91 --- /dev/null +++ b/tests/test_docs/test_generic_storage.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the tests for the content of generic-storage.md file.""" +import inspect +import re +from pathlib import Path + +from aea.helpers.storage.generic_storage import SyncCollection +from aea.skills.behaviours import TickerBehaviour + +from tests.conftest import ROOT_DIR +from tests.test_docs.helper import BaseTestMarkdownDocs + + +class TestGenericStorage(BaseTestMarkdownDocs): + """Test generic-storage.md documentation.""" + + DOC_PATH = Path(ROOT_DIR, "docs", "generic-storage.md") + + @classmethod + def setup_class(cls): + """ + Set up test. + + We test only the signatures and the + docstrings of 'SyncCollection' methods. Hence, + we need to remove the return statement. + """ + super().setup_class() + cls.content = inspect.getsource(SyncCollection) + cls.content = re.sub("\n +return.*", "", cls.content) + + def test_storage_abstract_methods(self): + """ + Test storage abstract methods. + + Note: the block index of the abstract methods is 0. + Please check generic-storage.md + """ + block_index = 0 + assert self.code_blocks[block_index]["text"] in self.content + + def test_test_behaviour(self): + """Test that the 'TestBehaviour' code is compilable.""" + block_index = 1 + code = self.code_blocks[block_index]["text"] + exec(code, {}, dict(TickerBehaviour=TickerBehaviour)) From 028f200a9ba5afa63fe30a068249f83d5beeef0a Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Wed, 30 Dec 2020 13:27:08 +0000 Subject: [PATCH 092/204] Use CertRequest for p2p connections PoR --- aea/helpers/acn/agent_record.py | 22 +++- aea/helpers/base.py | 12 +- aea/test_tools/test_cases.py | 4 +- .../connections/p2p_libp2p/connection.py | 42 ++++--- .../connections/p2p_libp2p/connection.yaml | 15 ++- .../p2p_libp2p_client/connection.py | 103 +++++++++++++----- .../p2p_libp2p_client/connection.yaml | 17 ++- packages/hashes.csv | 9 +- tests/conftest.py | 73 ++++++++++--- tests/data/dummy_connection/connection.yaml | 4 +- tests/data/hashes.csv | 2 +- .../test_p2p_libp2p/test_aea_cli.py | 21 +++- .../test_p2p_libp2p/test_errors.py | 26 +---- .../test_p2p_libp2p/test_fault_tolerance.py | 6 +- .../test_p2p_libp2p/test_public_dht.py | 19 +++- .../test_p2p_libp2p_client/test_aea_cli.py | 22 +++- .../test_communication.py | 18 ++- .../test_p2p_libp2p_client/test_errors.py | 29 ++++- 18 files changed, 316 insertions(+), 128 deletions(-) diff --git a/aea/helpers/acn/agent_record.py b/aea/helpers/acn/agent_record.py index 5d92df8608..06455610b0 100644 --- a/aea/helpers/acn/agent_record.py +++ b/aea/helpers/acn/agent_record.py @@ -21,12 +21,24 @@ import base64 from hashlib import sha256 +from typing import List from ecdsa import SECP256k1, VerifyingKey from ecdsa import util as ecdsa_util from aea.crypto.fetchai import FetchAIHelper +def recover_verify_keys_from_message(message: bytes, signature: str) -> List[str]: + """ get the public key used to produce the `signature` of the `message`""" + signature_b64 = base64.b64decode(signature) + verifying_keys = VerifyingKey.from_public_key_recovery( + signature_b64, message, SECP256k1, hashfunc=sha256, + ) + public_keys = [ + verifying_key.to_string("compressed").hex() + for verifying_key in verifying_keys + ] + return public_keys def verify_message(msg: bytes, signature: str, public_key: str) -> bool: """ @@ -91,7 +103,7 @@ def service_id(self) -> str: def __str__(self): """Get string representation.""" - return f"(address={self.address}, pubkey={self.public_key}, peer_pubkey={self.peer_public_key}, signature={self.signature})" + return f"(address={self.address}, public_key={self.public_key}, peer_public_key={self.peer_public_key}, signature={self.signature})" def is_valid_for(self, address: str, peer_public_key: str) -> bool: """ @@ -103,13 +115,17 @@ def is_valid_for(self, address: str, peer_public_key: str) -> bool: """ if self._address != address: + print("Wrong address") return False if self._peer_public_key != peer_public_key: + print("Wrong peer public key") return False if self._address != FetchAIHelper.get_address_from_public_key(self._public_key): + print(f"Wrong address '{self._address}' and public key '{FetchAIHelper.get_address_from_public_key(self._public_key)}'") return False - if not verify_message( - self._peer_public_key.encode("utf-8"), self._signature, self._public_key + if self._address not in FetchAIHelper.recover_message( + self._peer_public_key.encode("utf-8"), self._signature ): + print("Wrong signature") return False return True diff --git a/aea/helpers/base.py b/aea/helpers/base.py index 740363e0c5..7de7fe0bd4 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -626,10 +626,10 @@ def __init__( :param public_key: the public key, or the key id. :param identifier: certificate identifier. - :param not_before: specify the lower bound for certificate vailidity. + :param not_before: specify the lower bound for certificate validity. If it is a string, it must follow the format: 'YYYY-MM-DD'. It will be interpreted as timezone UTC. - :param not_before: specify the lower bound for certificate vailidity. + :param not_before: specify the lower bound for certificate validity. if it is a string, it must follow the format: 'YYYY-MM-DD' It will be interpreted as timezone UTC-0. :param save_path: the save_path where to save the certificate. @@ -642,7 +642,7 @@ def __init__( self._not_after_string = not_after self._not_before = self._parse_datetime(not_before) self._not_after = self._parse_datetime(not_after) - self._save_path = Path(save_path) + self._save_path = Path(os.path.abspath(save_path)) self._parse_public_key(public_key) self._check_validation_boundaries() @@ -750,9 +750,9 @@ def get_message(self, public_key: str) -> bytes: """Get the message to sign.""" message = ( public_key.encode("ascii") - + self.identifier.encode("ascii") - + self.not_before_string.encode("ascii") - + self.not_after_string.encode("ascii") + #+ self.identifier.encode("ascii") + #+ self.not_before_string.encode("ascii") + #+ self.not_after_string.encode("ascii") ) return message diff --git a/aea/test_tools/test_cases.py b/aea/test_tools/test_cases.py index d836900d6d..fa6ffaf53e 100644 --- a/aea/test_tools/test_cases.py +++ b/aea/test_tools/test_cases.py @@ -875,8 +875,8 @@ def teardown_class(cls): cls.stdout = {} cls.stderr = {} - with suppress(OSError, IOError): - shutil.rmtree(cls.t) + #with suppress(OSError, IOError): + # shutil.rmtree(cls.t) cls._is_teardown_class_called = True diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index 176d073c82..881dd84dff 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -17,6 +17,9 @@ # # ------------------------------------------------------------------------------ """This module contains the p2p libp2p connection.""" +from aea.crypto.fetchai import FetchAIHelper + +from aea.helpers.base import CertRequest import asyncio import logging import os @@ -28,7 +31,7 @@ from ipaddress import ip_address from pathlib import Path from socket import gethostbyname -from typing import IO, List, Optional, Sequence, cast +from typing import IO, List, Optional, Sequence, Tuple, cast from aea.configurations.base import PublicId from aea.configurations.constants import DEFAULT_LEDGER @@ -36,7 +39,7 @@ from aea.crypto.base import Crypto from aea.crypto.registries import make_crypto from aea.exceptions import enforce -from aea.helpers.acn.agent_record import AgentRecord +from aea.helpers.acn.agent_record import AgentRecord, recover_verify_keys_from_message from aea.helpers.acn.uri import Uri from aea.helpers.multiaddr.base import MultiAddr from aea.helpers.pipe import IPCChannel, make_ipc_channel @@ -71,7 +74,7 @@ LIBP2P_SUCCESS_MESSAGE = "Peer running in " -POR_DEFAULT_SERVICE_ID = "" +POR_DEFAULT_SERVICE_ID = "acn" def _ip_all_private_or_all_public(addrs: List[str]) -> bool: @@ -430,14 +433,15 @@ def stop(self) -> None: if os.path.exists(LIBP2P_NODE_ENV_FILE): os.remove(LIBP2P_NODE_ENV_FILE) - -def _get_por_from_configuration(key: Crypto) -> AgentRecord: - signature = key.sign_message(key.public_key.encode("utf-8")) - record = AgentRecord( - key.address, key.public_key, key.public_key, signature, POR_DEFAULT_SERVICE_ID - ) - return record - +def _get_signature_from_cert_request(cert: CertRequest, node_public_key: str, agent_address: str) -> Tuple[str, str]: + # must match aea/cli/issue_certificates.py:_process_certificate + signature = bytes.fromhex(Path(cert.save_path).read_bytes().decode("ascii")).decode("ascii") + public_keys = recover_verify_keys_from_message(cert.get_message(node_public_key), signature) + addresses = [ + FetchAIHelper.get_address_from_public_key(public_key) for public_key in public_keys + ] + verify_key = public_keys[addresses.index(agent_address)] + return signature, verify_key class P2PLibp2pConnection(Connection): """A libp2p p2p node connection.""" @@ -456,9 +460,6 @@ def __init__(self, **kwargs): ledger_id, SUPPORTED_LEDGER_IDS ) ) - libp2p_key_file = self.configuration.config.get( - "node_key_file" - ) # Optional[str] libp2p_local_uri = self.configuration.config.get("local_uri") # Optional[str] libp2p_public_uri = self.configuration.config.get("public_uri") # Optional[str] libp2p_delegate_uri = self.configuration.config.get( @@ -482,10 +483,8 @@ def __init__(self, **kwargs): and self.crypto_store.crypto_objects.get(ledger_id, None) is not None ): # pragma: no cover key = self.crypto_store.crypto_objects[ledger_id] - elif libp2p_key_file is not None: - key = make_crypto(ledger_id, private_key_path=libp2p_key_file) else: - key = make_crypto(ledger_id) + raise ValueError(f"Couldn't find connection key for {str(ledger_id)} in connections keys") uri = None if libp2p_local_uri is not None: @@ -543,7 +542,14 @@ def __init__(self, **kwargs): "Node's public ip and entry peers ip addresses are not in the same ip address space (private/public)" ) - record = _get_por_from_configuration(key) + cert_requests = self.configuration.cert_requests + if cert_requests is None or len(cert_requests) != 1: + raise ValueError("cert_requests field must be set") + if not Path(cert_requests[0].save_path).is_file(): + raise Exception("cert_request 'save_path' field is not file. Please ensure that 'issue-certificates' command is called beforehand") + + signature, agent_public_key = _get_signature_from_cert_request(cert_requests[0], key.public_key, self.address) + record = AgentRecord(self.address, agent_public_key, key.public_key, signature, POR_DEFAULT_SERVICE_ID) if not record.is_valid_for(self.address, key.public_key): raise ValueError("Invalid Proof-of-Representation {}".format(str(record))) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index 4078a29fc8..b44eb37652 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -16,10 +16,10 @@ fingerprint: aea/pipe_unix.go: QmSzbAexrSUSA9ZE6VT6MmcWF5MhEcmceQjAbnfRoK7Ruv aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK check_dependencies.py: QmcZATMqEThMQBdsud1AwJEYtKq86VfPSXhnfdE7TSWcsc - connection.py: Qmf3epj1Fdo51qGtKUjpesHDiwR1YZE39kZ6VZbUf4dQ7u - dht/dhtclient/dhtclient.go: QmeHnMoBdff59vSqRRsYUGGJPUMi5Euk8pKiKa5jTzHHQD - dht/dhtclient/dhtclient_test.go: QmPfnHSHXtbaW5VYuq1QsKQWey64pUEvLEaKKkT9eAcmws - dht/dhtclient/options.go: QmPorj38wNrxGrzsbFe5wwLmiHzxbTJ2VsgvSd8tLDYS8s + connection.py: QmeC7hDQr2Uo1DmFrS5MEneTvQZSc2bFkYXq4CRGv8WBXu + dht/dhtclient/dhtclient.go: QmZCgMowWw4ueCSsyYY12om3RMQA3Ydp98ti65Eqa5ccuz + dht/dhtclient/dhtclient_test.go: QmbTvSmNU1uqjLQo9LdD7zWmDcLuzEacDeg7AG4VXhmriV + dht/dhtclient/options.go: QmWs7fQqHRavjwe1PjihHFi85HeJu9qSEVyMdcRviZzaT2 dht/dhtnode/dhtnode.go: QmbyhgbCSAbQ1QsDw7FM7Nt5sZcvhbupA1jv5faxutbV7N dht/dhtnode/message.pb.go: QmbMVfoPm74uxHRdAtzhke7ftn8YUQQ7kBs3e7QAjfaEc2 dht/dhtnode/message.proto: QmfFz18nn89kn7ReBwQri7cijbEh6D9TTd3j2euCkKVvDk @@ -49,6 +49,13 @@ config: local_uri: 127.0.0.1:9000 log_file: libp2p_node.log public_uri: 127.0.0.1:9000 +cert_requests: +- identifier: acn + ledger_id: fetchai + not_after: '2022-01-01' + not_before: '2021-01-01' + public_key: fetchai + save_path: /source/conn_cert.txt excluded_protocols: [] restricted_to_protocols: [] dependencies: {} diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.py b/packages/fetchai/connections/p2p_libp2p_client/connection.py index 74a0aa49a7..258ee1ecb1 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.py +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.py @@ -19,12 +19,15 @@ """This module contains the libp2p client connection.""" +from aea.crypto.fetchai import FetchAIHelper +from pathlib import Path +from aea.helpers.base import CertRequest import asyncio import logging import random import struct from asyncio import CancelledError -from typing import List, Optional, Union, cast +from typing import List, Optional, Tuple, Union, cast from aea.configurations.base import PublicId from aea.configurations.constants import DEFAULT_LEDGER @@ -32,7 +35,7 @@ from aea.crypto.base import Crypto from aea.crypto.registries import make_crypto from aea.exceptions import enforce -from aea.helpers.acn.agent_record import AgentRecord +from aea.helpers.acn.agent_record import AgentRecord, recover_verify_keys_from_message from aea.helpers.acn.uri import Uri from aea.mail.base import Envelope @@ -49,17 +52,33 @@ SUPPORTED_LEDGER_IDS = ["fetchai", "cosmos", "ethereum"] -POR_DEFAULT_SERVICE_ID = "" +POR_DEFAULT_SERVICE_ID = "acn" ACN_CURRENT_VERSION = "0.1.0" -def _get_pors_from_connection_config(key: Crypto, count: int) -> List[AgentRecord]: - signature = key.sign_message(key.public_key.encode("utf-8")) - record = AgentRecord( - key.address, key.public_key, key.public_key, signature, POR_DEFAULT_SERVICE_ID - ) - return [record for i in range(count)] +def _get_signatures_from_cert_request( + certs: List[CertRequest], public_keys: List[str], agent_address: str +) -> Tuple[List[str], str]: + signatures: List[str] = [] + verify_key = "" + for i in range(len(certs)): + cert = certs[i] + public_key = public_keys[i] + signature = bytes.fromhex( + Path(cert.save_path).read_bytes().decode("ascii") + ).decode("ascii") + if verify_key == "": + public_keys = recover_verify_keys_from_message( + cert.get_message(public_key), signature + ) + addresses = [ + FetchAIHelper.get_address_from_public_key(public_key) + for public_key in public_keys + ] + verify_key = public_keys[addresses.index(agent_address)] + signatures.append(signature) + return signatures, verify_key class P2PLibp2pClientConnection(Connection): @@ -91,26 +110,38 @@ def __init__(self, **kwargs): raise ValueError("At least one node should be provided") nodes = list(cast(List, nodes)) - nodes_uris = [node["uri"] for node in nodes] + nodes_uris = [node.get("uri", None) for node in nodes] enforce( - len(nodes_uris) == len(nodes), - "Delegate Uri should be provided for each node", + len(nodes_uris) == len(nodes) and None not in nodes_uris, + "Delegate 'uri' should be provided for each node", ) - nodes_certs = [node["cert"] for node in nodes] + nodes_public_keys = [node.get("public_key", None) for node in nodes] enforce( - len(nodes_certs) == len(nodes), - "Delegate Cert should be provided for each node", + len(nodes_public_keys) == len(nodes) and None not in nodes_public_keys, + "Delegate 'public_key' should be provided for each node", ) + cert_requests = self.configuration.cert_requests + if cert_requests is None or len(cert_requests) != len(nodes): + raise ValueError("cert_requests field must be set") + for cert_request in cert_requests: + if not Path(cert_request.save_path).is_file(): + raise Exception( + "cert_request 'save_path' field is not a file. Please ensure that 'issue-certificates' command is called beforehand" + ) + # verify keys are correct - if ( - self.has_crypto_store - and self.crypto_store.crypto_objects.get(ledger_id, None) is not None - ): # pragma: no cover - key = self.crypto_store.crypto_objects[ledger_id] - elif key_file is not None: + # TOFIX(): we cannot use store as the key will be used for TLS tcp connection + # also, as of now all the connections share the same key + # if ( + # self.has_crypto_store + # and self.crypto_store.crypto_objects.get(ledger_id, None) is not None + # ): # pragma: no cover + # key = self.crypto_store.crypto_objects[ledger_id] + # elif key_file is not None: + if key_file is not None: key = make_crypto(ledger_id, private_key_path=key_file) else: key = make_crypto(ledger_id) @@ -123,13 +154,33 @@ def __init__(self, **kwargs): self.delegate_uris = [Uri(node_uri) for node_uri in nodes_uris] # delegates PoRs - self.delegate_pors = _get_pors_from_connection_config(key, len(nodes)) - for i in range(len(self.delegate_pors)): - record = self.delegate_pors[i] + self.delegate_pors = [] + signatures, agent_public_key = _get_signatures_from_cert_request( + cert_requests, nodes_public_keys, self.address + ) + records: List[AgentRecord] = [] + for i in range(len(signatures)): + records.append( + AgentRecord( + self.address, + agent_public_key, + nodes_public_keys[i], + signatures[i], + POR_DEFAULT_SERVICE_ID, + ) + ) + + for i in range(len(nodes_public_keys)): + public_key = nodes_public_keys[i] + uri = self.delegate_uris[i] + record = next( + record for record in records if record.peer_public_key == public_key + ) enforce( - True or record.is_valid_for(self.address, key.public_key), - f"Invalid Proof-of-Representation for node {self.delegate_uris[i]}", + True or record.is_valid_for(self.address, public_key), + f"Invalid Proof-of-Representation for node {uri}", ) + self.delegate_pors.append(record) # select a delegate index = random.randint(0, len(self.delegate_uris) - 1) # nosec diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml index 58cd360076..3fe776abaa 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml @@ -12,7 +12,7 @@ fingerprint: __init__.py: QmT1FEHkPGMHV5oiVEfQHHr25N2qdZxydSNRJabJvYiTgf acn_message.proto: QmfFz18nn89kn7ReBwQri7cijbEh6D9TTd3j2euCkKVvDk acn_message_pb2.py: QmXF5A7PiUiduTWEEkZQZuGRmX7QemyKx8ZDrWnevsi82i - connection.py: QmShEAFNqCKSM3gaYUb5yYHHu6ZBr1hnTB6f11EAnm23EL + connection.py: QmdCbbj2VWtShTQUhoRg7MbhEMPrJTu9YphmqyywKCuSez fingerprint_ignore_patterns: [] connections: [] protocols: [] @@ -21,7 +21,22 @@ config: ledger_id: fetchai nodes: - uri: acn.fetch.ai:11000 + public_key: 0217a59bd805c310aca4febe0e99ce22ee3712ae085dc1e5630430b1e15a584bb7 - uri: acn.fetch.ai:11001 + public_key: 03fa7cfae1037cba5218f0f5743802eced8de3247c55ecebaae46c7d3679e3f91d +cert_requests: +- identifier: acn + ledger_id: fetchai + not_after: '2022-01-01' + not_before: '2021-01-01' + public_key: 0217a59bd805c310aca4febe0e99ce22ee3712ae085dc1e5630430b1e15a584bb7 + save_path: ./acn_fetch_ai_11000.txt +- identifier: acn + ledger_id: fetchai + not_after: '2022-01-01' + not_before: '2021-01-01' + public_key: 03fa7cfae1037cba5218f0f5743802eced8de3247c55ecebaae46c7d3679e3f91d + save_path: ./acn_fetch_ai_11001.txt excluded_protocols: [] restricted_to_protocols: [] dependencies: {} diff --git a/packages/hashes.csv b/packages/hashes.csv index cb7c64adcf..420df3265d 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,13 +34,8 @@ fetchai/connections/http_server,QmQ8hNfEts6th9zfnocYHzyYw4dXkDNDj8zoKtov1wB49n fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 -<<<<<<< HEAD -fetchai/connections/p2p_libp2p,QmXqc4YrAYpqij1XEJcZ3ipFMmTnqtUVzgQoE9HAmFjf7h -fetchai/connections/p2p_libp2p_client,QmcHPi4VPha5WwydNkVVU4yGSvQvxtoStYZYEmKhw61fXp -======= -fetchai/connections/p2p_libp2p,QmSDodHZsSwTvfCvnSHV9D82Uo1fcHJVqy9yBfqkHzewAr -fetchai/connections/p2p_libp2p_client,QmQndwAPhGHTpe1J8c6Q6jabV2KfkND7jsYiEF9Ysmyby3 ->>>>>>> develop +fetchai/connections/p2p_libp2p,QmV6z1VJcPSvQ9g985eQQgG36JadN3YtjikpLfXFym8S6y +fetchai/connections/p2p_libp2p_client,QmPQTBEgX5C7znwwBAmtk35frw1Myp6nPQGfzHogoWG2DY fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T fetchai/connections/scaffold,QmaCupER4vedakPPLzn2Rw9YHPmqSPaZh5xnqi6xVxQHvn diff --git a/tests/conftest.py b/tests/conftest.py index 9643d48bb2..29e4e92282 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,6 +17,7 @@ # # ------------------------------------------------------------------------------ """Conftest module for Pytest.""" +from aea.helpers.base import CertRequest import difflib import inspect import logging @@ -79,6 +80,7 @@ LIBP2P_NODE_MODULE_NAME, MultiAddr, P2PLibp2pConnection, + POR_DEFAULT_SERVICE_ID, ) from packages.fetchai.connections.p2p_libp2p_client.connection import ( P2PLibp2pClientConnection, @@ -208,6 +210,8 @@ PUBLIC_DHT_P2P_MADDR_2 = "/dns4/acn.fetch.ai/tcp/9001/p2p/16Uiu2HAmVWnopQAqq4pniYLw44VRvYxBUoRHqjz1Hh2SoCyjbyRW" PUBLIC_DHT_DELEGATE_URI_1 = "acn.fetch.ai:11000" PUBLIC_DHT_DELEGATE_URI_2 = "acn.fetch.ai:11001" +PUBLIC_DHT_P2P_PUBLIC_KEY_1 = "0217a59bd805c310aca4febe0e99ce22ee3712ae085dc1e5630430b1e15a584bb7" +PUBLIC_DHT_P2P_PUBLIC_KEY_2 = "03fa7cfae1037cba5218f0f5743802eced8de3247c55ecebaae46c7d3679e3f91d" # testnets COSMOS_TESTNET_CONFIG = {"address": COSMOS_DEFAULT_ADDRESS} @@ -774,8 +778,13 @@ def _make_stub_connection(input_file_path: str, output_file_path: str): connection = StubConnection(configuration=configuration) return connection +def _process_cert(key: Crypto, cert: CertRequest): + # must match aea/cli/issue_certificates.py:_process_certificate + message = cert.get_message(cert.public_key) + signature = key.sign_message(message).encode("ascii").hex() + Path(cert.save_path).write_bytes(signature.encode("ascii")) -def _make_libp2p_connection_to_restore( +def _make_libp2p_connection( port: int = 10234, host: str = "127.0.0.1", relay: bool = True, @@ -784,16 +793,35 @@ def _make_libp2p_connection_to_restore( delegate_port: int = 11234, delegate_host: str = "127.0.0.1", node_key_file: Optional[str] = None, - agent_address: Optional[Address] = None, + agent_key: Optional[Crypto] = None, build_directory: Optional[str] = None, + peer_registration_delay: str = "0.0", ) -> P2PLibp2pConnection: log_file = "libp2p_node_{}.log".format(port) if os.path.exists(log_file): os.remove(log_file) - address = agent_address - if address is None: - address = make_crypto(COSMOS).address - identity = Identity("", address=address) + key = agent_key + if key is None: + key = make_crypto(DEFAULT_LEDGER) + identity = Identity("", address=key.address) + conn_crypto_store = None + if node_key_file is not None: + conn_crypto_store = CryptoStore({DEFAULT_LEDGER: node_key_file}) + else: + node_key = make_crypto(DEFAULT_LEDGER) + node_key_path = f"./{node_key.public_key}.txt" + with open(node_key_path, "wb") as f: + node_key.dump(f) + conn_crypto_store = CryptoStore({DEFAULT_LEDGER: node_key_path}) + cert_request = CertRequest( + conn_crypto_store.public_keys[DEFAULT_LEDGER], + POR_DEFAULT_SERVICE_ID, + DEFAULT_LEDGER, + "2021-01-01", + "2021-01-02", + f"./{key.address}_cert.txt", + ) + _process_cert(key, cert_request) if not build_directory: build_directory = os.getcwd() if relay and delegate: @@ -804,8 +832,10 @@ def _make_libp2p_connection_to_restore( entry_peers=entry_peers, log_file=log_file, delegate_uri="{}:{}".format(delegate_host, delegate_port), + peer_registration_delay=peer_registration_delay, connection_id=P2PLibp2pConnection.connection_id, build_directory=build_directory, + cert_requests=[cert_request], ) elif relay and not delegate: configuration = ConnectionConfig( @@ -814,8 +844,10 @@ def _make_libp2p_connection_to_restore( public_uri="{}:{}".format(host, port), entry_peers=entry_peers, log_file=log_file, + peer_registration_delay=peer_registration_delay, connection_id=P2PLibp2pConnection.connection_id, build_directory=build_directory, + cert_requests=[cert_request], ) else: configuration = ConnectionConfig( @@ -823,16 +855,20 @@ def _make_libp2p_connection_to_restore( local_uri="{}:{}".format(host, port), entry_peers=entry_peers, log_file=log_file, + peer_registration_delay=peer_registration_delay, connection_id=P2PLibp2pConnection.connection_id, build_directory=build_directory, + cert_requests=[cert_request], ) if not os.path.exists(os.path.join(build_directory, LIBP2P_NODE_MODULE_NAME)): build_node(build_directory) - connection = P2PLibp2pConnection(configuration=configuration, identity=identity) + connection = P2PLibp2pConnection( + configuration=configuration, identity=identity, crypto_store=conn_crypto_store + ) return connection -def _make_libp2p_connection( +def _make_libp2p_connection_delete_me( port: int = 10234, host: str = "127.0.0.1", relay: bool = True, @@ -900,26 +936,37 @@ def _make_libp2p_connection( return P2PLibp2pConnection(configuration=configuration, identity=identity) -def _make_libp2p_client_connection_to_restore( - node_port: int = 11234, node_host: str = "127.0.0.1", uri: Optional[str] = None, +def _make_libp2p_client_connection( + peer_public_key: str, node_port: int = 11234, node_host: str = "127.0.0.1", uri: Optional[str] = None, ) -> P2PLibp2pClientConnection: - crypto = make_crypto(COSMOS) + crypto = make_crypto(DEFAULT_LEDGER) identity = Identity("", address=crypto.address) + cert_request = CertRequest( + peer_public_key, + POR_DEFAULT_SERVICE_ID, + DEFAULT_LEDGER, + "2021-01-01", + "2021-01-02", + f"./{crypto.address}_cert.txt", + ) + _process_cert(crypto, cert_request) configuration = ConnectionConfig( client_key_file=None, nodes=[ { "uri": str(uri) if uri is not None - else "{}:{}".format(node_host, node_port) + else "{}:{}".format(node_host, node_port), + "public_key": peer_public_key, }, ], connection_id=P2PLibp2pClientConnection.connection_id, + cert_requests=[cert_request], ) return P2PLibp2pClientConnection(configuration=configuration, identity=identity) -def _make_libp2p_client_connection( +def _make_libp2p_client_connection_delete_me( node_port: int = 11234, node_host: str = "127.0.0.1", uri: Optional[str] = None, diff --git a/tests/data/dummy_connection/connection.yaml b/tests/data/dummy_connection/connection.yaml index 5dca25e586..9d10abd0a6 100644 --- a/tests/data/dummy_connection/connection.yaml +++ b/tests/data/dummy_connection/connection.yaml @@ -33,10 +33,10 @@ cert_requests: not_after: '2020-01-02' not_before: '2020-01-01' public_key: key_id - save_path: some/path_1 + save_path: /source/some/path_1 - identifier: cert_id_2 ledger_id: some_ledger_id not_after: '2020-01-02' not_before: '2020-01-01' public_key: '0xABCDEF123456' - save_path: some/path_2 + save_path: /source/some/path_2 diff --git a/tests/data/hashes.csv b/tests/data/hashes.csv index 0cdb6108e6..029c823b59 100644 --- a/tests/data/hashes.csv +++ b/tests/data/hashes.csv @@ -1,6 +1,6 @@ dummy_author/agents/dummy_aea,QmPWxJVM9xnkRFtB41Wr1Xi2NGKpeRgDXUzbAzA8rferGh dummy_author/skills/dummy_skill,QmSmzcjGNhiz2KpSE6wSi19AeesCVjzupJm1NpXYJVnJuZ -fetchai/connections/dummy_connection,QmYn34pVcTpKazRLUpgafNW917ndLZidq9nDFFiRW7YzbT +fetchai/connections/dummy_connection,Qmep3ive7fVWJi965SSdJv6fdfzAKD1wmP9A1kYRoQjRNn fetchai/contracts/dummy_contract,QmXhMKRF5upco1efaC9oNcesStEMkPWKhBUv6CK9Vngk2S fetchai/protocols/t_protocol,QmNhzDCznorQpySrvKt4F4sQFXjzF1ZecEd5inixiNavDH fetchai/protocols/t_protocol_no_ct,QmWwMwHxvuPZJaMh44Bd1d5aBzYiogep8TJswTKkRwLfBu diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py index 7a81394c21..02e86113cf 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py @@ -19,7 +19,10 @@ """This test module contains AEA cli tests for P2PLibp2p connection.""" +from aea.configurations.constants import DEFAULT_LEDGER +from aea.crypto.registries import make_crypto import os +from pathlib import Path from aea.test_tools.test_cases import AEATestCaseEmpty @@ -34,7 +37,7 @@ DEFAULT_DELEGATE_PORT = 11234 DEFAULT_NET_SIZE = 4 -LIBP2P_LAUNCH_TIMEOUT = 660 # may downloads up to ~66Mb +LIBP2P_LAUNCH_TIMEOUT = 110 # may downloads up to ~66Mb class TestP2PLibp2pConnectionAEARunningDefaultConfigNode(AEATestCaseEmpty): @@ -44,6 +47,10 @@ class TestP2PLibp2pConnectionAEARunningDefaultConfigNode(AEATestCaseEmpty): def setup_class(cls): """Set the test up""" super(TestP2PLibp2pConnectionAEARunningDefaultConfigNode, cls).setup_class() + cls.conn_key_file = os.path.join(os.path.abspath(os.getcwd()), "./conn_key.txt") + with open(cls.conn_key_file, "wb") as f: + key = make_crypto(DEFAULT_LEDGER) + key.dump(f) cls.log_files = [] @libp2p_log_on_failure @@ -52,6 +59,7 @@ def test_agent(self): self.add_item("connection", str(P2P_CONNECTION_PUBLIC_ID)) self.run_cli_command("build", cwd=self._get_cwd()) self.set_config("agent.default_connection", str(P2P_CONNECTION_PUBLIC_ID)) + self.nested_set_config("agent.connection_private_key_paths", {DEFAULT_LEDGER:self.conn_key_file}) # for logging config_path = "vendor.fetchai.connections.p2p_libp2p.config" @@ -60,6 +68,9 @@ def test_agent(self): self.set_config("{}.log_file".format(config_path), log_file) TestP2PLibp2pConnectionAEARunningDefaultConfigNode.log_files.append(log_file) + # generate certificates for connection + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) + process = self.run_agent() is_running = self.is_running(process, timeout=LIBP2P_LAUNCH_TIMEOUT) assert is_running, "AEA not running within timeout!" @@ -90,6 +101,10 @@ class TestP2PLibp2pConnectionAEARunningFullNode(AEATestCaseEmpty): def setup_class(cls): """Set the test up""" super(TestP2PLibp2pConnectionAEARunningFullNode, cls).setup_class() + cls.conn_key_file = os.path.join(os.path.abspath(os.getcwd()), "./conn_key.txt") + with open(cls.conn_key_file, "wb") as f: + key = make_crypto(DEFAULT_LEDGER) + key.dump(f) cls.log_files = [] @libp2p_log_on_failure @@ -97,6 +112,7 @@ def test_agent(self): """Test with aea.""" self.add_item("connection", str(P2P_CONNECTION_PUBLIC_ID)) self.run_cli_command("build", cwd=self._get_cwd()) + self.nested_set_config("agent.connection_private_key_paths", {DEFAULT_LEDGER:self.conn_key_file}) # setup a full node: with public uri, relay service, and delegate service config_path = "vendor.fetchai.connections.p2p_libp2p.config" @@ -117,6 +133,9 @@ def test_agent(self): self.set_config("{}.log_file".format(config_path), log_file) TestP2PLibp2pConnectionAEARunningFullNode.log_files.append(log_file) + # generate certificates for connection + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) + process = self.run_agent() is_running = self.is_running(process, timeout=LIBP2P_LAUNCH_TIMEOUT) diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py index 460501d0c9..d54d5bd18a 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py @@ -17,6 +17,7 @@ # # ------------------------------------------------------------------------------ """This test module contains Negative tests for Libp2p connection.""" +from aea.crypto.wallet import CryptoStore import os import shutil import tempfile @@ -182,27 +183,10 @@ def test_libp2pconnection_mixed_ip_address(): @patch.object(P2PLibp2pConnection, "_check_node_built") def test_libp2pconnection_node_config_registration_delay(mock): - """Test nod registration delay configuration""" - crypto = make_crypto(DEFAULT_LEDGER) - identity = Identity("", address=crypto.address) + """Test node registration delay configuration""" host = "localhost" port = "10000" - - configuration = ConnectionConfig( - local_uri="{}:{}".format(host, port), - public_uri="{}:{}".format(host, port), - peer_registration_delay="1.5", - connection_id=P2PLibp2pConnection.connection_id, - build_directory="some", - ) - P2PLibp2pConnection(configuration=configuration, identity=identity) - - configuration = ConnectionConfig( - local_uri="{}:{}".format(host, port), - public_uri="{}:{}".format(host, port), - peer_registration_delay="must_be_float", - connection_id=P2PLibp2pConnection.connection_id, - build_directory="some", - ) + + _make_libp2p_connection(port, host) with pytest.raises(ValueError): - P2PLibp2pConnection(configuration=configuration, identity=identity) + _make_libp2p_connection(port, host, peer_registration_delay="must_be_float") diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_fault_tolerance.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_fault_tolerance.py index a9eb6833b5..614d78f8a8 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_fault_tolerance.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_fault_tolerance.py @@ -216,15 +216,15 @@ def setup_class(cls): cls.multiplexer1.connect() cls.multiplexers.append(cls.multiplexer1) + cls.connection_key = make_crypto(DEFAULT_LEDGER) cls.connection2 = _make_libp2p_connection( - DEFAULT_PORT + 2, entry_peers=[genesis_peer] + DEFAULT_PORT + 2, entry_peers=[genesis_peer], agent_key=cls.connection_key, ) cls.multiplexer2 = Multiplexer([cls.connection2]) cls.log_files.append(cls.connection2.node.log_file) cls.multiplexer2.connect() cls.multiplexers.append(cls.multiplexer2) - cls.connection_addr = cls.connection2.address except Exception as e: cls.teardown_class() raise e @@ -267,7 +267,7 @@ def test_envelope_routed_after_peer_changed(self): TestLibp2pConnectionAgentMobility.connection2 = _make_libp2p_connection( port=DEFAULT_PORT + 2, entry_peers=[self.genesis.node.multiaddrs[0]], - agent_address=self.connection_addr, + agent_key=self.connection_key, ) TestLibp2pConnectionAgentMobility.multiplexer2 = Multiplexer([self.connection2]) self.multiplexer2.connect() diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py index c3eebb15f8..8b46993423 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py @@ -42,6 +42,8 @@ PUBLIC_DHT_DELEGATE_URI_2, PUBLIC_DHT_P2P_MADDR_1, PUBLIC_DHT_P2P_MADDR_2, + PUBLIC_DHT_P2P_PUBLIC_KEY_1, + PUBLIC_DHT_P2P_PUBLIC_KEY_2, _make_libp2p_client_connection, _make_libp2p_connection, libp2p_log_on_failure, @@ -52,6 +54,7 @@ DEFAULT_PORT = 10234 PUBLIC_DHT_MADDRS = [PUBLIC_DHT_P2P_MADDR_1, PUBLIC_DHT_P2P_MADDR_2] PUBLIC_DHT_DELEGATE_URIS = [PUBLIC_DHT_DELEGATE_URI_1, PUBLIC_DHT_DELEGATE_URI_2] +PUBLIC_DHT_PUBLIC_KEYS = [PUBLIC_DHT_P2P_PUBLIC_KEY_1, PUBLIC_DHT_P2P_PUBLIC_KEY_2] AEA_DEFAULT_LAUNCH_TIMEOUT = 15 AEA_LIBP2P_LAUNCH_TIMEOUT = 660 # may download up to ~66Mb @@ -227,8 +230,10 @@ class TestLibp2pConnectionPublicDHTDelegate: def test_connectivity(self): """Test connectivity.""" - for uri in PUBLIC_DHT_DELEGATE_URIS: - connection = _make_libp2p_client_connection(uri=uri) + for i in range(len(PUBLIC_DHT_DELEGATE_URIS)): + uri = PUBLIC_DHT_DELEGATE_URIS[i] + peer_public_key = PUBLIC_DHT_PUBLIC_KEYS[i] + connection = _make_libp2p_client_connection(peer_public_key, uri=uri) multiplexer = Multiplexer([connection]) try: @@ -243,15 +248,17 @@ def test_connectivity(self): def test_communication_direct(self): """Test communication direct.""" - for uri in PUBLIC_DHT_DELEGATE_URIS: + for i in range(len(PUBLIC_DHT_DELEGATE_URIS)): + uri = PUBLIC_DHT_DELEGATE_URIS[i] + peer_public_key = PUBLIC_DHT_PUBLIC_KEYS[i] multiplexers = [] try: - connection1 = _make_libp2p_client_connection(uri=uri) + connection1 = _make_libp2p_client_connection(peer_public_key, uri=uri) multiplexer1 = Multiplexer([connection1]) multiplexer1.connect() multiplexers.append(multiplexer1) - connection2 = _make_libp2p_client_connection(uri=uri) + connection2 = _make_libp2p_client_connection(peer_public_key, uri=uri) multiplexer2 = Multiplexer([connection2]) multiplexer2.connect() multiplexers.append(multiplexer2) @@ -301,6 +308,7 @@ def test_communication_indirect(self): multiplexers = [] try: connection1 = _make_libp2p_client_connection( + PUBLIC_DHT_PUBLIC_KEYS[i], uri=PUBLIC_DHT_DELEGATE_URIS[i] ) multiplexer1 = Multiplexer([connection1]) @@ -314,6 +322,7 @@ def test_communication_indirect(self): continue connection2 = _make_libp2p_client_connection( + PUBLIC_DHT_PUBLIC_KEYS[i], uri=PUBLIC_DHT_DELEGATE_URIS[j] ) multiplexer2 = Multiplexer([connection2]) diff --git a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py index bf82663690..78a416ce79 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py @@ -65,12 +65,28 @@ def test_node(self): def test_connection(self): """Test the connection can be used in an aea.""" self.add_item("connection", str(PUBLIC_ID)) - config_path = "vendor.fetchai.connections.p2p_libp2p_client.config" + conn_path = "vendor.fetchai.connections.p2p_libp2p_client" self.nested_set_config( - config_path, - {"nodes": [{"uri": "{}:{}".format(DEFAULT_HOST, DEFAULT_DELEGATE_PORT)}]}, + conn_path+".config", + {"nodes": [{ + "uri": "{}:{}".format(DEFAULT_HOST, DEFAULT_DELEGATE_PORT), + "public_key" : self.node_connection.node.pub, + }]}, ) + # generate certificates for connection + self.nested_set_config( + conn_path+".cert_requests", + [{ + "identifier" : "acn", + "not_after": "2022-01-01", + "not_before": "2021-01-01", + "public_key": self.node_connection.node.pub, + "save_path": "./cli_test_cert.txt", + }], + ) + #self.run_cli_command("issue-certificates", cwd=self._get_cwd()) + process = self.run_agent() is_running = self.is_running(process, timeout=DEFAULT_LAUNCH_TIMEOUT) assert is_running, "AEA not running within timeout!" diff --git a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_communication.py b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_communication.py index 516b5d6f6d..14db4c1b00 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_communication.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_communication.py @@ -58,7 +58,7 @@ def setup_class(cls): os.chdir(cls.t) cls.connection_node = _make_libp2p_connection(delegate=True) - cls.connection = _make_libp2p_client_connection() + cls.connection = _make_libp2p_client_connection(cls.connection_node.node.pub) @pytest.mark.asyncio async def test_libp2pclientconnection_connect_disconnect(self): @@ -106,11 +106,11 @@ def setup_class(cls): cls.multiplexer_node.connect() try: - cls.connection_client_1 = _make_libp2p_client_connection() + cls.connection_client_1 = _make_libp2p_client_connection(cls.connection_node.node.pub) cls.multiplexer_client_1 = Multiplexer([cls.connection_client_1]) cls.multiplexer_client_1.connect() - cls.connection_client_2 = _make_libp2p_client_connection() + cls.connection_client_2 = _make_libp2p_client_connection(cls.connection_node.node.pub) cls.multiplexer_client_2 = Multiplexer([cls.connection_client_2]) cls.multiplexer_client_2.connect() except Exception: @@ -274,6 +274,7 @@ def setup_class(cls): cls.mutliplexers.append(cls.multiplexer_node_2) cls.connection_client_1 = _make_libp2p_client_connection( + cls.connection_node_1.node.pub, DEFAULT_DELEGATE_PORT + 1 ) cls.multiplexer_client_1 = Multiplexer([cls.connection_client_1]) @@ -281,7 +282,8 @@ def setup_class(cls): cls.mutliplexers.append(cls.multiplexer_client_1) cls.connection_client_2 = _make_libp2p_client_connection( - DEFAULT_DELEGATE_PORT + 2 + cls.connection_node_2.node.pub, + DEFAULT_DELEGATE_PORT + 2 ) cls.multiplexer_client_2 = Multiplexer([cls.connection_client_2]) cls.multiplexer_client_2.connect() @@ -453,8 +455,12 @@ def setup_class(cls): ] for _ in range(DEFAULT_CLIENTS_PER_NODE): - for port in [DEFAULT_DELEGATE_PORT + 1, DEFAULT_DELEGATE_PORT + 2]: - conn = _make_libp2p_client_connection(port) + ports = [DEFAULT_DELEGATE_PORT + 1, DEFAULT_DELEGATE_PORT + 2] + peers_public_keys = [cls.connection_node_1.node.pub, cls.connection_node_2.node.pub] + for i in range(len(ports)): + port = ports[i] + peer_public_key = peers_public_keys[i] + conn = _make_libp2p_client_connection(peer_public_key, port) mux = Multiplexer([conn]) cls.connections.append(conn) diff --git a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_errors.py b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_errors.py index 4500e3c166..484f658d08 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_errors.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_errors.py @@ -19,6 +19,8 @@ """This test module contains negative tests for Libp2p tcp client connection.""" +from aea.helpers.base import CertRequest +from aea.configurations.constants import DEFAULT_LEDGER import os import shutil import tempfile @@ -31,13 +33,13 @@ from aea.multiplexer import Multiplexer from packages.fetchai.connections.p2p_libp2p_client.connection import ( - P2PLibp2pClientConnection, + P2PLibp2pClientConnection, POR_DEFAULT_SERVICE_ID, ) from tests.conftest import ( COSMOS, _make_libp2p_client_connection, - _make_libp2p_connection, + _make_libp2p_connection, _process_cert, libp2p_log_on_failure, ) @@ -49,7 +51,7 @@ class TestLibp2pClientConnectionFailureNodeNotConnected: @pytest.mark.asyncio async def test_node_not_running(self): """Test the node is not running.""" - conn = _make_libp2p_client_connection() + conn = _make_libp2p_client_connection(make_crypto(DEFAULT_LEDGER).public_key) with pytest.raises(Exception): await conn.connect() @@ -63,7 +65,7 @@ def setup_class(cls): cls.cwd = os.getcwd() cls.t = tempfile.mkdtemp() os.chdir(cls.t) - crypto = make_crypto(COSMOS) + crypto = make_crypto(DEFAULT_LEDGER) cls.node_host = "localhost" cls.node_port = "11234" cls.identity = Identity("", address=crypto.address) @@ -73,12 +75,27 @@ def setup_class(cls): crypto.dump(key_file_desc) key_file_desc.close() + cls.peer_crypto = make_crypto(DEFAULT_LEDGER) + cls.cert_request = CertRequest( + cls.peer_crypto.public_key, + POR_DEFAULT_SERVICE_ID, + DEFAULT_LEDGER, + "2021-01-01", + "2021-01-02", + f"./{crypto.address}_cert.txt", + ) + _process_cert(crypto, cls.cert_request) + def test_empty_nodes(self): """Test empty nodes.""" configuration = ConnectionConfig( client_key_file=self.key_file, - nodes=[{"uri": "{}:{}".format(self.node_host, self.node_port)}], + nodes=[{ + "uri": "{}:{}".format(self.node_host, self.node_port), + "public_key": self.peer_crypto.public_key, + }], connection_id=P2PLibp2pClientConnection.connection_id, + cert_requests=[self.cert_request], ) P2PLibp2pClientConnection(configuration=configuration, identity=self.identity) @@ -123,7 +140,7 @@ def setup_class(cls): cls.multiplexer_node.connect() cls.multiplexers.append(cls.multiplexer_node) - cls.connection_client = _make_libp2p_client_connection() + cls.connection_client = _make_libp2p_client_connection(cls.connection_node.node.pub) cls.multiplexer_client = Multiplexer([cls.connection_client]) cls.multiplexer_client.connect() cls.multiplexers.append(cls.multiplexer_client) From e6a0751fad23a1156e234bb5127e925155610898 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Wed, 30 Dec 2020 18:21:02 +0100 Subject: [PATCH 093/204] fix security check --- tests/test_docs/test_generic_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_docs/test_generic_storage.py b/tests/test_docs/test_generic_storage.py index 3cdfb36f91..7c4a4c4ff2 100644 --- a/tests/test_docs/test_generic_storage.py +++ b/tests/test_docs/test_generic_storage.py @@ -61,4 +61,4 @@ def test_test_behaviour(self): """Test that the 'TestBehaviour' code is compilable.""" block_index = 1 code = self.code_blocks[block_index]["text"] - exec(code, {}, dict(TickerBehaviour=TickerBehaviour)) + exec(code, {}, dict(TickerBehaviour=TickerBehaviour)) # nosec From f79034599a7f28326c215e1b38ca440c308b81b4 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Wed, 30 Dec 2020 17:24:52 +0000 Subject: [PATCH 094/204] Fix p2p_libp2p_client cli test - All libp2p tests pass --- aea/helpers/acn/agent_record.py | 19 ------------------ .../p2p_libp2p_client/connection.py | 6 +++--- .../p2p_libp2p_client/connection.yaml | 6 +++--- packages/hashes.csv | 2 +- .../test_p2p_libp2p_client/test_aea_cli.py | 20 +++++++++++-------- 5 files changed, 19 insertions(+), 34 deletions(-) diff --git a/aea/helpers/acn/agent_record.py b/aea/helpers/acn/agent_record.py index 06455610b0..ea4bef6d35 100644 --- a/aea/helpers/acn/agent_record.py +++ b/aea/helpers/acn/agent_record.py @@ -24,7 +24,6 @@ from typing import List from ecdsa import SECP256k1, VerifyingKey -from ecdsa import util as ecdsa_util from aea.crypto.fetchai import FetchAIHelper @@ -40,24 +39,6 @@ def recover_verify_keys_from_message(message: bytes, signature: str) -> List[str ] return public_keys -def verify_message(msg: bytes, signature: str, public_key: str) -> bool: - """ - Verifies signature generate by Crypto.sign_message() - - :param msg: original bytes - :param signature: signature previousely generated by sign_message() - :param public_key: verifying key - :return: True if signature is valid - """ - key = VerifyingKey.from_string( - bytes.fromhex(public_key), curve=SECP256k1, hashfunc=sha256 - ) - signature_bytes = base64.b64decode(signature) - return key.verify( - signature_bytes, msg, hashfunc=sha256, sigdecode=ecdsa_util.sigdecode_string - ) - - class AgentRecord: """Agent Proof-of-Representation to peer""" diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.py b/packages/fetchai/connections/p2p_libp2p_client/connection.py index 258ee1ecb1..4549b26312 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.py +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.py @@ -39,9 +39,9 @@ from aea.helpers.acn.uri import Uri from aea.mail.base import Envelope -from .acn_message_pb2 import AcnMessage -from .acn_message_pb2 import AgentRecord as AgentRecordPb -from .acn_message_pb2 import Register, Status +from packages.fetchai.connections.p2p_libp2p_client.acn_message_pb2 import AcnMessage +from packages.fetchai.connections.p2p_libp2p_client.acn_message_pb2 import AgentRecord as AgentRecordPb +from packages.fetchai.connections.p2p_libp2p_client.acn_message_pb2 import Register, Status _default_logger = logging.getLogger( diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml index 3fe776abaa..e8b902826a 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml @@ -12,7 +12,7 @@ fingerprint: __init__.py: QmT1FEHkPGMHV5oiVEfQHHr25N2qdZxydSNRJabJvYiTgf acn_message.proto: QmfFz18nn89kn7ReBwQri7cijbEh6D9TTd3j2euCkKVvDk acn_message_pb2.py: QmXF5A7PiUiduTWEEkZQZuGRmX7QemyKx8ZDrWnevsi82i - connection.py: QmdCbbj2VWtShTQUhoRg7MbhEMPrJTu9YphmqyywKCuSez + connection.py: QmV6D74dzGbAYcsAoso7k2JBbFM76oYYNhrk8vGjacfEg9 fingerprint_ignore_patterns: [] connections: [] protocols: [] @@ -30,13 +30,13 @@ cert_requests: not_after: '2022-01-01' not_before: '2021-01-01' public_key: 0217a59bd805c310aca4febe0e99ce22ee3712ae085dc1e5630430b1e15a584bb7 - save_path: ./acn_fetch_ai_11000.txt + save_path: /source/acn_fetch_ai_11000.txt - identifier: acn ledger_id: fetchai not_after: '2022-01-01' not_before: '2021-01-01' public_key: 03fa7cfae1037cba5218f0f5743802eced8de3247c55ecebaae46c7d3679e3f91d - save_path: ./acn_fetch_ai_11001.txt + save_path: /source/acn_fetch_ai_11001.txt excluded_protocols: [] restricted_to_protocols: [] dependencies: {} diff --git a/packages/hashes.csv b/packages/hashes.csv index 420df3265d..d0fb9c08e0 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -35,7 +35,7 @@ fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 fetchai/connections/p2p_libp2p,QmV6z1VJcPSvQ9g985eQQgG36JadN3YtjikpLfXFym8S6y -fetchai/connections/p2p_libp2p_client,QmPQTBEgX5C7znwwBAmtk35frw1Myp6nPQGfzHogoWG2DY +fetchai/connections/p2p_libp2p_client,QmcZfP63pCXH8H5zxgF1L9acfDkZU8hunCikKniwW4FJhR fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T fetchai/connections/scaffold,QmaCupER4vedakPPLzn2Rw9YHPmqSPaZh5xnqi6xVxQHvn diff --git a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py index 78a416ce79..61fbc91d90 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py @@ -18,6 +18,7 @@ # ------------------------------------------------------------------------------ """This test module contains AEA cli tests for Libp2p tcp client connection.""" +from aea.helpers.base import CertRequest from aea.multiplexer import Multiplexer from aea.test_tools.test_cases import AEATestCaseEmpty @@ -64,6 +65,8 @@ def test_node(self): def test_connection(self): """Test the connection can be used in an aea.""" + self.generate_private_key() + self.add_private_key() self.add_item("connection", str(PUBLIC_ID)) conn_path = "vendor.fetchai.connections.p2p_libp2p_client" self.nested_set_config( @@ -77,15 +80,16 @@ def test_connection(self): # generate certificates for connection self.nested_set_config( conn_path+".cert_requests", - [{ - "identifier" : "acn", - "not_after": "2022-01-01", - "not_before": "2021-01-01", - "public_key": self.node_connection.node.pub, - "save_path": "./cli_test_cert.txt", - }], + [CertRequest( + identifier="acn", + ledger_id="fetchai", + not_after="2022-01-01", + not_before="2021-01-01", + public_key=self.node_connection.node.pub, + save_path="./cli_test_cert.txt")], ) - #self.run_cli_command("issue-certificates", cwd=self._get_cwd()) + + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) process = self.run_agent() is_running = self.is_running(process, timeout=DEFAULT_LAUNCH_TIMEOUT) From 9f1ca4296e2da9084e605f0fb874bc0d9cacccfe Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Wed, 30 Dec 2020 20:31:37 +0000 Subject: [PATCH 095/204] Address ci comments --- aea/helpers/acn/agent_record.py | 19 ++- aea/helpers/base.py | 6 +- aea/test_tools/test_cases.py | 2 +- .../connections/p2p_libp2p/connection.py | 42 ++++-- .../p2p_libp2p/dht/dhtclient/options.go | 8 - .../p2p_libp2p/dht/dhtnode/message.pb.go | 140 ++++++++++-------- .../p2p_libp2p/dht/dhtnode/message.proto | 2 + .../p2p_libp2p_client/acn_message.proto | 2 + .../p2p_libp2p_client/acn_message_pb2.py | 68 ++++++--- .../p2p_libp2p_client/connection.py | 15 +- tests/conftest.py | 115 ++------------ .../test_p2p_libp2p/test_aea_cli.py | 12 +- .../test_p2p_libp2p/test_errors.py | 4 +- .../test_p2p_libp2p/test_fault_tolerance.py | 4 +- .../test_p2p_libp2p/test_public_dht.py | 6 +- .../test_p2p_libp2p_client/test_aea_cli.py | 35 +++-- .../test_communication.py | 19 ++- .../test_p2p_libp2p_client/test_errors.py | 24 +-- 18 files changed, 271 insertions(+), 252 deletions(-) diff --git a/aea/helpers/acn/agent_record.py b/aea/helpers/acn/agent_record.py index ea4bef6d35..5c66e21a78 100644 --- a/aea/helpers/acn/agent_record.py +++ b/aea/helpers/acn/agent_record.py @@ -27,6 +27,7 @@ from aea.crypto.fetchai import FetchAIHelper + def recover_verify_keys_from_message(message: bytes, signature: str) -> List[str]: """ get the public key used to produce the `signature` of the `message`""" signature_b64 = base64.b64decode(signature) @@ -34,11 +35,11 @@ def recover_verify_keys_from_message(message: bytes, signature: str) -> List[str signature_b64, message, SECP256k1, hashfunc=sha256, ) public_keys = [ - verifying_key.to_string("compressed").hex() - for verifying_key in verifying_keys + verifying_key.to_string("compressed").hex() for verifying_key in verifying_keys ] return public_keys + class AgentRecord: """Agent Proof-of-Representation to peer""" @@ -50,7 +51,15 @@ def __init__( signature: str, service_id: str, ): - """Initialize the AgentRecord""" + """ + Initialize the AgentRecord + + :param address: agent address + :param public key: agent public key (associated to the address) + :param peer_public_key: representative peer public key + :param signature: proof-of-representation of this AgentRecord + :param service_id: type of service for which the record is used + """ self._service_id = service_id self._address = address self._public_key = public_key @@ -102,7 +111,9 @@ def is_valid_for(self, address: str, peer_public_key: str) -> bool: print("Wrong peer public key") return False if self._address != FetchAIHelper.get_address_from_public_key(self._public_key): - print(f"Wrong address '{self._address}' and public key '{FetchAIHelper.get_address_from_public_key(self._public_key)}'") + print( + f"Wrong address '{self._address}' and public key '{FetchAIHelper.get_address_from_public_key(self._public_key)}'" + ) return False if self._address not in FetchAIHelper.recover_message( self._peer_public_key.encode("utf-8"), self._signature diff --git a/aea/helpers/base.py b/aea/helpers/base.py index 7de7fe0bd4..339d0d02c3 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -750,9 +750,9 @@ def get_message(self, public_key: str) -> bytes: """Get the message to sign.""" message = ( public_key.encode("ascii") - #+ self.identifier.encode("ascii") - #+ self.not_before_string.encode("ascii") - #+ self.not_after_string.encode("ascii") + # + self.identifier.encode("ascii") + # + self.not_before_string.encode("ascii") + # + self.not_after_string.encode("ascii") ) return message diff --git a/aea/test_tools/test_cases.py b/aea/test_tools/test_cases.py index fa6ffaf53e..93644ea510 100644 --- a/aea/test_tools/test_cases.py +++ b/aea/test_tools/test_cases.py @@ -875,7 +875,7 @@ def teardown_class(cls): cls.stdout = {} cls.stderr = {} - #with suppress(OSError, IOError): + # with suppress(OSError, IOError): # shutil.rmtree(cls.t) cls._is_teardown_class_called = True diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index 881dd84dff..1f964a31cf 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -17,9 +17,6 @@ # # ------------------------------------------------------------------------------ """This module contains the p2p libp2p connection.""" -from aea.crypto.fetchai import FetchAIHelper - -from aea.helpers.base import CertRequest import asyncio import logging import os @@ -37,10 +34,12 @@ from aea.configurations.constants import DEFAULT_LEDGER from aea.connections.base import Connection, ConnectionStates from aea.crypto.base import Crypto +from aea.crypto.fetchai import FetchAIHelper from aea.crypto.registries import make_crypto from aea.exceptions import enforce from aea.helpers.acn.agent_record import AgentRecord, recover_verify_keys_from_message from aea.helpers.acn.uri import Uri +from aea.helpers.base import CertRequest from aea.helpers.multiaddr.base import MultiAddr from aea.helpers.pipe import IPCChannel, make_ipc_channel from aea.mail.base import Envelope @@ -433,16 +432,25 @@ def stop(self) -> None: if os.path.exists(LIBP2P_NODE_ENV_FILE): os.remove(LIBP2P_NODE_ENV_FILE) -def _get_signature_from_cert_request(cert: CertRequest, node_public_key: str, agent_address: str) -> Tuple[str, str]: + +def _get_signature_from_cert_request( + cert: CertRequest, node_public_key: str, agent_address: str +) -> Tuple[str, str]: # must match aea/cli/issue_certificates.py:_process_certificate - signature = bytes.fromhex(Path(cert.save_path).read_bytes().decode("ascii")).decode("ascii") - public_keys = recover_verify_keys_from_message(cert.get_message(node_public_key), signature) + signature = bytes.fromhex(Path(cert.save_path).read_bytes().decode("ascii")).decode( + "ascii" + ) + public_keys = recover_verify_keys_from_message( + cert.get_message(node_public_key), signature + ) addresses = [ - FetchAIHelper.get_address_from_public_key(public_key) for public_key in public_keys + FetchAIHelper.get_address_from_public_key(public_key) + for public_key in public_keys ] verify_key = public_keys[addresses.index(agent_address)] return signature, verify_key + class P2PLibp2pConnection(Connection): """A libp2p p2p node connection.""" @@ -484,7 +492,9 @@ def __init__(self, **kwargs): ): # pragma: no cover key = self.crypto_store.crypto_objects[ledger_id] else: - raise ValueError(f"Couldn't find connection key for {str(ledger_id)} in connections keys") + raise ValueError( + f"Couldn't find connection key for {str(ledger_id)} in connections keys" + ) uri = None if libp2p_local_uri is not None: @@ -546,10 +556,20 @@ def __init__(self, **kwargs): if cert_requests is None or len(cert_requests) != 1: raise ValueError("cert_requests field must be set") if not Path(cert_requests[0].save_path).is_file(): - raise Exception("cert_request 'save_path' field is not file. Please ensure that 'issue-certificates' command is called beforehand") + raise Exception( + "cert_request 'save_path' field is not file. Please ensure that 'issue-certificates' command is called beforehand" + ) - signature, agent_public_key = _get_signature_from_cert_request(cert_requests[0], key.public_key, self.address) - record = AgentRecord(self.address, agent_public_key, key.public_key, signature, POR_DEFAULT_SERVICE_ID) + signature, agent_public_key = _get_signature_from_cert_request( + cert_requests[0], key.public_key, self.address + ) + record = AgentRecord( + self.address, + agent_public_key, + key.public_key, + signature, + POR_DEFAULT_SERVICE_ID, + ) if not record.is_valid_for(self.address, key.public_key): raise ValueError("Invalid Proof-of-Representation {}".format(str(record))) diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/options.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/options.go index 9f247f1253..e85c24c47b 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/options.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtclient/options.go @@ -42,14 +42,6 @@ func IdentityFromFetchAIKey(key string) Option { } // RegisterAgentAddress for dhtclient.New -func RegisterAgentAddressOld(addr string, isReady func() bool) Option { - return func(dhtClient *DHTClient) error { - dhtClient.myAgentAddress = addr - dhtClient.myAgentReady = isReady - return nil - } -} - func RegisterAgentAddress(record *aea.AgentRecord, isReady func() bool) Option { return func(dhtClient *DHTClient) error { pbRecord := &dhtnode.AgentRecord{} diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go index 308c5d8b13..2efc29deb6 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.pb.go @@ -108,6 +108,8 @@ type AgentRecord struct { PublicKey string `protobuf:"bytes,3,opt,name=public_key,json=publicKey,proto3" json:"public_key,omitempty"` PeerPublicKey string `protobuf:"bytes,4,opt,name=peer_public_key,json=peerPublicKey,proto3" json:"peer_public_key,omitempty"` Signature string `protobuf:"bytes,5,opt,name=signature,proto3" json:"signature,omitempty"` + NotBefore string `protobuf:"bytes,6,opt,name=not_before,json=notBefore,proto3" json:"not_before,omitempty"` + NotAfter string `protobuf:"bytes,7,opt,name=not_after,json=notAfter,proto3" json:"not_after,omitempty"` } func (x *AgentRecord) Reset() { @@ -177,6 +179,20 @@ func (x *AgentRecord) GetSignature() string { return "" } +func (x *AgentRecord) GetNotBefore() string { + if x != nil { + return x.NotBefore + } + return "" +} + +func (x *AgentRecord) GetNotAfter() string { + if x != nil { + return x.NotAfter + } + return "" +} + type Register struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -564,7 +580,7 @@ var File_dht_dhtnode_message_proto protoreflect.FileDescriptor var file_dht_dhtnode_message_proto_rawDesc = []byte{ 0x0a, 0x19, 0x64, 0x68, 0x74, 0x2f, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x07, 0x64, 0x68, 0x74, - 0x6e, 0x6f, 0x64, 0x65, 0x22, 0xab, 0x01, 0x0a, 0x0b, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x6e, 0x6f, 0x64, 0x65, 0x22, 0xe7, 0x01, 0x0a, 0x0b, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, @@ -575,67 +591,71 @@ var file_dht_dhtnode_message_proto_rawDesc = []byte{ 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x70, 0x65, 0x65, 0x72, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x22, 0x38, 0x0a, 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x2c, - 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, - 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, - 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0x34, 0x0a, 0x0d, - 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, - 0x0d, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, - 0x73, 0x73, 0x22, 0x49, 0x0a, 0x0e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, + 0x72, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x6f, 0x74, 0x5f, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x6f, 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, + 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x6e, 0x6f, 0x74, 0x5f, 0x61, 0x66, 0x74, 0x65, 0x72, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6e, 0x6f, 0x74, 0x41, 0x66, 0x74, 0x65, 0x72, 0x22, 0x38, + 0x0a, 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x2c, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, - 0x52, 0x0b, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0x51, 0x0a, - 0x0b, 0x41, 0x65, 0x61, 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, - 0x65, 0x6e, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x65, 0x6e, 0x76, - 0x65, 0x6c, 0x12, 0x2c, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x67, 0x65, - 0x6e, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, - 0x22, 0xdb, 0x02, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x0a, 0x04, 0x63, - 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x64, 0x68, 0x74, 0x6e, - 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x45, 0x72, 0x72, 0x43, 0x6f, - 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x73, 0x67, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x22, 0x8f, 0x02, 0x0a, - 0x07, 0x45, 0x72, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, - 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, - 0x4e, 0x53, 0x55, 0x50, 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, - 0x4f, 0x4e, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, - 0x45, 0x58, 0x50, 0x45, 0x43, 0x54, 0x45, 0x44, 0x5f, 0x50, 0x41, 0x59, 0x4c, 0x4f, 0x41, 0x44, - 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x47, 0x45, 0x4e, 0x45, - 0x52, 0x49, 0x43, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x53, - 0x45, 0x52, 0x49, 0x41, 0x4c, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x04, 0x12, 0x1d, - 0x0a, 0x19, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x57, 0x52, 0x4f, 0x4e, 0x47, 0x5f, 0x41, 0x47, - 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x0a, 0x12, 0x1a, 0x0a, - 0x16, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x57, 0x52, 0x4f, 0x4e, 0x47, 0x5f, 0x50, 0x55, 0x42, - 0x4c, 0x49, 0x43, 0x5f, 0x4b, 0x45, 0x59, 0x10, 0x0b, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x52, 0x52, - 0x4f, 0x52, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x4f, 0x46, - 0x10, 0x0c, 0x12, 0x1f, 0x0a, 0x1b, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, - 0x4f, 0x57, 0x4e, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, - 0x53, 0x10, 0x14, 0x12, 0x19, 0x0a, 0x15, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x41, 0x47, 0x45, - 0x4e, 0x54, 0x5f, 0x4e, 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x15, 0x22, 0xcd, - 0x02, 0x0a, 0x0a, 0x41, 0x63, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, - 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, - 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x48, 0x00, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x2f, 0x0a, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x52, - 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x48, 0x00, 0x52, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, - 0x74, 0x65, 0x72, 0x12, 0x3f, 0x0a, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x72, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, 0x68, - 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x42, 0x0a, 0x0f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x72, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, - 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x0c, 0x61, 0x65, 0x61, 0x5f, - 0x65, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, - 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x65, 0x61, 0x45, 0x6e, 0x76, 0x65, - 0x6c, 0x6f, 0x70, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x65, 0x61, 0x45, 0x6e, 0x76, 0x65, 0x6c, - 0x6f, 0x70, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0x34, 0x0a, 0x0d, 0x4c, 0x6f, 0x6f, 0x6b, + 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x67, 0x65, + 0x6e, 0x74, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0x49, + 0x0a, 0x0e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x37, 0x0a, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, + 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x0b, 0x61, 0x67, + 0x65, 0x6e, 0x74, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0x51, 0x0a, 0x0b, 0x41, 0x65, 0x61, + 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x6e, 0x76, 0x65, + 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x65, 0x6e, 0x76, 0x65, 0x6c, 0x12, 0x2c, + 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, + 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0xdb, 0x02, 0x0a, + 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x45, 0x72, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, + 0x63, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x04, 0x6d, 0x73, 0x67, 0x73, 0x22, 0x8f, 0x02, 0x0a, 0x07, 0x45, 0x72, 0x72, + 0x43, 0x6f, 0x64, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, + 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x53, 0x55, 0x50, + 0x50, 0x4f, 0x52, 0x54, 0x45, 0x44, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x10, 0x01, + 0x12, 0x1c, 0x0a, 0x18, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x45, 0x58, 0x50, 0x45, + 0x43, 0x54, 0x45, 0x44, 0x5f, 0x50, 0x41, 0x59, 0x4c, 0x4f, 0x41, 0x44, 0x10, 0x02, 0x12, 0x11, + 0x0a, 0x0d, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x10, + 0x03, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x53, 0x45, 0x52, 0x49, 0x41, + 0x4c, 0x49, 0x5a, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x04, 0x12, 0x1d, 0x0a, 0x19, 0x45, 0x52, + 0x52, 0x4f, 0x52, 0x5f, 0x57, 0x52, 0x4f, 0x4e, 0x47, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, + 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x0a, 0x12, 0x1a, 0x0a, 0x16, 0x45, 0x52, 0x52, + 0x4f, 0x52, 0x5f, 0x57, 0x52, 0x4f, 0x4e, 0x47, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, + 0x4b, 0x45, 0x59, 0x10, 0x0b, 0x12, 0x17, 0x0a, 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x49, + 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x4f, 0x46, 0x10, 0x0c, 0x12, 0x1f, + 0x0a, 0x1b, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, + 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x41, 0x44, 0x44, 0x52, 0x45, 0x53, 0x53, 0x10, 0x14, 0x12, + 0x19, 0x0a, 0x15, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x41, 0x47, 0x45, 0x4e, 0x54, 0x5f, 0x4e, + 0x4f, 0x54, 0x5f, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x15, 0x22, 0xcd, 0x02, 0x0a, 0x0a, 0x41, + 0x63, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x29, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x48, 0x00, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, + 0x0a, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x11, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, + 0x74, 0x65, 0x72, 0x48, 0x00, 0x52, 0x08, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, + 0x3f, 0x0a, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, 0x68, 0x74, 0x6e, 0x6f, 0x64, + 0x65, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, + 0x00, 0x52, 0x0d, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x42, 0x0a, 0x0f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x64, 0x68, 0x74, 0x6e, + 0x6f, 0x64, 0x65, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x0c, 0x61, 0x65, 0x61, 0x5f, 0x65, 0x6e, 0x76, 0x65, + 0x6c, 0x6f, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x68, 0x74, + 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x41, 0x65, 0x61, 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, + 0x48, 0x00, 0x52, 0x0b, 0x61, 0x65, 0x61, 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x42, + 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto index 13f02ded29..df100d5764 100644 --- a/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto +++ b/packages/fetchai/connections/p2p_libp2p/dht/dhtnode/message.proto @@ -8,6 +8,8 @@ message AgentRecord { string public_key = 3; string peer_public_key = 4; string signature = 5; + string not_before = 6; + string not_after = 7; } message Register { diff --git a/packages/fetchai/connections/p2p_libp2p_client/acn_message.proto b/packages/fetchai/connections/p2p_libp2p_client/acn_message.proto index 13f02ded29..df100d5764 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/acn_message.proto +++ b/packages/fetchai/connections/p2p_libp2p_client/acn_message.proto @@ -8,6 +8,8 @@ message AgentRecord { string public_key = 3; string peer_public_key = 4; string signature = 5; + string not_before = 6; + string not_after = 7; } message Register { diff --git a/packages/fetchai/connections/p2p_libp2p_client/acn_message_pb2.py b/packages/fetchai/connections/p2p_libp2p_client/acn_message_pb2.py index 1e3a7e8608..cfca598f9f 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/acn_message_pb2.py +++ b/packages/fetchai/connections/p2p_libp2p_client/acn_message_pb2.py @@ -20,7 +20,7 @@ package="dhtnode", syntax="proto3", serialized_pb=_b( - '\n\x11\x61\x63n_message.proto\x12\x07\x64htnode"r\n\x0b\x41gentRecord\x12\x12\n\nservice_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\t\x12\x12\n\npublic_key\x18\x03 \x01(\t\x12\x17\n\x0fpeer_public_key\x18\x04 \x01(\t\x12\x11\n\tsignature\x18\x05 \x01(\t"0\n\x08Register\x12$\n\x06record\x18\x01 \x01(\x0b\x32\x14.dhtnode.AgentRecord"&\n\rLookupRequest\x12\x15\n\ragent_address\x18\x01 \x01(\t"<\n\x0eLookupResponse\x12*\n\x0c\x61gent_record\x18\x01 \x01(\x0b\x32\x14.dhtnode.AgentRecord"B\n\x0b\x41\x65\x61\x45nvelope\x12\r\n\x05\x65nvel\x18\x01 \x01(\x0c\x12$\n\x06record\x18\x02 \x01(\x0b\x32\x14.dhtnode.AgentRecord"\xcf\x02\n\x06Status\x12%\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x17.dhtnode.Status.ErrCode\x12\x0c\n\x04msgs\x18\x02 \x03(\t"\x8f\x02\n\x07\x45rrCode\x12\x0b\n\x07SUCCESS\x10\x00\x12\x1d\n\x19\x45RROR_UNSUPPORTED_VERSION\x10\x01\x12\x1c\n\x18\x45RROR_UNEXPECTED_PAYLOAD\x10\x02\x12\x11\n\rERROR_GENERIC\x10\x03\x12\x17\n\x13\x45RROR_SERIALIZATION\x10\x04\x12\x1d\n\x19\x45RROR_WRONG_AGENT_ADDRESS\x10\n\x12\x1a\n\x16\x45RROR_WRONG_PUBLIC_KEY\x10\x0b\x12\x17\n\x13\x45RROR_INVALID_PROOF\x10\x0c\x12\x1f\n\x1b\x45RROR_UNKNOWN_AGENT_ADDRESS\x10\x14\x12\x19\n\x15\x45RROR_AGENT_NOT_READY\x10\x15"\x86\x02\n\nAcnMessage\x12\x0f\n\x07version\x18\x01 \x01(\t\x12!\n\x06status\x18\x02 \x01(\x0b\x32\x0f.dhtnode.StatusH\x00\x12%\n\x08register\x18\x03 \x01(\x0b\x32\x11.dhtnode.RegisterH\x00\x12\x30\n\x0elookup_request\x18\x04 \x01(\x0b\x32\x16.dhtnode.LookupRequestH\x00\x12\x32\n\x0flookup_response\x18\x05 \x01(\x0b\x32\x17.dhtnode.LookupResponseH\x00\x12,\n\x0c\x61\x65\x61_envelope\x18\x06 \x01(\x0b\x32\x14.dhtnode.AeaEnvelopeH\x00\x42\t\n\x07payloadb\x06proto3' + '\n\x11\x61\x63n_message.proto\x12\x07\x64htnode"\x99\x01\n\x0b\x41gentRecord\x12\x12\n\nservice_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x02 \x01(\t\x12\x12\n\npublic_key\x18\x03 \x01(\t\x12\x17\n\x0fpeer_public_key\x18\x04 \x01(\t\x12\x11\n\tsignature\x18\x05 \x01(\t\x12\x12\n\nnot_before\x18\x06 \x01(\t\x12\x11\n\tnot_after\x18\x07 \x01(\t"0\n\x08Register\x12$\n\x06record\x18\x01 \x01(\x0b\x32\x14.dhtnode.AgentRecord"&\n\rLookupRequest\x12\x15\n\ragent_address\x18\x01 \x01(\t"<\n\x0eLookupResponse\x12*\n\x0c\x61gent_record\x18\x01 \x01(\x0b\x32\x14.dhtnode.AgentRecord"B\n\x0b\x41\x65\x61\x45nvelope\x12\r\n\x05\x65nvel\x18\x01 \x01(\x0c\x12$\n\x06record\x18\x02 \x01(\x0b\x32\x14.dhtnode.AgentRecord"\xcf\x02\n\x06Status\x12%\n\x04\x63ode\x18\x01 \x01(\x0e\x32\x17.dhtnode.Status.ErrCode\x12\x0c\n\x04msgs\x18\x02 \x03(\t"\x8f\x02\n\x07\x45rrCode\x12\x0b\n\x07SUCCESS\x10\x00\x12\x1d\n\x19\x45RROR_UNSUPPORTED_VERSION\x10\x01\x12\x1c\n\x18\x45RROR_UNEXPECTED_PAYLOAD\x10\x02\x12\x11\n\rERROR_GENERIC\x10\x03\x12\x17\n\x13\x45RROR_SERIALIZATION\x10\x04\x12\x1d\n\x19\x45RROR_WRONG_AGENT_ADDRESS\x10\n\x12\x1a\n\x16\x45RROR_WRONG_PUBLIC_KEY\x10\x0b\x12\x17\n\x13\x45RROR_INVALID_PROOF\x10\x0c\x12\x1f\n\x1b\x45RROR_UNKNOWN_AGENT_ADDRESS\x10\x14\x12\x19\n\x15\x45RROR_AGENT_NOT_READY\x10\x15"\x86\x02\n\nAcnMessage\x12\x0f\n\x07version\x18\x01 \x01(\t\x12!\n\x06status\x18\x02 \x01(\x0b\x32\x0f.dhtnode.StatusH\x00\x12%\n\x08register\x18\x03 \x01(\x0b\x32\x11.dhtnode.RegisterH\x00\x12\x30\n\x0elookup_request\x18\x04 \x01(\x0b\x32\x16.dhtnode.LookupRequestH\x00\x12\x32\n\x0flookup_response\x18\x05 \x01(\x0b\x32\x17.dhtnode.LookupResponseH\x00\x12,\n\x0c\x61\x65\x61_envelope\x18\x06 \x01(\x0b\x32\x14.dhtnode.AeaEnvelopeH\x00\x42\t\n\x07payloadb\x06proto3' ), ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -73,8 +73,8 @@ ], containing_type=None, options=None, - serialized_start=431, - serialized_end=702, + serialized_start=471, + serialized_end=742, ) _sym_db.RegisterEnumDescriptor(_STATUS_ERRCODE) @@ -171,6 +171,40 @@ extension_scope=None, options=None, ), + _descriptor.FieldDescriptor( + name="not_before", + full_name="dhtnode.AgentRecord.not_before", + index=5, + number=6, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), + _descriptor.FieldDescriptor( + name="not_after", + full_name="dhtnode.AgentRecord.not_after", + index=6, + number=7, + type=9, + cpp_type=9, + label=1, + has_default_value=False, + default_value=_b("").decode("utf-8"), + message_type=None, + enum_type=None, + containing_type=None, + is_extension=False, + extension_scope=None, + options=None, + ), ], extensions=[], nested_types=[], @@ -180,8 +214,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=30, - serialized_end=144, + serialized_start=31, + serialized_end=184, ) @@ -218,8 +252,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=146, - serialized_end=194, + serialized_start=186, + serialized_end=234, ) @@ -256,8 +290,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=196, - serialized_end=234, + serialized_start=236, + serialized_end=274, ) @@ -294,8 +328,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=236, - serialized_end=296, + serialized_start=276, + serialized_end=336, ) @@ -349,8 +383,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=298, - serialized_end=364, + serialized_start=338, + serialized_end=404, ) @@ -404,8 +438,8 @@ syntax="proto3", extension_ranges=[], oneofs=[], - serialized_start=367, - serialized_end=702, + serialized_start=407, + serialized_end=742, ) @@ -535,8 +569,8 @@ fields=[], ), ], - serialized_start=705, - serialized_end=967, + serialized_start=745, + serialized_end=1007, ) _REGISTER.fields_by_name["record"].message_type = _AGENTRECORD diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.py b/packages/fetchai/connections/p2p_libp2p_client/connection.py index 4549b26312..ffe4fefa25 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.py +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.py @@ -19,29 +19,34 @@ """This module contains the libp2p client connection.""" -from aea.crypto.fetchai import FetchAIHelper -from pathlib import Path -from aea.helpers.base import CertRequest import asyncio import logging import random import struct from asyncio import CancelledError +from pathlib import Path from typing import List, Optional, Tuple, Union, cast from aea.configurations.base import PublicId from aea.configurations.constants import DEFAULT_LEDGER from aea.connections.base import Connection, ConnectionStates from aea.crypto.base import Crypto +from aea.crypto.fetchai import FetchAIHelper from aea.crypto.registries import make_crypto from aea.exceptions import enforce from aea.helpers.acn.agent_record import AgentRecord, recover_verify_keys_from_message from aea.helpers.acn.uri import Uri +from aea.helpers.base import CertRequest from aea.mail.base import Envelope from packages.fetchai.connections.p2p_libp2p_client.acn_message_pb2 import AcnMessage -from packages.fetchai.connections.p2p_libp2p_client.acn_message_pb2 import AgentRecord as AgentRecordPb -from packages.fetchai.connections.p2p_libp2p_client.acn_message_pb2 import Register, Status +from packages.fetchai.connections.p2p_libp2p_client.acn_message_pb2 import ( + AgentRecord as AgentRecordPb, +) +from packages.fetchai.connections.p2p_libp2p_client.acn_message_pb2 import ( + Register, + Status, +) _default_logger = logging.getLogger( diff --git a/tests/conftest.py b/tests/conftest.py index 29e4e92282..42957350a9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,7 +17,6 @@ # # ------------------------------------------------------------------------------ """Conftest module for Pytest.""" -from aea.helpers.base import CertRequest import difflib import inspect import logging @@ -69,6 +68,7 @@ from aea.crypto.ledger_apis import DEFAULT_LEDGER_CONFIGS from aea.crypto.registries import ledger_apis_registry, make_crypto from aea.crypto.wallet import CryptoStore +from aea.helpers.base import CertRequest from aea.identity.base import Identity from aea.test_tools.click_testing import CliRunner as ImportedCliRunner from aea.test_tools.constants import DEFAULT_AUTHOR @@ -210,8 +210,12 @@ PUBLIC_DHT_P2P_MADDR_2 = "/dns4/acn.fetch.ai/tcp/9001/p2p/16Uiu2HAmVWnopQAqq4pniYLw44VRvYxBUoRHqjz1Hh2SoCyjbyRW" PUBLIC_DHT_DELEGATE_URI_1 = "acn.fetch.ai:11000" PUBLIC_DHT_DELEGATE_URI_2 = "acn.fetch.ai:11001" -PUBLIC_DHT_P2P_PUBLIC_KEY_1 = "0217a59bd805c310aca4febe0e99ce22ee3712ae085dc1e5630430b1e15a584bb7" -PUBLIC_DHT_P2P_PUBLIC_KEY_2 = "03fa7cfae1037cba5218f0f5743802eced8de3247c55ecebaae46c7d3679e3f91d" +PUBLIC_DHT_P2P_PUBLIC_KEY_1 = ( + "0217a59bd805c310aca4febe0e99ce22ee3712ae085dc1e5630430b1e15a584bb7" +) +PUBLIC_DHT_P2P_PUBLIC_KEY_2 = ( + "03fa7cfae1037cba5218f0f5743802eced8de3247c55ecebaae46c7d3679e3f91d" +) # testnets COSMOS_TESTNET_CONFIG = {"address": COSMOS_DEFAULT_ADDRESS} @@ -778,12 +782,14 @@ def _make_stub_connection(input_file_path: str, output_file_path: str): connection = StubConnection(configuration=configuration) return connection + def _process_cert(key: Crypto, cert: CertRequest): # must match aea/cli/issue_certificates.py:_process_certificate message = cert.get_message(cert.public_key) signature = key.sign_message(message).encode("ascii").hex() Path(cert.save_path).write_bytes(signature.encode("ascii")) + def _make_libp2p_connection( port: int = 10234, host: str = "127.0.0.1", @@ -868,76 +874,11 @@ def _make_libp2p_connection( return connection -def _make_libp2p_connection_delete_me( - port: int = 10234, - host: str = "127.0.0.1", - relay: bool = True, - delegate: bool = False, - entry_peers: Optional[Sequence[MultiAddr]] = None, - delegate_port: int = 11234, - delegate_host: str = "127.0.0.1", - node_key_file: Optional[str] = None, - agent_address: Optional[Address] = None, - build_directory: Optional[str] = None, -) -> P2PLibp2pConnection: - log_file = "libp2p_node_{}.log".format(port) - if os.path.exists(log_file): - os.remove(log_file) - - shared_key = make_crypto(DEFAULT_LEDGER) - shared_key_path = f"./{shared_key.public_key}.txt" - with open(shared_key_path, "wb") as f: - shared_key.dump(f) - - address = agent_address - if address is None: - address = shared_key.address - identity = Identity("", address=address) - - key_file = node_key_file - if key_file is None: - key_file = shared_key_path - - if not build_directory: - build_directory = os.getcwd() - - if relay and delegate: - configuration = ConnectionConfig( - node_key_file=key_file, - local_uri="{}:{}".format(host, port), - public_uri="{}:{}".format(host, port), - entry_peers=entry_peers, - log_file=log_file, - delegate_uri="{}:{}".format(delegate_host, delegate_port), - connection_id=P2PLibp2pConnection.connection_id, - build_directory=build_directory, - ) - elif relay and not delegate: - configuration = ConnectionConfig( - node_key_file=key_file, - local_uri="{}:{}".format(host, port), - public_uri="{}:{}".format(host, port), - entry_peers=entry_peers, - log_file=log_file, - connection_id=P2PLibp2pConnection.connection_id, - build_directory=build_directory, - ) - else: - configuration = ConnectionConfig( - node_key_file=key_file, - local_uri="{}:{}".format(host, port), - entry_peers=entry_peers, - log_file=log_file, - connection_id=P2PLibp2pConnection.connection_id, - build_directory=build_directory, - ) - if not os.path.exists(os.path.join(build_directory, LIBP2P_NODE_MODULE_NAME)): - build_node(build_directory) - return P2PLibp2pConnection(configuration=configuration, identity=identity) - - def _make_libp2p_client_connection( - peer_public_key: str, node_port: int = 11234, node_host: str = "127.0.0.1", uri: Optional[str] = None, + peer_public_key: str, + node_port: int = 11234, + node_host: str = "127.0.0.1", + uri: Optional[str] = None, ) -> P2PLibp2pClientConnection: crypto = make_crypto(DEFAULT_LEDGER) identity = Identity("", address=crypto.address) @@ -966,36 +907,6 @@ def _make_libp2p_client_connection( return P2PLibp2pClientConnection(configuration=configuration, identity=identity) -def _make_libp2p_client_connection_delete_me( - node_port: int = 11234, - node_host: str = "127.0.0.1", - uri: Optional[str] = None, - cert: Optional[str] = None, - key: Optional[Crypto] = None, -) -> P2PLibp2pClientConnection: - crypto = key - if crypto is None: - crypto = make_crypto(COSMOS) - identity = Identity("", address=crypto.address) - key_file = f"./{crypto.public_key}.txt" - with open(key_file, "wb") as f: - crypto.dump(f) - - configuration = ConnectionConfig( - client_key_file=key_file, - nodes=[ - { - "uri": str(uri) - if uri is not None - else "{}:{}".format(node_host, node_port), - "cert": cert if cert is not None else crypto.public_key, - }, - ], - connection_id=P2PLibp2pClientConnection.connection_id, - ) - return P2PLibp2pClientConnection(configuration=configuration, identity=identity) - - def libp2p_log_on_failure(fn: Callable) -> Callable: """ Decorate a pytest method running a libp2p node to print its logs in case test fails. diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py index 02e86113cf..8561925faf 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py @@ -19,11 +19,11 @@ """This test module contains AEA cli tests for P2PLibp2p connection.""" -from aea.configurations.constants import DEFAULT_LEDGER -from aea.crypto.registries import make_crypto import os from pathlib import Path +from aea.configurations.constants import DEFAULT_LEDGER +from aea.crypto.registries import make_crypto from aea.test_tools.test_cases import AEATestCaseEmpty from packages.fetchai.connections.p2p_libp2p.connection import ( @@ -59,7 +59,9 @@ def test_agent(self): self.add_item("connection", str(P2P_CONNECTION_PUBLIC_ID)) self.run_cli_command("build", cwd=self._get_cwd()) self.set_config("agent.default_connection", str(P2P_CONNECTION_PUBLIC_ID)) - self.nested_set_config("agent.connection_private_key_paths", {DEFAULT_LEDGER:self.conn_key_file}) + self.nested_set_config( + "agent.connection_private_key_paths", {DEFAULT_LEDGER: self.conn_key_file} + ) # for logging config_path = "vendor.fetchai.connections.p2p_libp2p.config" @@ -112,7 +114,9 @@ def test_agent(self): """Test with aea.""" self.add_item("connection", str(P2P_CONNECTION_PUBLIC_ID)) self.run_cli_command("build", cwd=self._get_cwd()) - self.nested_set_config("agent.connection_private_key_paths", {DEFAULT_LEDGER:self.conn_key_file}) + self.nested_set_config( + "agent.connection_private_key_paths", {DEFAULT_LEDGER: self.conn_key_file} + ) # setup a full node: with public uri, relay service, and delegate service config_path = "vendor.fetchai.connections.p2p_libp2p.config" diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py index d54d5bd18a..1755b119b6 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py @@ -17,7 +17,6 @@ # # ------------------------------------------------------------------------------ """This test module contains Negative tests for Libp2p connection.""" -from aea.crypto.wallet import CryptoStore import os import shutil import tempfile @@ -27,6 +26,7 @@ from aea.configurations.base import ConnectionConfig from aea.crypto.registries import make_crypto +from aea.crypto.wallet import CryptoStore from aea.identity.base import Identity from aea.multiplexer import Multiplexer @@ -186,7 +186,7 @@ def test_libp2pconnection_node_config_registration_delay(mock): """Test node registration delay configuration""" host = "localhost" port = "10000" - + _make_libp2p_connection(port, host) with pytest.raises(ValueError): _make_libp2p_connection(port, host, peer_registration_delay="must_be_float") diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_fault_tolerance.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_fault_tolerance.py index 614d78f8a8..535a53f551 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_fault_tolerance.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_fault_tolerance.py @@ -218,7 +218,9 @@ def setup_class(cls): cls.connection_key = make_crypto(DEFAULT_LEDGER) cls.connection2 = _make_libp2p_connection( - DEFAULT_PORT + 2, entry_peers=[genesis_peer], agent_key=cls.connection_key, + DEFAULT_PORT + 2, + entry_peers=[genesis_peer], + agent_key=cls.connection_key, ) cls.multiplexer2 = Multiplexer([cls.connection2]) cls.log_files.append(cls.connection2.node.log_file) diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py index 8b46993423..38abf15816 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py @@ -308,8 +308,7 @@ def test_communication_indirect(self): multiplexers = [] try: connection1 = _make_libp2p_client_connection( - PUBLIC_DHT_PUBLIC_KEYS[i], - uri=PUBLIC_DHT_DELEGATE_URIS[i] + PUBLIC_DHT_PUBLIC_KEYS[i], uri=PUBLIC_DHT_DELEGATE_URIS[i] ) multiplexer1 = Multiplexer([connection1]) multiplexer1.connect() @@ -322,8 +321,7 @@ def test_communication_indirect(self): continue connection2 = _make_libp2p_client_connection( - PUBLIC_DHT_PUBLIC_KEYS[i], - uri=PUBLIC_DHT_DELEGATE_URIS[j] + PUBLIC_DHT_PUBLIC_KEYS[i], uri=PUBLIC_DHT_DELEGATE_URIS[j] ) multiplexer2 = Multiplexer([connection2]) multiplexer2.connect() diff --git a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py index 61fbc91d90..691c4ccf03 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py @@ -70,25 +70,32 @@ def test_connection(self): self.add_item("connection", str(PUBLIC_ID)) conn_path = "vendor.fetchai.connections.p2p_libp2p_client" self.nested_set_config( - conn_path+".config", - {"nodes": [{ - "uri": "{}:{}".format(DEFAULT_HOST, DEFAULT_DELEGATE_PORT), - "public_key" : self.node_connection.node.pub, - }]}, + conn_path + ".config", + { + "nodes": [ + { + "uri": "{}:{}".format(DEFAULT_HOST, DEFAULT_DELEGATE_PORT), + "public_key": self.node_connection.node.pub, + } + ] + }, ) # generate certificates for connection self.nested_set_config( - conn_path+".cert_requests", - [CertRequest( - identifier="acn", - ledger_id="fetchai", - not_after="2022-01-01", - not_before="2021-01-01", - public_key=self.node_connection.node.pub, - save_path="./cli_test_cert.txt")], + conn_path + ".cert_requests", + [ + CertRequest( + identifier="acn", + ledger_id="fetchai", + not_after="2022-01-01", + not_before="2021-01-01", + public_key=self.node_connection.node.pub, + save_path="./cli_test_cert.txt", + ) + ], ) - + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) process = self.run_agent() diff --git a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_communication.py b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_communication.py index 14db4c1b00..15852fee94 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_communication.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_communication.py @@ -106,11 +106,15 @@ def setup_class(cls): cls.multiplexer_node.connect() try: - cls.connection_client_1 = _make_libp2p_client_connection(cls.connection_node.node.pub) + cls.connection_client_1 = _make_libp2p_client_connection( + cls.connection_node.node.pub + ) cls.multiplexer_client_1 = Multiplexer([cls.connection_client_1]) cls.multiplexer_client_1.connect() - cls.connection_client_2 = _make_libp2p_client_connection(cls.connection_node.node.pub) + cls.connection_client_2 = _make_libp2p_client_connection( + cls.connection_node.node.pub + ) cls.multiplexer_client_2 = Multiplexer([cls.connection_client_2]) cls.multiplexer_client_2.connect() except Exception: @@ -274,16 +278,14 @@ def setup_class(cls): cls.mutliplexers.append(cls.multiplexer_node_2) cls.connection_client_1 = _make_libp2p_client_connection( - cls.connection_node_1.node.pub, - DEFAULT_DELEGATE_PORT + 1 + cls.connection_node_1.node.pub, DEFAULT_DELEGATE_PORT + 1 ) cls.multiplexer_client_1 = Multiplexer([cls.connection_client_1]) cls.multiplexer_client_1.connect() cls.mutliplexers.append(cls.multiplexer_client_1) cls.connection_client_2 = _make_libp2p_client_connection( - cls.connection_node_2.node.pub, - DEFAULT_DELEGATE_PORT + 2 + cls.connection_node_2.node.pub, DEFAULT_DELEGATE_PORT + 2 ) cls.multiplexer_client_2 = Multiplexer([cls.connection_client_2]) cls.multiplexer_client_2.connect() @@ -456,7 +458,10 @@ def setup_class(cls): for _ in range(DEFAULT_CLIENTS_PER_NODE): ports = [DEFAULT_DELEGATE_PORT + 1, DEFAULT_DELEGATE_PORT + 2] - peers_public_keys = [cls.connection_node_1.node.pub, cls.connection_node_2.node.pub] + peers_public_keys = [ + cls.connection_node_1.node.pub, + cls.connection_node_2.node.pub, + ] for i in range(len(ports)): port = ports[i] peer_public_key = peers_public_keys[i] diff --git a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_errors.py b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_errors.py index 484f658d08..ca0c909994 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_errors.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_errors.py @@ -19,8 +19,6 @@ """This test module contains negative tests for Libp2p tcp client connection.""" -from aea.helpers.base import CertRequest -from aea.configurations.constants import DEFAULT_LEDGER import os import shutil import tempfile @@ -28,18 +26,22 @@ import pytest from aea.configurations.base import ConnectionConfig +from aea.configurations.constants import DEFAULT_LEDGER from aea.crypto.registries import make_crypto +from aea.helpers.base import CertRequest from aea.identity.base import Identity from aea.multiplexer import Multiplexer from packages.fetchai.connections.p2p_libp2p_client.connection import ( - P2PLibp2pClientConnection, POR_DEFAULT_SERVICE_ID, + P2PLibp2pClientConnection, + POR_DEFAULT_SERVICE_ID, ) from tests.conftest import ( COSMOS, _make_libp2p_client_connection, - _make_libp2p_connection, _process_cert, + _make_libp2p_connection, + _process_cert, libp2p_log_on_failure, ) @@ -90,10 +92,12 @@ def test_empty_nodes(self): """Test empty nodes.""" configuration = ConnectionConfig( client_key_file=self.key_file, - nodes=[{ - "uri": "{}:{}".format(self.node_host, self.node_port), - "public_key": self.peer_crypto.public_key, - }], + nodes=[ + { + "uri": "{}:{}".format(self.node_host, self.node_port), + "public_key": self.peer_crypto.public_key, + } + ], connection_id=P2PLibp2pClientConnection.connection_id, cert_requests=[self.cert_request], ) @@ -140,7 +144,9 @@ def setup_class(cls): cls.multiplexer_node.connect() cls.multiplexers.append(cls.multiplexer_node) - cls.connection_client = _make_libp2p_client_connection(cls.connection_node.node.pub) + cls.connection_client = _make_libp2p_client_connection( + cls.connection_node.node.pub + ) cls.multiplexer_client = Multiplexer([cls.connection_client]) cls.multiplexer_client.connect() cls.multiplexers.append(cls.multiplexer_client) From 82b81ba27655434b3169ac6783522f297fd4d432 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Wed, 30 Dec 2020 22:01:08 +0000 Subject: [PATCH 096/204] Address ci comments --- aea/helpers/acn/agent_record.py | 10 ++++++++-- aea/helpers/base.py | 9 ++------- aea/test_tools/test_cases.py | 4 ++-- .../fetchai/connections/p2p_libp2p/connection.py | 1 - .../connections/p2p_libp2p/connection.yaml | 8 ++++---- .../connections/p2p_libp2p_client/connection.py | 16 ++++------------ .../p2p_libp2p_client/connection.yaml | 6 +++--- packages/hashes.csv | 4 ++-- scripts/whitelist.py | 4 ++++ .../test_p2p_libp2p/test_aea_cli.py | 1 - .../test_p2p_libp2p/test_errors.py | 1 - .../test_p2p_libp2p_client/test_errors.py | 1 - 12 files changed, 29 insertions(+), 36 deletions(-) diff --git a/aea/helpers/acn/agent_record.py b/aea/helpers/acn/agent_record.py index 5c66e21a78..5a0a6c2820 100644 --- a/aea/helpers/acn/agent_record.py +++ b/aea/helpers/acn/agent_record.py @@ -29,7 +29,13 @@ def recover_verify_keys_from_message(message: bytes, signature: str) -> List[str]: - """ get the public key used to produce the `signature` of the `message`""" + """ + Get the public key used to produce the `signature` of the `message` + + :param message: raw bytes used to produce signature + :param signature: signature of the message + """ + signature_b64 = base64.b64decode(signature) verifying_keys = VerifyingKey.from_public_key_recovery( signature_b64, message, SECP256k1, hashfunc=sha256, @@ -53,7 +59,7 @@ def __init__( ): """ Initialize the AgentRecord - + :param address: agent address :param public key: agent public key (associated to the address) :param peer_public_key: representative peer public key diff --git a/aea/helpers/base.py b/aea/helpers/base.py index 339d0d02c3..4706ce8f4f 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -746,14 +746,9 @@ def save_path(self) -> Path: """Get the save_path""" return self._save_path - def get_message(self, public_key: str) -> bytes: + def get_message(self, public_key: str) -> bytes: # pylint: disable=no-self-use """Get the message to sign.""" - message = ( - public_key.encode("ascii") - # + self.identifier.encode("ascii") - # + self.not_before_string.encode("ascii") - # + self.not_after_string.encode("ascii") - ) + message = public_key.encode("ascii") return message @property diff --git a/aea/test_tools/test_cases.py b/aea/test_tools/test_cases.py index 93644ea510..d836900d6d 100644 --- a/aea/test_tools/test_cases.py +++ b/aea/test_tools/test_cases.py @@ -875,8 +875,8 @@ def teardown_class(cls): cls.stdout = {} cls.stderr = {} - # with suppress(OSError, IOError): - # shutil.rmtree(cls.t) + with suppress(OSError, IOError): + shutil.rmtree(cls.t) cls._is_teardown_class_called = True diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index 1f964a31cf..be9578433e 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -35,7 +35,6 @@ from aea.connections.base import Connection, ConnectionStates from aea.crypto.base import Crypto from aea.crypto.fetchai import FetchAIHelper -from aea.crypto.registries import make_crypto from aea.exceptions import enforce from aea.helpers.acn.agent_record import AgentRecord, recover_verify_keys_from_message from aea.helpers.acn.uri import Uri diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index b44eb37652..80f83f5975 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -16,13 +16,13 @@ fingerprint: aea/pipe_unix.go: QmSzbAexrSUSA9ZE6VT6MmcWF5MhEcmceQjAbnfRoK7Ruv aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK check_dependencies.py: QmcZATMqEThMQBdsud1AwJEYtKq86VfPSXhnfdE7TSWcsc - connection.py: QmeC7hDQr2Uo1DmFrS5MEneTvQZSc2bFkYXq4CRGv8WBXu + connection.py: QmUBr2UXkN8RYWxKppM6zWAoqGGd5qYpuwY56VFqssohRc dht/dhtclient/dhtclient.go: QmZCgMowWw4ueCSsyYY12om3RMQA3Ydp98ti65Eqa5ccuz dht/dhtclient/dhtclient_test.go: QmbTvSmNU1uqjLQo9LdD7zWmDcLuzEacDeg7AG4VXhmriV - dht/dhtclient/options.go: QmWs7fQqHRavjwe1PjihHFi85HeJu9qSEVyMdcRviZzaT2 + dht/dhtclient/options.go: QmW3UjASUP6zGwK8b61hHQhVoLqBxQ13vZ1gcHuTmdrsDf dht/dhtnode/dhtnode.go: QmbyhgbCSAbQ1QsDw7FM7Nt5sZcvhbupA1jv5faxutbV7N - dht/dhtnode/message.pb.go: QmbMVfoPm74uxHRdAtzhke7ftn8YUQQ7kBs3e7QAjfaEc2 - dht/dhtnode/message.proto: QmfFz18nn89kn7ReBwQri7cijbEh6D9TTd3j2euCkKVvDk + dht/dhtnode/message.pb.go: QmdXBSaZxyb82k94N3RxzGwuLuw2ZchfUddyhdcHXxDJQZ + dht/dhtnode/message.proto: QmSwpCP72J94kt2MaLSPQdt8puqtLMVPp3Migwp4xRwadZ dht/dhtnode/streams.go: Qmc2JcyiU4wHsgDj6aUunMAp4c5yMzo2ixeqRZHSW5PVwo dht/dhtnode/utils.go: QmRhtiX1aUy7APRteJyFtpsJjoz8VshYDY1UcJkD8zJX1o dht/dhtpeer/benchmarks_test.go: QmQcy8WN5DY8Nfp5nkDgRaS6NjQbMzc6DSMbdY8QDPx9ao diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.py b/packages/fetchai/connections/p2p_libp2p_client/connection.py index ffe4fefa25..19db28c590 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.py +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.py @@ -30,7 +30,6 @@ from aea.configurations.base import PublicId from aea.configurations.constants import DEFAULT_LEDGER from aea.connections.base import Connection, ConnectionStates -from aea.crypto.base import Crypto from aea.crypto.fetchai import FetchAIHelper from aea.crypto.registries import make_crypto from aea.exceptions import enforce @@ -67,7 +66,7 @@ def _get_signatures_from_cert_request( ) -> Tuple[List[str], str]: signatures: List[str] = [] verify_key = "" - for i in range(len(certs)): + for i, cert in enumerate(certs): cert = certs[i] public_key = public_keys[i] signature = bytes.fromhex( @@ -140,12 +139,6 @@ def __init__(self, **kwargs): # TOFIX(): we cannot use store as the key will be used for TLS tcp connection # also, as of now all the connections share the same key - # if ( - # self.has_crypto_store - # and self.crypto_store.crypto_objects.get(ledger_id, None) is not None - # ): # pragma: no cover - # key = self.crypto_store.crypto_objects[ledger_id] - # elif key_file is not None: if key_file is not None: key = make_crypto(ledger_id, private_key_path=key_file) else: @@ -164,19 +157,18 @@ def __init__(self, **kwargs): cert_requests, nodes_public_keys, self.address ) records: List[AgentRecord] = [] - for i in range(len(signatures)): + for i, signature in enumerate(signatures): records.append( AgentRecord( self.address, agent_public_key, nodes_public_keys[i], - signatures[i], + signature, POR_DEFAULT_SERVICE_ID, ) ) - for i in range(len(nodes_public_keys)): - public_key = nodes_public_keys[i] + for i, public_key in enumerate(nodes_public_keys): uri = self.delegate_uris[i] record = next( record for record in records if record.peer_public_key == public_key diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml index e8b902826a..40e1209fbc 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml @@ -10,9 +10,9 @@ aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmP9mRtKmt8c1s9XcFeycewPbAQ9BDfkA1Zo4fNcjsTFra __init__.py: QmT1FEHkPGMHV5oiVEfQHHr25N2qdZxydSNRJabJvYiTgf - acn_message.proto: QmfFz18nn89kn7ReBwQri7cijbEh6D9TTd3j2euCkKVvDk - acn_message_pb2.py: QmXF5A7PiUiduTWEEkZQZuGRmX7QemyKx8ZDrWnevsi82i - connection.py: QmV6D74dzGbAYcsAoso7k2JBbFM76oYYNhrk8vGjacfEg9 + acn_message.proto: QmSwpCP72J94kt2MaLSPQdt8puqtLMVPp3Migwp4xRwadZ + acn_message_pb2.py: QmbVqUUCHyQpfx7dHvW5e5j3RGcjNYqsDjuPV8eNcca2Nf + connection.py: Qmcxx2GAhfafLvAuAae727k4C273oLdbr6CdxzZHDFCDWB fingerprint_ignore_patterns: [] connections: [] protocols: [] diff --git a/packages/hashes.csv b/packages/hashes.csv index d0fb9c08e0..f240b2f82d 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,8 +34,8 @@ fetchai/connections/http_server,QmQ8hNfEts6th9zfnocYHzyYw4dXkDNDj8zoKtov1wB49n fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 -fetchai/connections/p2p_libp2p,QmV6z1VJcPSvQ9g985eQQgG36JadN3YtjikpLfXFym8S6y -fetchai/connections/p2p_libp2p_client,QmcZfP63pCXH8H5zxgF1L9acfDkZU8hunCikKniwW4FJhR +fetchai/connections/p2p_libp2p,QmbqgF14cRh37m4QyJ7NW4qDma1sxcayuAtA78Sr2Zwr58 +fetchai/connections/p2p_libp2p_client,QmPyoVeDiUzUsx8cfKPxgbyxNULaZX89JBwnE6TZ33ciuN fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T fetchai/connections/scaffold,QmaCupER4vedakPPLzn2Rw9YHPmqSPaZh5xnqi6xVxQHvn diff --git a/scripts/whitelist.py b/scripts/whitelist.py index d5fd09a2d3..bb61a44022 100644 --- a/scripts/whitelist.py +++ b/scripts/whitelist.py @@ -242,3 +242,7 @@ AgentRecord # unused class (aea/helpers/acn/agent_record.py:49) is_valid_for # unused method (aea/helpers/acn/agent_record.py:96) Uri # unused class (aea/helpers/acn/uri.py:26) +recover_verify_keys_from_message # unused function (aea/helpers/acn/agent_record.py:49) +not_before_string # unused property (aea/helpers/base.py:724) +not_after_string # unused property (aea/helpers/base.py:729) + diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py index 8561925faf..44289dffa9 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py @@ -20,7 +20,6 @@ """This test module contains AEA cli tests for P2PLibp2p connection.""" import os -from pathlib import Path from aea.configurations.constants import DEFAULT_LEDGER from aea.crypto.registries import make_crypto diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py index 1755b119b6..0a401c77be 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py @@ -26,7 +26,6 @@ from aea.configurations.base import ConnectionConfig from aea.crypto.registries import make_crypto -from aea.crypto.wallet import CryptoStore from aea.identity.base import Identity from aea.multiplexer import Multiplexer diff --git a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_errors.py b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_errors.py index ca0c909994..2714272a06 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_errors.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_errors.py @@ -38,7 +38,6 @@ ) from tests.conftest import ( - COSMOS, _make_libp2p_client_connection, _make_libp2p_connection, _process_cert, From 95abe617e84cc4d52e2eda2e9d8ab0ba180a703e Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Wed, 30 Dec 2020 22:06:32 +0000 Subject: [PATCH 097/204] Apply black --- scripts/whitelist.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/whitelist.py b/scripts/whitelist.py index 9eb85dbf85..f5db9340d9 100644 --- a/scripts/whitelist.py +++ b/scripts/whitelist.py @@ -246,4 +246,3 @@ recover_verify_keys_from_message # unused function (aea/helpers/acn/agent_record.py:49) not_before_string # unused property (aea/helpers/base.py:724) not_after_string # unused property (aea/helpers/base.py:729) - From 591d68205c64b30e815d1c4c14499ecbb6d4e70a Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Wed, 30 Dec 2020 22:16:16 +0000 Subject: [PATCH 098/204] Address mypy comment --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index 0cecc2efc4..443ef8ac6d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -784,6 +784,7 @@ def _make_stub_connection(input_file_path: str, output_file_path: str): def _process_cert(key: Crypto, cert: CertRequest): # must match aea/cli/issue_certificates.py:_process_certificate + assert cert.public_key is not None message = cert.get_message(cert.public_key) signature = key.sign_message(message).encode("ascii").hex() Path(cert.save_path).write_bytes(signature.encode("ascii")) From e0ecb09df2e48e76a120733302f03035db73cbdb Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Thu, 31 Dec 2020 08:31:30 +0000 Subject: [PATCH 099/204] Fix 5 of 8 failing tests --- tests/data/dummy_aea/aea-config.yaml | 4 +++- tests/test_cli/test_issue_certificates.py | 4 ++-- tests/test_helpers/test_base.py | 2 +- .../test_connections/test_p2p_libp2p/test_aea_cli.py | 4 ++++ 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/data/dummy_aea/aea-config.yaml b/tests/data/dummy_aea/aea-config.yaml index 3e6a5bc0f5..bc25c0d544 100644 --- a/tests/data/dummy_aea/aea-config.yaml +++ b/tests/data/dummy_aea/aea-config.yaml @@ -21,14 +21,16 @@ skills: - dummy_author/dummy:0.1.0 - fetchai/error:0.10.0 default_connection: fetchai/local:0.13.0 -default_ledger: cosmos +default_ledger: fetchai logging_config: disable_existing_loggers: false version: 1 private_key_paths: + fetchai: fetch_private_key.txt cosmos: cosmos_private_key.txt ethereum: ethereum_private_key.txt connection_private_key_paths: + fetchai: fetch_private_key.txt cosmos: cosmos_private_key.txt ethereum: ethereum_private_key.txt registry_path: ../../packages diff --git a/tests/test_cli/test_issue_certificates.py b/tests/test_cli/test_issue_certificates.py index cb8a0418fc..3e423cf647 100644 --- a/tests/test_cli/test_issue_certificates.py +++ b/tests/test_cli/test_issue_certificates.py @@ -71,8 +71,8 @@ def setup_class(cls): """Set up the class.""" super().setup_class() - cls.expected_path_1 = "path_1" - cls.expected_path_2 = "path_2" + cls.expected_path_1 = os.path.abspath("path_1") + cls.expected_path_2 = os.path.abspath("path_2") cls.cert_id_1 = "cert_id_1" cls.cert_id_2 = "cert_id_2" cls.cert_request_1 = CertRequest( diff --git a/tests/test_helpers/test_base.py b/tests/test_helpers/test_base.py index e55d0f1f66..76727ce8b1 100644 --- a/tests/test_helpers/test_base.py +++ b/tests/test_helpers/test_base.py @@ -510,7 +510,7 @@ def setup_class(cls): cls.expected_ledger_id = "ledger_id" cls.not_before = "2020-01-01" cls.not_after = "2020-01-02" - cls.expected_path = "some/path" + cls.expected_path = os.path.abspath("some/path") cls.cert_request = CertRequest( cls.expected_public_key, cls.expected_identifier, diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py index 44289dffa9..4cb394e9cd 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py @@ -55,6 +55,8 @@ def setup_class(cls): @libp2p_log_on_failure def test_agent(self): """Test with aea.""" + self.generate_private_key() + self.add_private_key() self.add_item("connection", str(P2P_CONNECTION_PUBLIC_ID)) self.run_cli_command("build", cwd=self._get_cwd()) self.set_config("agent.default_connection", str(P2P_CONNECTION_PUBLIC_ID)) @@ -111,6 +113,8 @@ def setup_class(cls): @libp2p_log_on_failure def test_agent(self): """Test with aea.""" + self.generate_private_key() + self.add_private_key() self.add_item("connection", str(P2P_CONNECTION_PUBLIC_ID)) self.run_cli_command("build", cwd=self._get_cwd()) self.nested_set_config( From e4a5f1c2ba289ce4a76671a872c69059223eeb49 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Thu, 31 Dec 2020 08:42:51 +0000 Subject: [PATCH 100/204] Fix hash --- tests/data/dummy_aea/aea-config.yaml | 4 ++-- tests/data/hashes.csv | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/data/dummy_aea/aea-config.yaml b/tests/data/dummy_aea/aea-config.yaml index bc25c0d544..df74b841ea 100644 --- a/tests/data/dummy_aea/aea-config.yaml +++ b/tests/data/dummy_aea/aea-config.yaml @@ -26,12 +26,12 @@ logging_config: disable_existing_loggers: false version: 1 private_key_paths: - fetchai: fetch_private_key.txt cosmos: cosmos_private_key.txt ethereum: ethereum_private_key.txt -connection_private_key_paths: fetchai: fetch_private_key.txt +connection_private_key_paths: cosmos: cosmos_private_key.txt ethereum: ethereum_private_key.txt + fetchai: fetch_private_key.txt registry_path: ../../packages default_routing: {} diff --git a/tests/data/hashes.csv b/tests/data/hashes.csv index 029c823b59..aeed33872b 100644 --- a/tests/data/hashes.csv +++ b/tests/data/hashes.csv @@ -1,4 +1,4 @@ -dummy_author/agents/dummy_aea,QmPWxJVM9xnkRFtB41Wr1Xi2NGKpeRgDXUzbAzA8rferGh +dummy_author/agents/dummy_aea,Qmf2EJ4ajaGn3RakmVTJVPQK66N8C9CinxutcyWeghCNcK dummy_author/skills/dummy_skill,QmSmzcjGNhiz2KpSE6wSi19AeesCVjzupJm1NpXYJVnJuZ fetchai/connections/dummy_connection,Qmep3ive7fVWJi965SSdJv6fdfzAKD1wmP9A1kYRoQjRNn fetchai/contracts/dummy_contract,QmXhMKRF5upco1efaC9oNcesStEMkPWKhBUv6CK9Vngk2S From 3367826fae73b003013618c601a121db18c5a23a Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 31 Dec 2020 10:54:48 +0000 Subject: [PATCH 101/204] fixes and refactorings to align with cli commands --- aea/cli/issue_certificates.py | 20 +++++++++---- aea/cli/run.py | 2 ++ aea/crypto/base.py | 14 +++++++++ aea/crypto/cosmos.py | 23 ++++++++++++--- aea/crypto/ethereum.py | 14 +++++++++ aea/helpers/acn/agent_record.py | 29 ++----------------- aea/helpers/base.py | 3 ++ .../connections/p2p_libp2p/connection.py | 8 +++-- .../connections/p2p_libp2p/connection.yaml | 4 +-- .../p2p_libp2p_client/connection.py | 10 ++++--- .../p2p_libp2p_client/connection.yaml | 6 ++-- packages/hashes.csv | 4 +-- scripts/whitelist.py | 1 - tests/test_cli/test_fingerprint.py | 4 +-- .../test_p2p_libp2p/test_aea_cli.py | 24 +++------------ .../test_p2p_libp2p_client/test_aea_cli.py | 2 -- 16 files changed, 94 insertions(+), 74 deletions(-) diff --git a/aea/cli/issue_certificates.py b/aea/cli/issue_certificates.py index 6c26d9c8cd..728edf9e1c 100644 --- a/aea/cli/issue_certificates.py +++ b/aea/cli/issue_certificates.py @@ -18,6 +18,7 @@ # ------------------------------------------------------------------------------ """Implementation of the 'aea issue_certificates' subcommand.""" +import os from pathlib import Path from typing import cast @@ -33,7 +34,7 @@ from aea.configurations.constants import CONNECTION from aea.crypto.registries import crypto_registry from aea.exceptions import enforce -from aea.helpers.base import CertRequest +from aea.helpers.base import CertRequest, ensure_dir @click.command() @@ -42,14 +43,20 @@ def issue_certificates(click_context): """Issue certificates for connections that require them.""" ctx = cast(Context, click_context.obj) + issue_certificates_(ctx) + +def issue_certificates_(ctx): + """Issue certificates for connections that require them.""" for connection_id in ctx.agent_config.connections: _process_connection(ctx, connection_id) click.echo("All certificates have been issued.") -def _process_certificate(ctx: Context, cert_request: CertRequest): +def _process_certificate( + ctx: Context, cert_request: CertRequest, connection_id: PublicId +): """Process a single certificate request.""" ledger_id = cert_request.ledger_id output_path = cert_request.save_path @@ -60,7 +67,7 @@ def _process_certificate(ctx: Context, cert_request: CertRequest): ) if connection_private_key_path is None: raise ClickException( - f"Cannot find connection private key with id '{key_identifier}'" + f"Cannot find connection private key with id '{key_identifier}'. Connection '{connection_id}' requires this. Please use `aea generate-key {key_identifier} connection_{key_identifier}_private_key.txt` and `aea add-key {key_identifier} connection_{key_identifier}_private_key.txt --connection` to add a connection private key with id '{key_identifier}'." ) connection_crypto = crypto_registry.make( key_identifier, private_key_path=connection_private_key_path @@ -74,11 +81,14 @@ def _process_certificate(ctx: Context, cert_request: CertRequest): ) crypto_private_key_path = ctx.agent_config.private_key_paths.read(ledger_id) if crypto_private_key_path is None: - raise ClickException(f"Cannot find private key with id '{ledger_id}'") + raise ClickException( + f"Cannot find private key with id '{ledger_id}'. Please use `aea generate-key {key_identifier}` and `aea add-key {key_identifier}` to add a private key with id '{key_identifier}'." + ) crypto = crypto_registry.make(ledger_id, private_key_path=crypto_private_key_path) message = cert_request.get_message(public_key) signature = crypto.sign_message(message).encode("ascii").hex() click.echo(f"Generated signature: '{signature}'") + ensure_dir(os.path.dirname(os.path.join(ctx.cwd, output_path))) Path(output_path).write_bytes(signature.encode("ascii")) @@ -97,7 +107,7 @@ def _process_connection(ctx: Context, connection_id: PublicId): click.echo( f"Issuing certificate '{cert_request.identifier}' for connection {connection_config.public_id}..." ) - _process_certificate(ctx, cert_request) + _process_certificate(ctx, cert_request, connection_id) click.echo( f"Dumped certificate '{cert_request.identifier}' in '{cert_request.save_path}' for connection {connection_id}." ) diff --git a/aea/cli/run.py b/aea/cli/run.py index bd1e7ace68..6d959a44b2 100644 --- a/aea/cli/run.py +++ b/aea/cli/run.py @@ -27,6 +27,7 @@ from aea.aea import AEA from aea.aea_builder import AEABuilder, DEFAULT_ENV_DOTFILE from aea.cli.install import do_install +from aea.cli.issue_certificates import issue_certificates_ from aea.cli.utils.click_utils import ConnectionsOption from aea.cli.utils.constants import AEA_LOGO, REQUIREMENTS from aea.cli.utils.context import Context @@ -84,6 +85,7 @@ def run( ): """Run the agent.""" ctx = cast(Context, click_context.obj) + issue_certificates_(ctx) profiling = int(profiling) if profiling > 0: with _profiling_context(period=profiling): diff --git a/aea/crypto/base.py b/aea/crypto/base.py index b15313009b..31935093e6 100644 --- a/aea/crypto/base.py +++ b/aea/crypto/base.py @@ -203,6 +203,20 @@ def recover_message( :return: the recovered addresses """ + @classmethod + @abstractmethod + def recover_verifying_keys_from_message( + cls, message: bytes, signature: str, is_deprecated_mode: bool = False + ) -> Tuple[str, ...]: + """ + Get the public key used to produce the `signature` of the `message` + + :param message: raw bytes used to produce signature + :param signature: signature of the message + :param is_deprecated_mode: if the deprecated signing was used + :return: the recovered public keys + """ + @staticmethod @abstractmethod def get_hash(message: bytes) -> str: diff --git a/aea/crypto/cosmos.py b/aea/crypto/cosmos.py index b919dcdb57..e2926c7463 100644 --- a/aea/crypto/cosmos.py +++ b/aea/crypto/cosmos.py @@ -150,6 +150,24 @@ def recover_message( :param is_deprecated_mode: if the deprecated signing was used :return: the recovered addresses """ + public_keys = cls.recover_verifying_keys_from_message(message, signature) + addresses = [ + cls.get_address_from_public_key(public_key) for public_key in public_keys + ] + return tuple(addresses) + + @classmethod + def recover_verifying_keys_from_message( + cls, message: bytes, signature: str, is_deprecated_mode: bool = False + ) -> Tuple[str, ...]: + """ + Get the public key used to produce the `signature` of the `message` + + :param message: raw bytes used to produce signature + :param signature: signature of the message + :param is_deprecated_mode: if the deprecated signing was used + :return: the recovered public keys + """ signature_b64 = base64.b64decode(signature) verifying_keys = VerifyingKey.from_public_key_recovery( signature_b64, message, SECP256k1, hashfunc=hashlib.sha256, @@ -158,10 +176,7 @@ def recover_message( verifying_key.to_string("compressed").hex() for verifying_key in verifying_keys ] - addresses = [ - cls.get_address_from_public_key(public_key) for public_key in public_keys - ] - return tuple(addresses) + return tuple(public_keys) @staticmethod def get_hash(message: bytes) -> str: diff --git a/aea/crypto/ethereum.py b/aea/crypto/ethereum.py index f51c741ba7..595c55b8b4 100644 --- a/aea/crypto/ethereum.py +++ b/aea/crypto/ethereum.py @@ -365,6 +365,20 @@ def recover_message( ) return (address,) + @classmethod + def recover_verifying_keys_from_message( + cls, message: bytes, signature: str, is_deprecated_mode: bool = False + ) -> Tuple[str, ...]: + """ + Get the public key used to produce the `signature` of the `message` + + :param message: raw bytes used to produce signature + :param signature: signature of the message + :param is_deprecated_mode: if the deprecated signing was used + :return: the recovered public keys + """ + raise NotImplementedError + @staticmethod def get_hash(message: bytes) -> str: """ diff --git a/aea/helpers/acn/agent_record.py b/aea/helpers/acn/agent_record.py index 5a0a6c2820..db47331660 100644 --- a/aea/helpers/acn/agent_record.py +++ b/aea/helpers/acn/agent_record.py @@ -19,33 +19,9 @@ """This module contains types and helpers for acn Proof-of-Representation.""" -import base64 -from hashlib import sha256 -from typing import List - -from ecdsa import SECP256k1, VerifyingKey - from aea.crypto.fetchai import FetchAIHelper -def recover_verify_keys_from_message(message: bytes, signature: str) -> List[str]: - """ - Get the public key used to produce the `signature` of the `message` - - :param message: raw bytes used to produce signature - :param signature: signature of the message - """ - - signature_b64 = base64.b64decode(signature) - verifying_keys = VerifyingKey.from_public_key_recovery( - signature_b64, message, SECP256k1, hashfunc=sha256, - ) - public_keys = [ - verifying_key.to_string("compressed").hex() for verifying_key in verifying_keys - ] - return public_keys - - class AgentRecord: """Agent Proof-of-Representation to peer""" @@ -116,9 +92,10 @@ def is_valid_for(self, address: str, peer_public_key: str) -> bool: if self._peer_public_key != peer_public_key: print("Wrong peer public key") return False - if self._address != FetchAIHelper.get_address_from_public_key(self._public_key): + recovered_address = FetchAIHelper.get_address_from_public_key(self._public_key) + if self._address != recovered_address: print( - f"Wrong address '{self._address}' and public key '{FetchAIHelper.get_address_from_public_key(self._public_key)}'" + f"Wrong address '{self._address}' and public key '{recovered_address}'" ) return False if self._address not in FetchAIHelper.recover_message( diff --git a/aea/helpers/base.py b/aea/helpers/base.py index 4706ce8f4f..35cd67a2e7 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -749,6 +749,9 @@ def save_path(self) -> Path: def get_message(self, public_key: str) -> bytes: # pylint: disable=no-self-use """Get the message to sign.""" message = public_key.encode("ascii") + # + self.identifier.encode("ascii") # noqa: E800 + # + self.not_before_string.encode("ascii") # noqa: E800 + # + self.not_after_string.encode("ascii") # noqa: E800 return message @property diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index be9578433e..e52f0d79fd 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -36,7 +36,7 @@ from aea.crypto.base import Crypto from aea.crypto.fetchai import FetchAIHelper from aea.exceptions import enforce -from aea.helpers.acn.agent_record import AgentRecord, recover_verify_keys_from_message +from aea.helpers.acn.agent_record import AgentRecord from aea.helpers.acn.uri import Uri from aea.helpers.base import CertRequest from aea.helpers.multiaddr.base import MultiAddr @@ -439,7 +439,7 @@ def _get_signature_from_cert_request( signature = bytes.fromhex(Path(cert.save_path).read_bytes().decode("ascii")).decode( "ascii" ) - public_keys = recover_verify_keys_from_message( + public_keys = FetchAIHelper.recover_verifying_keys_from_message( cert.get_message(node_public_key), signature ) addresses = [ @@ -553,7 +553,9 @@ def __init__(self, **kwargs): cert_requests = self.configuration.cert_requests if cert_requests is None or len(cert_requests) != 1: - raise ValueError("cert_requests field must be set") + raise ValueError( + "cert_requests field must be set and contain exactly one entry!" + ) if not Path(cert_requests[0].save_path).is_file(): raise Exception( "cert_request 'save_path' field is not file. Please ensure that 'issue-certificates' command is called beforehand" diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index 80f83f5975..691b6d6f7d 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -16,7 +16,7 @@ fingerprint: aea/pipe_unix.go: QmSzbAexrSUSA9ZE6VT6MmcWF5MhEcmceQjAbnfRoK7Ruv aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK check_dependencies.py: QmcZATMqEThMQBdsud1AwJEYtKq86VfPSXhnfdE7TSWcsc - connection.py: QmUBr2UXkN8RYWxKppM6zWAoqGGd5qYpuwY56VFqssohRc + connection.py: QmQRxza9EKQwWD6Uaq4NXKo4DdZ8QatNNLTMv8mVe4daY4 dht/dhtclient/dhtclient.go: QmZCgMowWw4ueCSsyYY12om3RMQA3Ydp98ti65Eqa5ccuz dht/dhtclient/dhtclient_test.go: QmbTvSmNU1uqjLQo9LdD7zWmDcLuzEacDeg7AG4VXhmriV dht/dhtclient/options.go: QmW3UjASUP6zGwK8b61hHQhVoLqBxQ13vZ1gcHuTmdrsDf @@ -55,7 +55,7 @@ cert_requests: not_after: '2022-01-01' not_before: '2021-01-01' public_key: fetchai - save_path: /source/conn_cert.txt + save_path: /Users/davidminarsch/work/agents-aea/source/conn_cert.txt excluded_protocols: [] restricted_to_protocols: [] dependencies: {} diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.py b/packages/fetchai/connections/p2p_libp2p_client/connection.py index 19db28c590..323ddcc49c 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.py +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.py @@ -33,7 +33,7 @@ from aea.crypto.fetchai import FetchAIHelper from aea.crypto.registries import make_crypto from aea.exceptions import enforce -from aea.helpers.acn.agent_record import AgentRecord, recover_verify_keys_from_message +from aea.helpers.acn.agent_record import AgentRecord from aea.helpers.acn.uri import Uri from aea.helpers.base import CertRequest from aea.mail.base import Envelope @@ -62,7 +62,7 @@ def _get_signatures_from_cert_request( - certs: List[CertRequest], public_keys: List[str], agent_address: str + certs: List[CertRequest], public_keys: Tuple[str, ...], agent_address: str ) -> Tuple[List[str], str]: signatures: List[str] = [] verify_key = "" @@ -73,7 +73,7 @@ def _get_signatures_from_cert_request( Path(cert.save_path).read_bytes().decode("ascii") ).decode("ascii") if verify_key == "": - public_keys = recover_verify_keys_from_message( + public_keys = FetchAIHelper.recover_verifying_keys_from_message( cert.get_message(public_key), signature ) addresses = [ @@ -128,7 +128,9 @@ def __init__(self, **kwargs): cert_requests = self.configuration.cert_requests if cert_requests is None or len(cert_requests) != len(nodes): - raise ValueError("cert_requests field must be set") + raise ValueError( + "cert_requests field must be set and contain exactly as many entries as 'nodes'!" + ) for cert_request in cert_requests: if not Path(cert_request.save_path).is_file(): raise Exception( diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml index 40e1209fbc..5620b2b0dd 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml @@ -12,7 +12,7 @@ fingerprint: __init__.py: QmT1FEHkPGMHV5oiVEfQHHr25N2qdZxydSNRJabJvYiTgf acn_message.proto: QmSwpCP72J94kt2MaLSPQdt8puqtLMVPp3Migwp4xRwadZ acn_message_pb2.py: QmbVqUUCHyQpfx7dHvW5e5j3RGcjNYqsDjuPV8eNcca2Nf - connection.py: Qmcxx2GAhfafLvAuAae727k4C273oLdbr6CdxzZHDFCDWB + connection.py: QmX5BNMazDSxPdSNuGTwYTucrgb7Bpjuk1Q8eGTX65XiwB fingerprint_ignore_patterns: [] connections: [] protocols: [] @@ -30,13 +30,13 @@ cert_requests: not_after: '2022-01-01' not_before: '2021-01-01' public_key: 0217a59bd805c310aca4febe0e99ce22ee3712ae085dc1e5630430b1e15a584bb7 - save_path: /source/acn_fetch_ai_11000.txt + save_path: /Users/davidminarsch/work/agents-aea/source/acn_fetch_ai_11000.txt - identifier: acn ledger_id: fetchai not_after: '2022-01-01' not_before: '2021-01-01' public_key: 03fa7cfae1037cba5218f0f5743802eced8de3247c55ecebaae46c7d3679e3f91d - save_path: /source/acn_fetch_ai_11001.txt + save_path: /Users/davidminarsch/work/agents-aea/source/acn_fetch_ai_11001.txt excluded_protocols: [] restricted_to_protocols: [] dependencies: {} diff --git a/packages/hashes.csv b/packages/hashes.csv index 870945e310..e7c14d183b 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,8 +34,8 @@ fetchai/connections/http_server,QmQ8hNfEts6th9zfnocYHzyYw4dXkDNDj8zoKtov1wB49n fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 -fetchai/connections/p2p_libp2p,QmbqgF14cRh37m4QyJ7NW4qDma1sxcayuAtA78Sr2Zwr58 -fetchai/connections/p2p_libp2p_client,QmPyoVeDiUzUsx8cfKPxgbyxNULaZX89JBwnE6TZ33ciuN +fetchai/connections/p2p_libp2p,QmUGmrpDj6sSXzcaY11LT2dqcqBVze9PWRNZwATRQ2rYNy +fetchai/connections/p2p_libp2p_client,QmR7LZUpAF1cdomcAavvfN1eyR6HPfy1JcrLcjvs32UxET fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T fetchai/connections/scaffold,QmaCupER4vedakPPLzn2Rw9YHPmqSPaZh5xnqi6xVxQHvn diff --git a/scripts/whitelist.py b/scripts/whitelist.py index f5db9340d9..e84f958863 100644 --- a/scripts/whitelist.py +++ b/scripts/whitelist.py @@ -243,6 +243,5 @@ AgentRecord # unused class (aea/helpers/acn/agent_record.py:49) is_valid_for # unused method (aea/helpers/acn/agent_record.py:96) Uri # unused class (aea/helpers/acn/uri.py:26) -recover_verify_keys_from_message # unused function (aea/helpers/acn/agent_record.py:49) not_before_string # unused property (aea/helpers/base.py:724) not_after_string # unused property (aea/helpers/base.py:729) diff --git a/tests/test_cli/test_fingerprint.py b/tests/test_cli/test_fingerprint.py index 325db9bd29..db60c18c4d 100644 --- a/tests/test_cli/test_fingerprint.py +++ b/tests/test_cli/test_fingerprint.py @@ -93,14 +93,14 @@ def test_by_path_exceptions(self, *mocks): """Test fingerprint by_path works raises exceptions.""" with pytest.raises( ClickException, - match=f"No package config file found in `.*`. Incorrect directory?", + match="No package config file found in `.*`. Incorrect directory?", ): with mock.patch("os.listdir", return_value=[]): self._run_fingerprint_by_path() with pytest.raises( ClickException, - match=f"Too many config files in the directory, only one has to present!", + match="Too many config files in the directory, only one has to present!", ): with mock.patch( "os.listdir", diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py index 4cb394e9cd..5827338284 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py @@ -21,8 +21,6 @@ import os -from aea.configurations.constants import DEFAULT_LEDGER -from aea.crypto.registries import make_crypto from aea.test_tools.test_cases import AEATestCaseEmpty from packages.fetchai.connections.p2p_libp2p.connection import ( @@ -47,9 +45,6 @@ def setup_class(cls): """Set the test up""" super(TestP2PLibp2pConnectionAEARunningDefaultConfigNode, cls).setup_class() cls.conn_key_file = os.path.join(os.path.abspath(os.getcwd()), "./conn_key.txt") - with open(cls.conn_key_file, "wb") as f: - key = make_crypto(DEFAULT_LEDGER) - key.dump(f) cls.log_files = [] @libp2p_log_on_failure @@ -57,12 +52,11 @@ def test_agent(self): """Test with aea.""" self.generate_private_key() self.add_private_key() + self.generate_private_key(private_key_file=self.conn_key_file) + self.add_private_key(private_key_filepath=self.conn_key_file, connection=True) self.add_item("connection", str(P2P_CONNECTION_PUBLIC_ID)) self.run_cli_command("build", cwd=self._get_cwd()) self.set_config("agent.default_connection", str(P2P_CONNECTION_PUBLIC_ID)) - self.nested_set_config( - "agent.connection_private_key_paths", {DEFAULT_LEDGER: self.conn_key_file} - ) # for logging config_path = "vendor.fetchai.connections.p2p_libp2p.config" @@ -71,9 +65,6 @@ def test_agent(self): self.set_config("{}.log_file".format(config_path), log_file) TestP2PLibp2pConnectionAEARunningDefaultConfigNode.log_files.append(log_file) - # generate certificates for connection - self.run_cli_command("issue-certificates", cwd=self._get_cwd()) - process = self.run_agent() is_running = self.is_running(process, timeout=LIBP2P_LAUNCH_TIMEOUT) assert is_running, "AEA not running within timeout!" @@ -105,9 +96,6 @@ def setup_class(cls): """Set the test up""" super(TestP2PLibp2pConnectionAEARunningFullNode, cls).setup_class() cls.conn_key_file = os.path.join(os.path.abspath(os.getcwd()), "./conn_key.txt") - with open(cls.conn_key_file, "wb") as f: - key = make_crypto(DEFAULT_LEDGER) - key.dump(f) cls.log_files = [] @libp2p_log_on_failure @@ -115,11 +103,10 @@ def test_agent(self): """Test with aea.""" self.generate_private_key() self.add_private_key() + self.generate_private_key(private_key_file=self.conn_key_file) + self.add_private_key(private_key_filepath=self.conn_key_file, connection=True) self.add_item("connection", str(P2P_CONNECTION_PUBLIC_ID)) self.run_cli_command("build", cwd=self._get_cwd()) - self.nested_set_config( - "agent.connection_private_key_paths", {DEFAULT_LEDGER: self.conn_key_file} - ) # setup a full node: with public uri, relay service, and delegate service config_path = "vendor.fetchai.connections.p2p_libp2p.config" @@ -140,9 +127,6 @@ def test_agent(self): self.set_config("{}.log_file".format(config_path), log_file) TestP2PLibp2pConnectionAEARunningFullNode.log_files.append(log_file) - # generate certificates for connection - self.run_cli_command("issue-certificates", cwd=self._get_cwd()) - process = self.run_agent() is_running = self.is_running(process, timeout=LIBP2P_LAUNCH_TIMEOUT) diff --git a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py index 691c4ccf03..daa1952364 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py @@ -96,8 +96,6 @@ def test_connection(self): ], ) - self.run_cli_command("issue-certificates", cwd=self._get_cwd()) - process = self.run_agent() is_running = self.is_running(process, timeout=DEFAULT_LAUNCH_TIMEOUT) assert is_running, "AEA not running within timeout!" From 2e1a3fee22d2ab4bee4e65b449b141c1f90a65fc Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Thu, 31 Dec 2020 11:08:36 +0000 Subject: [PATCH 102/204] Fix remaining tests --- aea/helpers/acn/agent_record.py | 61 ++++++++++++++----- .../connections/p2p_libp2p/connection.py | 45 +++++--------- .../connections/p2p_libp2p/connection.yaml | 2 +- .../p2p_libp2p_client/connection.py | 61 +++++++------------ .../p2p_libp2p_client/connection.yaml | 2 +- packages/hashes.csv | 4 +- tests/test_aea_builder.py | 3 + tests/test_multiplexer.py | 3 + 8 files changed, 95 insertions(+), 86 deletions(-) diff --git a/aea/helpers/acn/agent_record.py b/aea/helpers/acn/agent_record.py index 5a0a6c2820..ca33b53da6 100644 --- a/aea/helpers/acn/agent_record.py +++ b/aea/helpers/acn/agent_record.py @@ -21,11 +21,13 @@ import base64 from hashlib import sha256 -from typing import List +from pathlib import Path +from typing import List, Tuple from ecdsa import SECP256k1, VerifyingKey from aea.crypto.fetchai import FetchAIHelper +from aea.helpers.base import CertRequest def recover_verify_keys_from_message(message: bytes, signature: str) -> List[str]: @@ -45,6 +47,38 @@ def recover_verify_keys_from_message(message: bytes, signature: str) -> List[str ] return public_keys +def signature_from_cert_request( + cert: CertRequest, message: str, signer_address: str +) -> Tuple[str, str]: + """ + Get signature and its verifying key from a CertRequest and its message. + Note: must match aea/cli/issue_certificates.py:_process_certificate + + :param cert: cert request containing the signature + :param message: the message used to generate signature + :param signer_address: the address of the signer + :return: the signature and the verifying public key + """ + + signature = bytes.fromhex(Path(cert.save_path).read_bytes().decode("ascii")).decode( + "ascii" + ) + public_keys = recover_verify_keys_from_message( + cert.get_message(message), signature + ) + if len(public_keys) == 0: + raise Exception("Malformed signature") + addresses = [ + FetchAIHelper.get_address_from_public_key(public_key) + for public_key in public_keys + ] + try: + verify_key = public_keys[addresses.index(signer_address)] + except ValueError: + raise Exception("Not signed by agent") + return signature, verify_key + + class AgentRecord: """Agent Proof-of-Representation to peer""" @@ -101,29 +135,26 @@ def __str__(self): """Get string representation.""" return f"(address={self.address}, public_key={self.public_key}, peer_public_key={self.peer_public_key}, signature={self.signature})" - def is_valid_for(self, address: str, peer_public_key: str) -> bool: + def check_validity(self, address: str, peer_public_key: str) -> None: """ - Check if the agent record is valid for `address` and `peer_public_key` + Check if the agent record is valid for `address` and `peer_public_key`. + Raises an Exception if invalid. :param address: the expected agent address concerned by the record :param peer_public_key: the expected representative peer public key - :return: True if record is valid """ if self._address != address: - print("Wrong address") - return False + raise Exception( + "Proof-of-representation is not generated for the intended agent" + ) if self._peer_public_key != peer_public_key: - print("Wrong peer public key") - return False - if self._address != FetchAIHelper.get_address_from_public_key(self._public_key): - print( - f"Wrong address '{self._address}' and public key '{FetchAIHelper.get_address_from_public_key(self._public_key)}'" + raise Exception( + "Proof-of-representation is not generated for intended peer" ) - return False + if self._address != FetchAIHelper.get_address_from_public_key(self._public_key): + raise Exception("Agent address and public key doesn't match") if self._address not in FetchAIHelper.recover_message( self._peer_public_key.encode("utf-8"), self._signature ): - print("Wrong signature") - return False - return True + raise Exception("Invalid signature") diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index be9578433e..1d3a4fee0e 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -28,17 +28,15 @@ from ipaddress import ip_address from pathlib import Path from socket import gethostbyname -from typing import IO, List, Optional, Sequence, Tuple, cast +from typing import IO, List, Optional, Sequence, cast from aea.configurations.base import PublicId from aea.configurations.constants import DEFAULT_LEDGER from aea.connections.base import Connection, ConnectionStates from aea.crypto.base import Crypto -from aea.crypto.fetchai import FetchAIHelper from aea.exceptions import enforce -from aea.helpers.acn.agent_record import AgentRecord, recover_verify_keys_from_message +from aea.helpers.acn.agent_record import AgentRecord, signature_from_cert_request from aea.helpers.acn.uri import Uri -from aea.helpers.base import CertRequest from aea.helpers.multiaddr.base import MultiAddr from aea.helpers.pipe import IPCChannel, make_ipc_channel from aea.mail.base import Envelope @@ -432,24 +430,6 @@ def stop(self) -> None: os.remove(LIBP2P_NODE_ENV_FILE) -def _get_signature_from_cert_request( - cert: CertRequest, node_public_key: str, agent_address: str -) -> Tuple[str, str]: - # must match aea/cli/issue_certificates.py:_process_certificate - signature = bytes.fromhex(Path(cert.save_path).read_bytes().decode("ascii")).decode( - "ascii" - ) - public_keys = recover_verify_keys_from_message( - cert.get_message(node_public_key), signature - ) - addresses = [ - FetchAIHelper.get_address_from_public_key(public_key) - for public_key in public_keys - ] - verify_key = public_keys[addresses.index(agent_address)] - return signature, verify_key - - class P2PLibp2pConnection(Connection): """A libp2p p2p node connection.""" @@ -492,7 +472,8 @@ def __init__(self, **kwargs): key = self.crypto_store.crypto_objects[ledger_id] else: raise ValueError( - f"Couldn't find connection key for {str(ledger_id)} in connections keys" + f"Couldn't find connection key for {str(ledger_id)} in connections keys. " + "Please ensure agent private key is added" ) uri = None @@ -556,12 +537,16 @@ def __init__(self, **kwargs): raise ValueError("cert_requests field must be set") if not Path(cert_requests[0].save_path).is_file(): raise Exception( - "cert_request 'save_path' field is not file. Please ensure that 'issue-certificates' command is called beforehand" + f"cert_request 'save_path' field {cert_requests[0].save_path} is not a file. " + "Please ensure that 'issue-certificates' command is called beforehand" + ) + try: + signature, agent_public_key = signature_from_cert_request( + cert_requests[0], key.public_key, self.address ) + except Exception as e: + raise ValueError(f"Incorrect certificate from file {cert_requests[0].save_path} : {str(e)}") - signature, agent_public_key = _get_signature_from_cert_request( - cert_requests[0], key.public_key, self.address - ) record = AgentRecord( self.address, agent_public_key, @@ -569,8 +554,10 @@ def __init__(self, **kwargs): signature, POR_DEFAULT_SERVICE_ID, ) - if not record.is_valid_for(self.address, key.public_key): - raise ValueError("Invalid Proof-of-Representation {}".format(str(record))) + try: + record.check_validity(self.address, key.public_key) + except Exception as e: + raise ValueError("Invalid Proof-of-Representation: {}".format(str(e))) # libp2p local node self.logger.debug("Public key used by libp2p node: {}".format(key.public_key)) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index 80f83f5975..e279b48e52 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -16,7 +16,7 @@ fingerprint: aea/pipe_unix.go: QmSzbAexrSUSA9ZE6VT6MmcWF5MhEcmceQjAbnfRoK7Ruv aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK check_dependencies.py: QmcZATMqEThMQBdsud1AwJEYtKq86VfPSXhnfdE7TSWcsc - connection.py: QmUBr2UXkN8RYWxKppM6zWAoqGGd5qYpuwY56VFqssohRc + connection.py: Qmeb146vv5Y8HPfbLqAnmZC8svPDJ3fAWdxchPQnFn18JX dht/dhtclient/dhtclient.go: QmZCgMowWw4ueCSsyYY12om3RMQA3Ydp98ti65Eqa5ccuz dht/dhtclient/dhtclient_test.go: QmbTvSmNU1uqjLQo9LdD7zWmDcLuzEacDeg7AG4VXhmriV dht/dhtclient/options.go: QmW3UjASUP6zGwK8b61hHQhVoLqBxQ13vZ1gcHuTmdrsDf diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.py b/packages/fetchai/connections/p2p_libp2p_client/connection.py index 19db28c590..93305f78b9 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.py +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.py @@ -25,17 +25,15 @@ import struct from asyncio import CancelledError from pathlib import Path -from typing import List, Optional, Tuple, Union, cast +from typing import List, Optional, Union, cast from aea.configurations.base import PublicId from aea.configurations.constants import DEFAULT_LEDGER from aea.connections.base import Connection, ConnectionStates -from aea.crypto.fetchai import FetchAIHelper from aea.crypto.registries import make_crypto from aea.exceptions import enforce -from aea.helpers.acn.agent_record import AgentRecord, recover_verify_keys_from_message +from aea.helpers.acn.agent_record import AgentRecord, signature_from_cert_request from aea.helpers.acn.uri import Uri -from aea.helpers.base import CertRequest from aea.mail.base import Envelope from packages.fetchai.connections.p2p_libp2p_client.acn_message_pb2 import AcnMessage @@ -61,30 +59,6 @@ ACN_CURRENT_VERSION = "0.1.0" -def _get_signatures_from_cert_request( - certs: List[CertRequest], public_keys: List[str], agent_address: str -) -> Tuple[List[str], str]: - signatures: List[str] = [] - verify_key = "" - for i, cert in enumerate(certs): - cert = certs[i] - public_key = public_keys[i] - signature = bytes.fromhex( - Path(cert.save_path).read_bytes().decode("ascii") - ).decode("ascii") - if verify_key == "": - public_keys = recover_verify_keys_from_message( - cert.get_message(public_key), signature - ) - addresses = [ - FetchAIHelper.get_address_from_public_key(public_key) - for public_key in public_keys - ] - verify_key = public_keys[addresses.index(agent_address)] - signatures.append(signature) - return signatures, verify_key - - class P2PLibp2pClientConnection(Connection): """ A libp2p client connection. @@ -132,11 +106,10 @@ def __init__(self, **kwargs): for cert_request in cert_requests: if not Path(cert_request.save_path).is_file(): raise Exception( - "cert_request 'save_path' field is not a file. Please ensure that 'issue-certificates' command is called beforehand" + "cert_request 'save_path' field is not a file. " + "Please ensure that 'issue-certificates' command is called beforehand" ) - # verify keys are correct - # TOFIX(): we cannot use store as the key will be used for TLS tcp connection # also, as of now all the connections share the same key if key_file is not None: @@ -153,9 +126,16 @@ def __init__(self, **kwargs): # delegates PoRs self.delegate_pors = [] - signatures, agent_public_key = _get_signatures_from_cert_request( - cert_requests, nodes_public_keys, self.address - ) + agent_public_key = "" + signatures: List[str] = [] + for i, cert in enumerate(cert_requests): + try: + signature, agent_public_key = signature_from_cert_request(cert, nodes_public_keys[i], self.address) + signatures.append(signature) + except Exception as e: + raise ValueError(f"Incorrect certificate from file {cert.save_path} " + f"for node {nodes_uris[i]} : {str(e)}") + records: List[AgentRecord] = [] for i, signature in enumerate(signatures): records.append( @@ -173,10 +153,15 @@ def __init__(self, **kwargs): record = next( record for record in records if record.peer_public_key == public_key ) - enforce( - True or record.is_valid_for(self.address, public_key), - f"Invalid Proof-of-Representation for node {uri}", - ) + try: + record.check_validity(self.address, public_key) + except Exception as e: + raise ValueError( + "Invalid Proof-of-Representation for node {}: {}".format( + str(uri), str(e) + ) + ) + self.delegate_pors.append(record) # select a delegate diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml index 40e1209fbc..5ec1c3069c 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml @@ -12,7 +12,7 @@ fingerprint: __init__.py: QmT1FEHkPGMHV5oiVEfQHHr25N2qdZxydSNRJabJvYiTgf acn_message.proto: QmSwpCP72J94kt2MaLSPQdt8puqtLMVPp3Migwp4xRwadZ acn_message_pb2.py: QmbVqUUCHyQpfx7dHvW5e5j3RGcjNYqsDjuPV8eNcca2Nf - connection.py: Qmcxx2GAhfafLvAuAae727k4C273oLdbr6CdxzZHDFCDWB + connection.py: QmXxvFxKuhrLRCyMMmNXtpwyuZmRdF8sFHhxSMH1f1hVCp fingerprint_ignore_patterns: [] connections: [] protocols: [] diff --git a/packages/hashes.csv b/packages/hashes.csv index 870945e310..6e3b1baf9d 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,8 +34,8 @@ fetchai/connections/http_server,QmQ8hNfEts6th9zfnocYHzyYw4dXkDNDj8zoKtov1wB49n fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 -fetchai/connections/p2p_libp2p,QmbqgF14cRh37m4QyJ7NW4qDma1sxcayuAtA78Sr2Zwr58 -fetchai/connections/p2p_libp2p_client,QmPyoVeDiUzUsx8cfKPxgbyxNULaZX89JBwnE6TZ33ciuN +fetchai/connections/p2p_libp2p,QmSGgWErrqPh4ebEe9MqCJKJ8cTwtzqMHGLupS8oegfRmf +fetchai/connections/p2p_libp2p_client,QmQHBzo4APdFwHk8skEhUBFL9aTEm9yepkW2CdkscQ4zZ5 fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T fetchai/connections/scaffold,QmaCupER4vedakPPLzn2Rw9YHPmqSPaZh5xnqi6xVxQHvn diff --git a/tests/test_aea_builder.py b/tests/test_aea_builder.py index 7dd2a0e419..9b365785e4 100644 --- a/tests/test_aea_builder.py +++ b/tests/test_aea_builder.py @@ -639,12 +639,14 @@ def test_from_project(self): self.new_handler_args = {"handler_arg_1": 42} self.new_model_args = {"model_arg_1": 42} self._add_dummy_skill_config() + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) builder = AEABuilder.from_aea_project(Path(self._get_cwd())) with cd(self._get_cwd()): builder.call_all_build_entrypoints() aea = builder.build() + dummy_skill = aea.resources.get_skill(DUMMY_SKILL_PUBLIC_ID) dummy_behaviour = dummy_skill.behaviours["dummy"] assert dummy_behaviour.config == {"behaviour_arg_1": 42, "behaviour_arg_2": "2"} @@ -679,6 +681,7 @@ def _add_dummy_skill_config(self): def test_from_project(self): """Test builder set from project dir.""" self._add_dummy_skill_config() + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) builder = AEABuilder.from_aea_project(Path(self._get_cwd())) with cd(self._get_cwd()): builder.call_all_build_entrypoints() diff --git a/tests/test_multiplexer.py b/tests/test_multiplexer.py index 0fd286b028..bb2fb14b93 100644 --- a/tests/test_multiplexer.py +++ b/tests/test_multiplexer.py @@ -764,6 +764,9 @@ def test_multiplexer_disconnected_on_early_interruption(self): result = self.runner.invoke(cli, [*CLI_LOG_OPTION, "build"]) assert result.exit_code == 0, result.stdout_bytes + result = self.runner.invoke(cli, [*CLI_LOG_OPTION, "issue-certificates"]) + assert result.exit_code == 0, result.stdout_bytes + self.proc = PexpectWrapper( # nosec [sys.executable, "-m", "aea.cli", "-v", "DEBUG", "run"], env=os.environ, From dd32ae24a10b5692ea63162ba0f471d93583abf5 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Thu, 31 Dec 2020 12:20:16 +0000 Subject: [PATCH 103/204] Address ci comments --- aea/helpers/acn/agent_record.py | 16 +++++----- .../connections/p2p_libp2p/connection.py | 4 ++- .../connections/p2p_libp2p/connection.yaml | 4 +-- .../p2p_libp2p_client/connection.py | 10 +++++-- .../p2p_libp2p_client/connection.yaml | 6 ++-- packages/hashes.csv | 4 +-- scripts/whitelist.py | 3 +- tests/test_aea_builder.py | 1 - tests/test_multiplexer.py | 29 ++++++++++++++----- 9 files changed, 50 insertions(+), 27 deletions(-) diff --git a/aea/helpers/acn/agent_record.py b/aea/helpers/acn/agent_record.py index 4678139536..29ef3378ec 100644 --- a/aea/helpers/acn/agent_record.py +++ b/aea/helpers/acn/agent_record.py @@ -47,25 +47,25 @@ def recover_verify_keys_from_message(message: bytes, signature: str) -> List[str ] return public_keys + def signature_from_cert_request( cert: CertRequest, message: str, signer_address: str ) -> Tuple[str, str]: """ Get signature and its verifying key from a CertRequest and its message. - Note: must match aea/cli/issue_certificates.py:_process_certificate + + Must match aea/cli/issue_certificates.py:_process_certificate :param cert: cert request containing the signature :param message: the message used to generate signature :param signer_address: the address of the signer :return: the signature and the verifying public key """ - + signature = bytes.fromhex(Path(cert.save_path).read_bytes().decode("ascii")).decode( "ascii" ) - public_keys = recover_verify_keys_from_message( - cert.get_message(message), signature - ) + public_keys = recover_verify_keys_from_message(cert.get_message(message), signature) if len(public_keys) == 0: raise Exception("Malformed signature") addresses = [ @@ -79,7 +79,6 @@ def signature_from_cert_request( return signature, verify_key - class AgentRecord: """Agent Proof-of-Representation to peer""" @@ -138,6 +137,7 @@ def __str__(self): def check_validity(self, address: str, peer_public_key: str) -> None: """ Check if the agent record is valid for `address` and `peer_public_key`. + Raises an Exception if invalid. :param address: the expected agent address concerned by the record @@ -154,7 +154,9 @@ def check_validity(self, address: str, peer_public_key: str) -> None: ) recovered_address = FetchAIHelper.get_address_from_public_key(self._public_key) if self._address != recovered_address: - raise Exception(f"Agent address {self._address} and public key doesn't match") + raise Exception( + f"Agent address {self._address} and public key doesn't match" + ) if self._address not in FetchAIHelper.recover_message( self._peer_public_key.encode("utf-8"), self._signature ): diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index 5b51804089..d291f46049 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -547,7 +547,9 @@ def __init__(self, **kwargs): cert_requests[0], key.public_key, self.address ) except Exception as e: - raise ValueError(f"Incorrect certificate from file {cert_requests[0].save_path} : {str(e)}") + raise ValueError( + f"Incorrect certificate from file {cert_requests[0].save_path} : {str(e)}" + ) record = AgentRecord( self.address, diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index 4ebcb7c4da..f9a6d66de9 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -16,7 +16,7 @@ fingerprint: aea/pipe_unix.go: QmSzbAexrSUSA9ZE6VT6MmcWF5MhEcmceQjAbnfRoK7Ruv aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK check_dependencies.py: QmcZATMqEThMQBdsud1AwJEYtKq86VfPSXhnfdE7TSWcsc - connection.py: QmWNzmDKYuWLXCE2SKb2kXZC5KzCkpp68qd6KQt7pF5FU6 + connection.py: QmbZaNALbsN4j3YkEWiQzNzfpbuTeoTx6CEdaZZ8FL3S9V dht/dhtclient/dhtclient.go: QmZCgMowWw4ueCSsyYY12om3RMQA3Ydp98ti65Eqa5ccuz dht/dhtclient/dhtclient_test.go: QmbTvSmNU1uqjLQo9LdD7zWmDcLuzEacDeg7AG4VXhmriV dht/dhtclient/options.go: QmW3UjASUP6zGwK8b61hHQhVoLqBxQ13vZ1gcHuTmdrsDf @@ -55,7 +55,7 @@ cert_requests: not_after: '2022-01-01' not_before: '2021-01-01' public_key: fetchai - save_path: /Users/davidminarsch/work/agents-aea/source/conn_cert.txt + save_path: ./conn_cert.txt excluded_protocols: [] restricted_to_protocols: [] dependencies: {} diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.py b/packages/fetchai/connections/p2p_libp2p_client/connection.py index 9c331ec739..39c70dd8c3 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.py +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.py @@ -132,11 +132,15 @@ def __init__(self, **kwargs): signatures: List[str] = [] for i, cert in enumerate(cert_requests): try: - signature, agent_public_key = signature_from_cert_request(cert, nodes_public_keys[i], self.address) + signature, agent_public_key = signature_from_cert_request( + cert, nodes_public_keys[i], self.address + ) signatures.append(signature) except Exception as e: - raise ValueError(f"Incorrect certificate from file {cert.save_path} " - f"for node {nodes_uris[i]} : {str(e)}") + raise ValueError( + f"Incorrect certificate from file {cert.save_path} " + f"for node {nodes_uris[i]} : {str(e)}" + ) records: List[AgentRecord] = [] for i, signature in enumerate(signatures): diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml index 63269ab55f..35f0159c6f 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml @@ -12,7 +12,7 @@ fingerprint: __init__.py: QmT1FEHkPGMHV5oiVEfQHHr25N2qdZxydSNRJabJvYiTgf acn_message.proto: QmSwpCP72J94kt2MaLSPQdt8puqtLMVPp3Migwp4xRwadZ acn_message_pb2.py: QmbVqUUCHyQpfx7dHvW5e5j3RGcjNYqsDjuPV8eNcca2Nf - connection.py: QmdGQkxrBA9pxrziFAmwxNEkNFxT52j2JWpKhq2zwJMw6Z + connection.py: QmVcc6cqVC4bx7te9yTgxzLn2RFWBsrYk6fBniSKoiMo4t fingerprint_ignore_patterns: [] connections: [] protocols: [] @@ -30,13 +30,13 @@ cert_requests: not_after: '2022-01-01' not_before: '2021-01-01' public_key: 0217a59bd805c310aca4febe0e99ce22ee3712ae085dc1e5630430b1e15a584bb7 - save_path: /Users/davidminarsch/work/agents-aea/source/acn_fetch_ai_11000.txt + save_path: ./acn_fetch_ai_11000.txt - identifier: acn ledger_id: fetchai not_after: '2022-01-01' not_before: '2021-01-01' public_key: 03fa7cfae1037cba5218f0f5743802eced8de3247c55ecebaae46c7d3679e3f91d - save_path: /Users/davidminarsch/work/agents-aea/source/acn_fetch_ai_11001.txt + save_path: ./acn_fetch_ai_11001.txt excluded_protocols: [] restricted_to_protocols: [] dependencies: {} diff --git a/packages/hashes.csv b/packages/hashes.csv index b5d80340b8..73fc65c5cd 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,8 +34,8 @@ fetchai/connections/http_server,QmQ8hNfEts6th9zfnocYHzyYw4dXkDNDj8zoKtov1wB49n fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 -fetchai/connections/p2p_libp2p,QmTngK3ScVc9CChHZdWuYerJ3qp9F1MUotWVvWkJkDw6HB -fetchai/connections/p2p_libp2p_client,QmSrwhW46P8VJavcT21oQg48jUovA33pAQrUCUoqmwFNnQ +fetchai/connections/p2p_libp2p,QmSE3Awr7gDJX3S4tHqPyELK4XEPHbR5saQXEhCzYC3Trc +fetchai/connections/p2p_libp2p_client,QmeuqPXvijyU22QGfmAQLHmpUSLLs1h6Pz1H5PX32aqC2q fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T fetchai/connections/scaffold,QmaCupER4vedakPPLzn2Rw9YHPmqSPaZh5xnqi6xVxQHvn diff --git a/scripts/whitelist.py b/scripts/whitelist.py index e84f958863..6e2242e902 100644 --- a/scripts/whitelist.py +++ b/scripts/whitelist.py @@ -241,7 +241,8 @@ _.set_agent_overrides # unused method (aea/manager/manager.py:432) by_path # unused function (aea/cli/fingerprint.py:94) AgentRecord # unused class (aea/helpers/acn/agent_record.py:49) -is_valid_for # unused method (aea/helpers/acn/agent_record.py:96) +check_validity # unused method (aea/helpers/acn/agent_record.py:96) +signature_from_cert_request # unused function (aea/helpers/acn/agent_record.py:51) Uri # unused class (aea/helpers/acn/uri.py:26) not_before_string # unused property (aea/helpers/base.py:724) not_after_string # unused property (aea/helpers/base.py:729) diff --git a/tests/test_aea_builder.py b/tests/test_aea_builder.py index 9b365785e4..d17de766e4 100644 --- a/tests/test_aea_builder.py +++ b/tests/test_aea_builder.py @@ -646,7 +646,6 @@ def test_from_project(self): builder.call_all_build_entrypoints() aea = builder.build() - dummy_skill = aea.resources.get_skill(DUMMY_SKILL_PUBLIC_ID) dummy_behaviour = dummy_skill.behaviours["dummy"] assert dummy_behaviour.config == {"behaviour_arg_1": 42, "behaviour_arg_2": "2"} diff --git a/tests/test_multiplexer.py b/tests/test_multiplexer.py index e9e4171dc0..15c20aa6ee 100644 --- a/tests/test_multiplexer.py +++ b/tests/test_multiplexer.py @@ -18,7 +18,6 @@ # ------------------------------------------------------------------------------ """This module contains the tests for the Multiplexer.""" -from aea.configurations.constants import DEFAULT_LEDGER import asyncio import logging import os @@ -38,6 +37,7 @@ import aea from aea.cli.core import cli from aea.configurations.base import PublicId +from aea.configurations.constants import DEFAULT_LEDGER from aea.connections.base import ConnectionStates from aea.exceptions import AEAEnforceError from aea.helpers.exception_policy import ExceptionPolicyEnum @@ -742,8 +742,8 @@ def setup(self): self.t = tempfile.mkdtemp() shutil.copytree(Path(ROOT_DIR, "packages"), Path(self.t, "packages")) os.chdir(self.t) - self.key_path = os.path.join(self.t,"fetchai_private_key.txt") - self.conn_key_path = os.path.join(self.t,"conn_private_key.txt") + self.key_path = os.path.join(self.t, "fetchai_private_key.txt") + self.conn_key_path = os.path.join(self.t, "conn_private_key.txt") result = self.runner.invoke( cli, [*CLI_LOG_OPTION, "init", "--local", "--author", AUTHOR] @@ -767,16 +767,31 @@ def test_multiplexer_disconnected_on_early_interruption(self): result = self.runner.invoke(cli, [*CLI_LOG_OPTION, "build"]) assert result.exit_code == 0, result.stdout_bytes - result = self.runner.invoke(cli, [*CLI_LOG_OPTION, "generate-key", DEFAULT_LEDGER, self.key_path]) + result = self.runner.invoke( + cli, [*CLI_LOG_OPTION, "generate-key", DEFAULT_LEDGER, self.key_path] + ) assert result.exit_code == 0, result.stdout_bytes - result = self.runner.invoke(cli, [*CLI_LOG_OPTION, "add-key", DEFAULT_LEDGER, self.key_path]) + result = self.runner.invoke( + cli, [*CLI_LOG_OPTION, "add-key", DEFAULT_LEDGER, self.key_path] + ) assert result.exit_code == 0, result.stdout_bytes - result = self.runner.invoke(cli, [*CLI_LOG_OPTION, "generate-key", DEFAULT_LEDGER, self.conn_key_path]) + result = self.runner.invoke( + cli, [*CLI_LOG_OPTION, "generate-key", DEFAULT_LEDGER, self.conn_key_path] + ) assert result.exit_code == 0, result.stdout_bytes - result = self.runner.invoke(cli, [*CLI_LOG_OPTION, "add-key", DEFAULT_LEDGER, self.conn_key_path, "--connection"]) + result = self.runner.invoke( + cli, + [ + *CLI_LOG_OPTION, + "add-key", + DEFAULT_LEDGER, + self.conn_key_path, + "--connection", + ], + ) assert result.exit_code == 0, result.stdout_bytes result = self.runner.invoke(cli, [*CLI_LOG_OPTION, "issue-certificates"]) From 0303c0a91ac8d49cdec0df1256c0ec4307441038 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 31 Dec 2020 12:31:42 +0000 Subject: [PATCH 104/204] reintroduce overwriten changes --- aea/helpers/acn/agent_record.py | 28 +++---------------- .../connections/p2p_libp2p/connection.yaml | 2 +- .../p2p_libp2p_client/connection.yaml | 4 +-- packages/hashes.csv | 4 +-- 4 files changed, 9 insertions(+), 29 deletions(-) diff --git a/aea/helpers/acn/agent_record.py b/aea/helpers/acn/agent_record.py index 29ef3378ec..65b1862a82 100644 --- a/aea/helpers/acn/agent_record.py +++ b/aea/helpers/acn/agent_record.py @@ -19,35 +19,13 @@ """This module contains types and helpers for acn Proof-of-Representation.""" -import base64 -from hashlib import sha256 from pathlib import Path -from typing import List, Tuple - -from ecdsa import SECP256k1, VerifyingKey +from typing import Tuple from aea.crypto.fetchai import FetchAIHelper from aea.helpers.base import CertRequest -def recover_verify_keys_from_message(message: bytes, signature: str) -> List[str]: - """ - Get the public key used to produce the `signature` of the `message` - - :param message: raw bytes used to produce signature - :param signature: signature of the message - """ - - signature_b64 = base64.b64decode(signature) - verifying_keys = VerifyingKey.from_public_key_recovery( - signature_b64, message, SECP256k1, hashfunc=sha256, - ) - public_keys = [ - verifying_key.to_string("compressed").hex() for verifying_key in verifying_keys - ] - return public_keys - - def signature_from_cert_request( cert: CertRequest, message: str, signer_address: str ) -> Tuple[str, str]: @@ -65,7 +43,9 @@ def signature_from_cert_request( signature = bytes.fromhex(Path(cert.save_path).read_bytes().decode("ascii")).decode( "ascii" ) - public_keys = recover_verify_keys_from_message(cert.get_message(message), signature) + public_keys = FetchAIHelper.recover_verifying_keys_from_message( + cert.get_message(message), signature + ) if len(public_keys) == 0: raise Exception("Malformed signature") addresses = [ diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index f9a6d66de9..be7642502e 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -55,7 +55,7 @@ cert_requests: not_after: '2022-01-01' not_before: '2021-01-01' public_key: fetchai - save_path: ./conn_cert.txt + save_path: /Users/davidminarsch/work/agents-aea/.source/conn_cert.txt excluded_protocols: [] restricted_to_protocols: [] dependencies: {} diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml index 35f0159c6f..7cbf6055ca 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml @@ -30,13 +30,13 @@ cert_requests: not_after: '2022-01-01' not_before: '2021-01-01' public_key: 0217a59bd805c310aca4febe0e99ce22ee3712ae085dc1e5630430b1e15a584bb7 - save_path: ./acn_fetch_ai_11000.txt + save_path: /Users/davidminarsch/work/agents-aea/.source/acn_fetch_ai_11000.txt - identifier: acn ledger_id: fetchai not_after: '2022-01-01' not_before: '2021-01-01' public_key: 03fa7cfae1037cba5218f0f5743802eced8de3247c55ecebaae46c7d3679e3f91d - save_path: ./acn_fetch_ai_11001.txt + save_path: /Users/davidminarsch/work/agents-aea/.source/acn_fetch_ai_11001.txt excluded_protocols: [] restricted_to_protocols: [] dependencies: {} diff --git a/packages/hashes.csv b/packages/hashes.csv index 73fc65c5cd..16d40e9033 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,8 +34,8 @@ fetchai/connections/http_server,QmQ8hNfEts6th9zfnocYHzyYw4dXkDNDj8zoKtov1wB49n fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 -fetchai/connections/p2p_libp2p,QmSE3Awr7gDJX3S4tHqPyELK4XEPHbR5saQXEhCzYC3Trc -fetchai/connections/p2p_libp2p_client,QmeuqPXvijyU22QGfmAQLHmpUSLLs1h6Pz1H5PX32aqC2q +fetchai/connections/p2p_libp2p,QmPPYb1BbgrMNfqyn8qmBcj498KAVpyd18HnbvsJzcZgDD +fetchai/connections/p2p_libp2p_client,QmXfPXUsNj34fAxAo9mK4nzNhZRvTowB6Qo6T5tUxZBq8Y fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T fetchai/connections/scaffold,QmaCupER4vedakPPLzn2Rw9YHPmqSPaZh5xnqi6xVxQHvn From cfe26acafb8e0a13cb183f8069e7e0605db419fc Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 31 Dec 2020 12:44:09 +0000 Subject: [PATCH 105/204] remove absolute path resolution --- aea/cli/run.py | 2 -- aea/helpers/base.py | 2 +- packages/fetchai/connections/p2p_libp2p/connection.yaml | 2 +- .../fetchai/connections/p2p_libp2p_client/connection.yaml | 4 ++-- packages/hashes.csv | 4 ++-- .../test_connections/test_p2p_libp2p/test_aea_cli.py | 4 ++++ 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/aea/cli/run.py b/aea/cli/run.py index 6d959a44b2..bd1e7ace68 100644 --- a/aea/cli/run.py +++ b/aea/cli/run.py @@ -27,7 +27,6 @@ from aea.aea import AEA from aea.aea_builder import AEABuilder, DEFAULT_ENV_DOTFILE from aea.cli.install import do_install -from aea.cli.issue_certificates import issue_certificates_ from aea.cli.utils.click_utils import ConnectionsOption from aea.cli.utils.constants import AEA_LOGO, REQUIREMENTS from aea.cli.utils.context import Context @@ -85,7 +84,6 @@ def run( ): """Run the agent.""" ctx = cast(Context, click_context.obj) - issue_certificates_(ctx) profiling = int(profiling) if profiling > 0: with _profiling_context(period=profiling): diff --git a/aea/helpers/base.py b/aea/helpers/base.py index 35cd67a2e7..965c89dfe8 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -642,7 +642,7 @@ def __init__( self._not_after_string = not_after self._not_before = self._parse_datetime(not_before) self._not_after = self._parse_datetime(not_after) - self._save_path = Path(os.path.abspath(save_path)) + self._save_path = Path(save_path) self._parse_public_key(public_key) self._check_validation_boundaries() diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index be7642502e..560dbd517a 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -55,7 +55,7 @@ cert_requests: not_after: '2022-01-01' not_before: '2021-01-01' public_key: fetchai - save_path: /Users/davidminarsch/work/agents-aea/.source/conn_cert.txt + save_path: .source/conn_cert.txt excluded_protocols: [] restricted_to_protocols: [] dependencies: {} diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml index 7cbf6055ca..53dbfa4cca 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml @@ -30,13 +30,13 @@ cert_requests: not_after: '2022-01-01' not_before: '2021-01-01' public_key: 0217a59bd805c310aca4febe0e99ce22ee3712ae085dc1e5630430b1e15a584bb7 - save_path: /Users/davidminarsch/work/agents-aea/.source/acn_fetch_ai_11000.txt + save_path: .source/acn_fetch_ai_11000.txt - identifier: acn ledger_id: fetchai not_after: '2022-01-01' not_before: '2021-01-01' public_key: 03fa7cfae1037cba5218f0f5743802eced8de3247c55ecebaae46c7d3679e3f91d - save_path: /Users/davidminarsch/work/agents-aea/.source/acn_fetch_ai_11001.txt + save_path: .source/acn_fetch_ai_11001.txt excluded_protocols: [] restricted_to_protocols: [] dependencies: {} diff --git a/packages/hashes.csv b/packages/hashes.csv index 16d40e9033..fd97b0ae35 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,8 +34,8 @@ fetchai/connections/http_server,QmQ8hNfEts6th9zfnocYHzyYw4dXkDNDj8zoKtov1wB49n fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 -fetchai/connections/p2p_libp2p,QmPPYb1BbgrMNfqyn8qmBcj498KAVpyd18HnbvsJzcZgDD -fetchai/connections/p2p_libp2p_client,QmXfPXUsNj34fAxAo9mK4nzNhZRvTowB6Qo6T5tUxZBq8Y +fetchai/connections/p2p_libp2p,QmYfWRHfumMdrMrjizBCeW1DJDRdyehk9d7R9vKfaevEp6 +fetchai/connections/p2p_libp2p_client,QmSNv83o2sfmrQjtNAuFqMpq1BqXWE7re8ufMtYEu8wvgs fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T fetchai/connections/scaffold,QmaCupER4vedakPPLzn2Rw9YHPmqSPaZh5xnqi6xVxQHvn diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py index 5827338284..8f0e8f1e35 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_aea_cli.py @@ -65,6 +65,8 @@ def test_agent(self): self.set_config("{}.log_file".format(config_path), log_file) TestP2PLibp2pConnectionAEARunningDefaultConfigNode.log_files.append(log_file) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) + process = self.run_agent() is_running = self.is_running(process, timeout=LIBP2P_LAUNCH_TIMEOUT) assert is_running, "AEA not running within timeout!" @@ -127,6 +129,8 @@ def test_agent(self): self.set_config("{}.log_file".format(config_path), log_file) TestP2PLibp2pConnectionAEARunningFullNode.log_files.append(log_file) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) + process = self.run_agent() is_running = self.is_running(process, timeout=LIBP2P_LAUNCH_TIMEOUT) From 084421e637dfc68c3fc83ec146c281d5ff939be2 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 31 Dec 2020 13:33:59 +0000 Subject: [PATCH 106/204] fix end-to-end tests for certificates usage --- docs/car-park-skills.md | 24 +++++++++++- tests/conftest.py | 8 ++-- tests/data/dummy_aea/aea-config.yaml | 4 +- tests/test_configurations/test_base.py | 4 +- .../md_files/bash-car-park-skills.md | 15 +++++++- .../test_p2p_libp2p/test_public_dht.py | 2 +- .../test_p2p_libp2p_client/test_aea_cli.py | 1 + .../test_skills_integration/test_carpark.py | 35 ++++++++---------- .../test_skills_integration/test_erc1155.py | 19 +++++----- .../test_skills_integration/test_generic.py | 35 ++++++++---------- .../test_skills_integration/test_ml_skills.py | 33 +++++++++-------- .../test_skills_integration/test_tac.py | 37 +++++++++++-------- .../test_thermometer.py | 33 +++++++++-------- .../test_skills_integration/test_weather.py | 33 +++++++++-------- 14 files changed, 164 insertions(+), 119 deletions(-) diff --git a/docs/car-park-skills.md b/docs/car-park-skills.md index 3df566a160..83fedd1ddf 100644 --- a/docs/car-park-skills.md +++ b/docs/car-park-skills.md @@ -126,7 +126,17 @@ First, create the private key for the car data seller AEA based on the network y ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` + +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, create certificates to associate the keys: +``` bash +aea issue-certificates ``` ### Add keys and generate wealth for the car data buyer AEA @@ -137,7 +147,6 @@ First, create the private key for the car data buyer AEA based on the network yo ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection ``` Then, create some wealth for your car data buyer based on the network you want to transact with. On the Fetch.ai `AgentLand` network: @@ -145,6 +154,17 @@ Then, create some wealth for your car data buyer based on the network you want t aea generate-wealth fetchai ``` +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, create certificates to associate the keys: +``` bash +aea issue-certificates +``` + ## Run the AEAs Run both AEAs from their respective terminals. diff --git a/tests/conftest.py b/tests/conftest.py index 443ef8ac6d..cce9b4af67 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -191,19 +191,19 @@ FETCHAI_P2P_ADDRESS = "/dns4/127.0.0.1/tcp/9000/p2p/16Uiu2HAmLBCAqHL8SuFosyDhAKYsLKXBZBWXBsB9oFw2qU4Kckun" # relates to NON_FUNDED_FETCHAI_PRIVATE_KEY_1 NON_GENESIS_CONFIG = { "delegate_uri": "127.0.0.1:11001", - "entry_peers": [COSMOS_P2P_ADDRESS], + "entry_peers": [FETCHAI_P2P_ADDRESS], "local_uri": "127.0.0.1:9001", "log_file": "libp2p_node.log", "public_uri": "127.0.0.1:9001", - "ledger_id": "cosmos", + "ledger_id": "fetchai", } NON_GENESIS_CONFIG_TWO = { "delegate_uri": "127.0.0.1:11002", - "entry_peers": [COSMOS_P2P_ADDRESS], + "entry_peers": [FETCHAI_P2P_ADDRESS], "local_uri": "127.0.0.1:9002", "log_file": "libp2p_node.log", "public_uri": "127.0.0.1:9002", - "ledger_id": "cosmos", + "ledger_id": "fetchai", } PUBLIC_DHT_P2P_MADDR_1 = "/dns4/acn.fetch.ai/tcp/9000/p2p/16Uiu2HAkw1ypeQYQbRFV5hKUxGRHocwU5ohmVmCnyJNg36tnPFdx" PUBLIC_DHT_P2P_MADDR_2 = "/dns4/acn.fetch.ai/tcp/9001/p2p/16Uiu2HAmVWnopQAqq4pniYLw44VRvYxBUoRHqjz1Hh2SoCyjbyRW" diff --git a/tests/data/dummy_aea/aea-config.yaml b/tests/data/dummy_aea/aea-config.yaml index df74b841ea..0fdb353c5f 100644 --- a/tests/data/dummy_aea/aea-config.yaml +++ b/tests/data/dummy_aea/aea-config.yaml @@ -28,10 +28,10 @@ logging_config: private_key_paths: cosmos: cosmos_private_key.txt ethereum: ethereum_private_key.txt - fetchai: fetch_private_key.txt + fetchai: fetchai_private_key.txt connection_private_key_paths: cosmos: cosmos_private_key.txt ethereum: ethereum_private_key.txt - fetchai: fetch_private_key.txt + fetchai: fetchai_private_key.txt registry_path: ../../packages default_routing: {} diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index eade52496b..67cac4557e 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -363,7 +363,9 @@ def test_update(self): """Test the update method.""" new_private_key_paths = dict(ethereum="foo") expected_private_key_paths = dict( - ethereum="foo", cosmos="cosmos_private_key.txt" + ethereum="foo", + cosmos="cosmos_private_key.txt", + fetchai="fetchai_private_key.txt", ) self.aea_config.update( dict( diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md index 4808d6837a..5da905c255 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-car-park-skills.md @@ -45,7 +45,13 @@ aea config set --type dict agent.default_routing \ ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates ``` ``` bash aea generate-key fetchai @@ -56,6 +62,13 @@ aea add-key fetchai fetchai_private_key.txt --connection aea generate-wealth fetchai ``` ``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates +``` +``` bash aea run ``` ``` bash diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py index 38abf15816..59c359b86e 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py @@ -56,7 +56,7 @@ PUBLIC_DHT_DELEGATE_URIS = [PUBLIC_DHT_DELEGATE_URI_1, PUBLIC_DHT_DELEGATE_URI_2] PUBLIC_DHT_PUBLIC_KEYS = [PUBLIC_DHT_P2P_PUBLIC_KEY_1, PUBLIC_DHT_P2P_PUBLIC_KEY_2] AEA_DEFAULT_LAUNCH_TIMEOUT = 15 -AEA_LIBP2P_LAUNCH_TIMEOUT = 660 # may download up to ~66Mb +AEA_LIBP2P_LAUNCH_TIMEOUT = 20 @pytest.mark.integration diff --git a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py index daa1952364..5e60ddddd3 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_aea_cli.py @@ -95,6 +95,7 @@ def test_connection(self): ) ], ) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) process = self.run_agent() is_running = self.is_running(process, timeout=DEFAULT_LAUNCH_TIMEOUT) diff --git a/tests/test_packages/test_skills_integration/test_carpark.py b/tests/test_packages/test_skills_integration/test_carpark.py index e6127fbc93..30a2db7fc6 100644 --- a/tests/test_packages/test_skills_integration/test_carpark.py +++ b/tests/test_packages/test_skills_integration/test_carpark.py @@ -28,12 +28,11 @@ from packages.fetchai.connections.p2p_libp2p.connection import LIBP2P_SUCCESS_MESSAGE from tests.conftest import ( - COSMOS, - COSMOS_PRIVATE_KEY_FILE_CONNECTION, FETCHAI, FETCHAI_PRIVATE_KEY_FILE, + FETCHAI_PRIVATE_KEY_FILE_CONNECTION, MAX_FLAKY_RERUNS_INTEGRATION, - NON_FUNDED_COSMOS_PRIVATE_KEY_1, + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, NON_GENESIS_CONFIG, wait_for_localhost_ports_to_close, ) @@ -81,24 +80,22 @@ def test_carpark(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file( - NON_FUNDED_COSMOS_PRIVATE_KEY_1, COSMOS_PRIVATE_KEY_FILE_CONNECTION + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, FETCHAI_PRIVATE_KEY_FILE_CONNECTION ) - setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) - # replace location setting_path = ( "vendor.fetchai.skills.carpark_detection.models.strategy.args.location" ) self.nested_set_config(setting_path, location) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) # Setup agent two self.set_agent_context(carpark_client_aea_name) @@ -118,10 +115,10 @@ def test_carpark(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) # set p2p configs @@ -134,6 +131,7 @@ def test_carpark(self): ) self.nested_set_config(setting_path, location) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) # Fire the sub-processes and the threads. self.set_agent_context(carpark_aea_name) @@ -257,24 +255,22 @@ def test_carpark(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file( - NON_FUNDED_COSMOS_PRIVATE_KEY_1, COSMOS_PRIVATE_KEY_FILE_CONNECTION + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, FETCHAI_PRIVATE_KEY_FILE_CONNECTION ) - setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) - # replace location setting_path = ( "vendor.fetchai.skills.carpark_detection.models.strategy.args.location" ) self.nested_set_config(setting_path, location) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) # Setup agent two self.set_agent_context(carpark_client_aea_name) @@ -297,10 +293,10 @@ def test_carpark(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) # fund key @@ -316,6 +312,7 @@ def test_carpark(self): ) self.nested_set_config(setting_path, location) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) # Fire the sub-processes and the threads. self.set_agent_context(carpark_aea_name) diff --git a/tests/test_packages/test_skills_integration/test_erc1155.py b/tests/test_packages/test_skills_integration/test_erc1155.py index 57cb73adf9..e9480dc307 100644 --- a/tests/test_packages/test_skills_integration/test_erc1155.py +++ b/tests/test_packages/test_skills_integration/test_erc1155.py @@ -26,16 +26,15 @@ from packages.fetchai.connections.p2p_libp2p.connection import LIBP2P_SUCCESS_MESSAGE from tests.conftest import ( - COSMOS, - COSMOS_PRIVATE_KEY_FILE_CONNECTION, ETHEREUM, ETHEREUM_PRIVATE_KEY_FILE, FETCHAI, FETCHAI_PRIVATE_KEY_FILE, + FETCHAI_PRIVATE_KEY_FILE_CONNECTION, FUNDED_ETH_PRIVATE_KEY_2, FUNDED_ETH_PRIVATE_KEY_3, MAX_FLAKY_RERUNS_ETH, - NON_FUNDED_COSMOS_PRIVATE_KEY_1, + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, NON_GENESIS_CONFIG, UseGanache, wait_for_localhost_ports_to_close, @@ -94,18 +93,18 @@ def test_generic(self): FUNDED_ETH_PRIVATE_KEY_3, ETHEREUM_PRIVATE_KEY_FILE ) self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file( - NON_FUNDED_COSMOS_PRIVATE_KEY_1, COSMOS_PRIVATE_KEY_FILE_CONNECTION + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, FETCHAI_PRIVATE_KEY_FILE_CONNECTION ) setting_path = "vendor.fetchai.connections.soef.config.chain_identifier" self.set_config(setting_path, "ethereum") setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) + self.set_config(setting_path, FETCHAI) self.run_install() # replace location @@ -139,10 +138,10 @@ def test_generic(self): FUNDED_ETH_PRIVATE_KEY_2, ETHEREUM_PRIVATE_KEY_FILE ) self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) setting_path = "vendor.fetchai.connections.soef.config.chain_identifier" self.set_config(setting_path, "ethereum") @@ -159,6 +158,7 @@ def test_generic(self): # run agents self.set_agent_context(deploy_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) deploy_aea_process = self.run_agent() check_strings = ( @@ -197,6 +197,7 @@ def test_generic(self): self.set_agent_context(client_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) client_aea_process = self.run_agent() check_strings = ( diff --git a/tests/test_packages/test_skills_integration/test_generic.py b/tests/test_packages/test_skills_integration/test_generic.py index 4252847a2b..f12ab3ca19 100644 --- a/tests/test_packages/test_skills_integration/test_generic.py +++ b/tests/test_packages/test_skills_integration/test_generic.py @@ -27,12 +27,11 @@ from packages.fetchai.connections.p2p_libp2p.connection import LIBP2P_SUCCESS_MESSAGE from tests.conftest import ( - COSMOS, - COSMOS_PRIVATE_KEY_FILE_CONNECTION, FETCHAI, FETCHAI_PRIVATE_KEY_FILE, + FETCHAI_PRIVATE_KEY_FILE_CONNECTION, MAX_FLAKY_RERUNS_INTEGRATION, - NON_FUNDED_COSMOS_PRIVATE_KEY_1, + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, NON_GENESIS_CONFIG, wait_for_localhost_ports_to_close, ) @@ -82,18 +81,15 @@ def test_generic(self, pytestconfig): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file( - NON_FUNDED_COSMOS_PRIVATE_KEY_1, COSMOS_PRIVATE_KEY_FILE_CONNECTION + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, FETCHAI_PRIVATE_KEY_FILE_CONNECTION ) - setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) - # make runable: setting_path = "vendor.fetchai.skills.generic_seller.is_abstract" self.set_config(setting_path, False, "bool") @@ -104,6 +100,7 @@ def test_generic(self, pytestconfig): ) self.nested_set_config(setting_path, location) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) # prepare buyer agent self.set_agent_context(buyer_aea_name) @@ -123,10 +120,10 @@ def test_generic(self, pytestconfig): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) # set p2p configs @@ -143,6 +140,7 @@ def test_generic(self, pytestconfig): ) self.nested_set_config(setting_path, location) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) # run AEAs self.set_agent_context(seller_aea_name) @@ -266,18 +264,15 @@ def test_generic(self, pytestconfig): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file( - NON_FUNDED_COSMOS_PRIVATE_KEY_1, COSMOS_PRIVATE_KEY_FILE_CONNECTION + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, FETCHAI_PRIVATE_KEY_FILE_CONNECTION ) - setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) - # make runable: setting_path = "vendor.fetchai.skills.generic_seller.is_abstract" self.set_config(setting_path, False, "bool") @@ -288,6 +283,7 @@ def test_generic(self, pytestconfig): ) self.nested_set_config(setting_path, location) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) # prepare buyer agent self.set_agent_context(buyer_aea_name) @@ -313,10 +309,10 @@ def test_generic(self, pytestconfig): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) # fund key @@ -332,6 +328,7 @@ def test_generic(self, pytestconfig): ) self.nested_set_config(setting_path, location) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) # run AEAs self.set_agent_context(seller_aea_name) diff --git a/tests/test_packages/test_skills_integration/test_ml_skills.py b/tests/test_packages/test_skills_integration/test_ml_skills.py index 40ece8d0a3..69f9886c8b 100644 --- a/tests/test_packages/test_skills_integration/test_ml_skills.py +++ b/tests/test_packages/test_skills_integration/test_ml_skills.py @@ -28,12 +28,11 @@ from packages.fetchai.connections.p2p_libp2p.connection import LIBP2P_SUCCESS_MESSAGE from tests.conftest import ( - COSMOS, - COSMOS_PRIVATE_KEY_FILE_CONNECTION, FETCHAI, FETCHAI_PRIVATE_KEY_FILE, + FETCHAI_PRIVATE_KEY_FILE_CONNECTION, MAX_FLAKY_RERUNS_INTEGRATION, - NON_FUNDED_COSMOS_PRIVATE_KEY_1, + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, NON_GENESIS_CONFIG, wait_for_localhost_ports_to_close, ) @@ -89,17 +88,17 @@ def test_ml_skills(self, pytestconfig): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file( - NON_FUNDED_COSMOS_PRIVATE_KEY_1, COSMOS_PRIVATE_KEY_FILE_CONNECTION + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, FETCHAI_PRIVATE_KEY_FILE_CONNECTION ) setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) + self.set_config(setting_path, FETCHAI) # replace location setting_path = ( @@ -125,10 +124,10 @@ def test_ml_skills(self, pytestconfig): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) # set p2p configs @@ -141,6 +140,7 @@ def test_ml_skills(self, pytestconfig): self.set_agent_context(data_provider_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) data_provider_aea_process = self.run_agent() check_strings = ( @@ -160,6 +160,7 @@ def test_ml_skills(self, pytestconfig): self.set_agent_context(model_trainer_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) model_trainer_aea_process = self.run_agent() check_strings = ( @@ -266,17 +267,17 @@ def test_ml_skills(self, pytestconfig): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file( - NON_FUNDED_COSMOS_PRIVATE_KEY_1, COSMOS_PRIVATE_KEY_FILE_CONNECTION + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, FETCHAI_PRIVATE_KEY_FILE_CONNECTION ) setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) + self.set_config(setting_path, FETCHAI) # replace location setting_path = ( @@ -305,10 +306,10 @@ def test_ml_skills(self, pytestconfig): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) # fund key @@ -324,6 +325,7 @@ def test_ml_skills(self, pytestconfig): self.set_agent_context(data_provider_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) data_provider_aea_process = self.run_agent() check_strings = ( @@ -343,6 +345,7 @@ def test_ml_skills(self, pytestconfig): self.set_agent_context(model_trainer_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) model_trainer_aea_process = self.run_agent() check_strings = ( diff --git a/tests/test_packages/test_skills_integration/test_tac.py b/tests/test_packages/test_skills_integration/test_tac.py index e97186a5e9..b3607ba5d4 100644 --- a/tests/test_packages/test_skills_integration/test_tac.py +++ b/tests/test_packages/test_skills_integration/test_tac.py @@ -29,18 +29,17 @@ from packages.fetchai.connections.p2p_libp2p.connection import LIBP2P_SUCCESS_MESSAGE from tests.conftest import ( - COSMOS, - COSMOS_PRIVATE_KEY_FILE_CONNECTION, ETHEREUM, ETHEREUM_PRIVATE_KEY_FILE, FETCHAI, FETCHAI_PRIVATE_KEY_FILE, + FETCHAI_PRIVATE_KEY_FILE_CONNECTION, FUNDED_ETH_PRIVATE_KEY_1, FUNDED_ETH_PRIVATE_KEY_2, FUNDED_ETH_PRIVATE_KEY_3, MAX_FLAKY_RERUNS_ETH, MAX_FLAKY_RERUNS_INTEGRATION, - NON_FUNDED_COSMOS_PRIVATE_KEY_1, + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, NON_GENESIS_CONFIG, NON_GENESIS_CONFIG_TWO, UseGanache, @@ -104,16 +103,16 @@ def test_tac(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file( - NON_FUNDED_COSMOS_PRIVATE_KEY_1, COSMOS_PRIVATE_KEY_FILE_CONNECTION + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, FETCHAI_PRIVATE_KEY_FILE_CONNECTION ) setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) + self.set_config(setting_path, FETCHAI) # replace location setting_path = ( @@ -161,10 +160,10 @@ def test_tac(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) # set p2p configs @@ -197,6 +196,7 @@ def test_tac(self): setting_path = "vendor.fetchai.skills.tac_control.models.parameters.args.registration_start_time" self.set_config(setting_path, start_time) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) tac_controller_process = self.run_agent() check_strings = ( @@ -215,10 +215,12 @@ def test_tac(self): # run two agents (participants) self.set_agent_context(tac_aea_one) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) tac_aea_one_process = self.run_agent() self.set_agent_context(tac_aea_two) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) tac_aea_two_process = self.run_agent() check_strings = ( @@ -362,19 +364,19 @@ def test_tac(self): # add keys self.generate_private_key(ETHEREUM) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(ETHEREUM, ETHEREUM_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file( FUNDED_ETH_PRIVATE_KEY_1, ETHEREUM_PRIVATE_KEY_FILE ) self.replace_private_key_in_file( - NON_FUNDED_COSMOS_PRIVATE_KEY_1, COSMOS_PRIVATE_KEY_FILE_CONNECTION + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, FETCHAI_PRIVATE_KEY_FILE_CONNECTION ) setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) + self.set_config(setting_path, FETCHAI) setting_path = "vendor.fetchai.connections.soef.config.chain_identifier" self.set_config(setting_path, ETHEREUM) setting_path = "vendor.fetchai.skills.tac_control.is_abstract" @@ -435,10 +437,10 @@ def test_tac(self): # add keys self.generate_private_key(ETHEREUM) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(ETHEREUM, ETHEREUM_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file(private_key, ETHEREUM_PRIVATE_KEY_FILE) @@ -446,7 +448,7 @@ def test_tac(self): setting_path = "vendor.fetchai.connections.p2p_libp2p.config" self.nested_set_config(setting_path, config) setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) + self.set_config(setting_path, FETCHAI) setting_path = "vendor.fetchai.connections.soef.config.chain_identifier" self.set_config(setting_path, ETHEREUM) @@ -478,6 +480,7 @@ def test_tac(self): setting_path = "vendor.fetchai.skills.tac_control_contract.models.parameters.args.registration_start_time" self.set_config(setting_path, start_time) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) tac_controller_process = self.run_agent() check_strings = ( @@ -508,10 +511,12 @@ def test_tac(self): # run two agents (participants) self.set_agent_context(tac_aea_one) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) tac_aea_one_process = self.run_agent() self.set_agent_context(tac_aea_two) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) tac_aea_two_process = self.run_agent() check_strings = ( diff --git a/tests/test_packages/test_skills_integration/test_thermometer.py b/tests/test_packages/test_skills_integration/test_thermometer.py index 84e334c4df..28d1a6b717 100644 --- a/tests/test_packages/test_skills_integration/test_thermometer.py +++ b/tests/test_packages/test_skills_integration/test_thermometer.py @@ -27,12 +27,11 @@ from packages.fetchai.connections.p2p_libp2p.connection import LIBP2P_SUCCESS_MESSAGE from tests.conftest import ( - COSMOS, - COSMOS_PRIVATE_KEY_FILE_CONNECTION, FETCHAI, FETCHAI_PRIVATE_KEY_FILE, + FETCHAI_PRIVATE_KEY_FILE_CONNECTION, MAX_FLAKY_RERUNS_INTEGRATION, - NON_FUNDED_COSMOS_PRIVATE_KEY_1, + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, NON_GENESIS_CONFIG, wait_for_localhost_ports_to_close, ) @@ -81,16 +80,16 @@ def test_thermometer(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file( - NON_FUNDED_COSMOS_PRIVATE_KEY_1, COSMOS_PRIVATE_KEY_FILE_CONNECTION + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, FETCHAI_PRIVATE_KEY_FILE_CONNECTION ) setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) + self.set_config(setting_path, FETCHAI) # replace location setting_path = "vendor.fetchai.skills.thermometer.models.strategy.args.location" @@ -114,10 +113,10 @@ def test_thermometer(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) # set p2p configs @@ -133,6 +132,7 @@ def test_thermometer(self): # run AEAs self.set_agent_context(thermometer_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) thermometer_aea_process = self.run_agent() check_strings = ( @@ -150,6 +150,7 @@ def test_thermometer(self): self.set_agent_context(thermometer_client_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) thermometer_client_aea_process = self.run_agent() check_strings = ( @@ -258,16 +259,16 @@ def test_thermometer(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file( - NON_FUNDED_COSMOS_PRIVATE_KEY_1, COSMOS_PRIVATE_KEY_FILE_CONNECTION + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, FETCHAI_PRIVATE_KEY_FILE_CONNECTION ) setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) + self.set_config(setting_path, FETCHAI) # replace location setting_path = "vendor.fetchai.skills.thermometer.models.strategy.args.location" @@ -294,10 +295,10 @@ def test_thermometer(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) # fund key @@ -316,6 +317,7 @@ def test_thermometer(self): # run AEAs self.set_agent_context(thermometer_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) thermometer_aea_process = self.run_agent() check_strings = ( @@ -333,6 +335,7 @@ def test_thermometer(self): self.set_agent_context(thermometer_client_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) thermometer_client_aea_process = self.run_agent() check_strings = ( diff --git a/tests/test_packages/test_skills_integration/test_weather.py b/tests/test_packages/test_skills_integration/test_weather.py index 7032784534..fd8cd2a2e5 100644 --- a/tests/test_packages/test_skills_integration/test_weather.py +++ b/tests/test_packages/test_skills_integration/test_weather.py @@ -27,12 +27,11 @@ from packages.fetchai.connections.p2p_libp2p.connection import LIBP2P_SUCCESS_MESSAGE from tests.conftest import ( - COSMOS, - COSMOS_PRIVATE_KEY_FILE_CONNECTION, FETCHAI, FETCHAI_PRIVATE_KEY_FILE, + FETCHAI_PRIVATE_KEY_FILE_CONNECTION, MAX_FLAKY_RERUNS_INTEGRATION, - NON_FUNDED_COSMOS_PRIVATE_KEY_1, + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, NON_GENESIS_CONFIG, wait_for_localhost_ports_to_close, ) @@ -81,16 +80,16 @@ def test_weather(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file( - NON_FUNDED_COSMOS_PRIVATE_KEY_1, COSMOS_PRIVATE_KEY_FILE_CONNECTION + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, FETCHAI_PRIVATE_KEY_FILE_CONNECTION ) setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) + self.set_config(setting_path, FETCHAI) # replace location setting_path = ( @@ -117,10 +116,10 @@ def test_weather(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) # set p2p configs @@ -136,6 +135,7 @@ def test_weather(self): # run agents self.set_agent_context(weather_station_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) weather_station_process = self.run_agent() check_strings = ( @@ -153,6 +153,7 @@ def test_weather(self): self.set_agent_context(weather_client_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) weather_client_process = self.run_agent() check_strings = ( @@ -253,16 +254,16 @@ def test_weather(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file( - NON_FUNDED_COSMOS_PRIVATE_KEY_1, COSMOS_PRIVATE_KEY_FILE_CONNECTION + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, FETCHAI_PRIVATE_KEY_FILE_CONNECTION ) setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) + self.set_config(setting_path, FETCHAI) # replace location setting_path = ( @@ -291,10 +292,10 @@ def test_weather(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) # fund key @@ -312,6 +313,7 @@ def test_weather(self): self.set_agent_context(weather_station_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) weather_station_process = self.run_agent() check_strings = ( @@ -329,6 +331,7 @@ def test_weather(self): self.set_agent_context(weather_client_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) weather_client_process = self.run_agent() check_strings = ( From ee87a88d7d306d95c4aaabf224ad89bff4a433e6 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 31 Dec 2020 13:42:54 +0000 Subject: [PATCH 107/204] fix hashes --- tests/data/hashes.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/hashes.csv b/tests/data/hashes.csv index aeed33872b..17ee712718 100644 --- a/tests/data/hashes.csv +++ b/tests/data/hashes.csv @@ -1,4 +1,4 @@ -dummy_author/agents/dummy_aea,Qmf2EJ4ajaGn3RakmVTJVPQK66N8C9CinxutcyWeghCNcK +dummy_author/agents/dummy_aea,QmRDbp8LFBXqRLF8sScUcgcTf5JhqbAQJJvCVgRVa8oyPs dummy_author/skills/dummy_skill,QmSmzcjGNhiz2KpSE6wSi19AeesCVjzupJm1NpXYJVnJuZ fetchai/connections/dummy_connection,Qmep3ive7fVWJi965SSdJv6fdfzAKD1wmP9A1kYRoQjRNn fetchai/contracts/dummy_contract,QmXhMKRF5upco1efaC9oNcesStEMkPWKhBUv6CK9Vngk2S From 4a2242b9babbb9106c5fa040822897fa96f7973c Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Thu, 31 Dec 2020 14:22:32 +0000 Subject: [PATCH 108/204] Fix private key file name issue tests --- packages/fetchai/connections/p2p_libp2p/connection.yaml | 2 +- .../fetchai/connections/p2p_libp2p_client/connection.yaml | 4 ++-- packages/hashes.csv | 4 ++-- .../{fetch_private_key.txt => fetchai_private_key.txt} | 0 tests/data/hashes.csv | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) rename tests/data/dummy_aea/{fetch_private_key.txt => fetchai_private_key.txt} (100%) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index 560dbd517a..e0fb6206b4 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -55,7 +55,7 @@ cert_requests: not_after: '2022-01-01' not_before: '2021-01-01' public_key: fetchai - save_path: .source/conn_cert.txt + save_path: conn_cert.txt excluded_protocols: [] restricted_to_protocols: [] dependencies: {} diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml index 53dbfa4cca..8dd23f5489 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml @@ -30,13 +30,13 @@ cert_requests: not_after: '2022-01-01' not_before: '2021-01-01' public_key: 0217a59bd805c310aca4febe0e99ce22ee3712ae085dc1e5630430b1e15a584bb7 - save_path: .source/acn_fetch_ai_11000.txt + save_path: acn_fetch_ai_11000.txt - identifier: acn ledger_id: fetchai not_after: '2022-01-01' not_before: '2021-01-01' public_key: 03fa7cfae1037cba5218f0f5743802eced8de3247c55ecebaae46c7d3679e3f91d - save_path: .source/acn_fetch_ai_11001.txt + save_path: acn_fetch_ai_11001.txt excluded_protocols: [] restricted_to_protocols: [] dependencies: {} diff --git a/packages/hashes.csv b/packages/hashes.csv index fd97b0ae35..64b0b457c0 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,8 +34,8 @@ fetchai/connections/http_server,QmQ8hNfEts6th9zfnocYHzyYw4dXkDNDj8zoKtov1wB49n fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 -fetchai/connections/p2p_libp2p,QmYfWRHfumMdrMrjizBCeW1DJDRdyehk9d7R9vKfaevEp6 -fetchai/connections/p2p_libp2p_client,QmSNv83o2sfmrQjtNAuFqMpq1BqXWE7re8ufMtYEu8wvgs +fetchai/connections/p2p_libp2p,QmdzN27WT42mBXPUASYikJxUB3MQTE86DkuQT4VcKamVE5 +fetchai/connections/p2p_libp2p_client,QmScicGKTje9Zp2j9s5hzuqSP8EZoFYDkmqKfRrTAvaivA fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T fetchai/connections/scaffold,QmaCupER4vedakPPLzn2Rw9YHPmqSPaZh5xnqi6xVxQHvn diff --git a/tests/data/dummy_aea/fetch_private_key.txt b/tests/data/dummy_aea/fetchai_private_key.txt similarity index 100% rename from tests/data/dummy_aea/fetch_private_key.txt rename to tests/data/dummy_aea/fetchai_private_key.txt diff --git a/tests/data/hashes.csv b/tests/data/hashes.csv index 17ee712718..dbfeb31e3c 100644 --- a/tests/data/hashes.csv +++ b/tests/data/hashes.csv @@ -1,4 +1,4 @@ -dummy_author/agents/dummy_aea,QmRDbp8LFBXqRLF8sScUcgcTf5JhqbAQJJvCVgRVa8oyPs +dummy_author/agents/dummy_aea,QmfM8jdCnfg8duVMoVnM5qtMH9w3GRLyLkuCPZHbJMdtx8 dummy_author/skills/dummy_skill,QmSmzcjGNhiz2KpSE6wSi19AeesCVjzupJm1NpXYJVnJuZ fetchai/connections/dummy_connection,Qmep3ive7fVWJi965SSdJv6fdfzAKD1wmP9A1kYRoQjRNn fetchai/contracts/dummy_contract,QmXhMKRF5upco1efaC9oNcesStEMkPWKhBUv6CK9Vngk2S From bdc21f7e78195434fd7cb1b2d9927f79bb65512f Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 31 Dec 2020 16:01:20 +0000 Subject: [PATCH 109/204] fix doc tests and cli tests --- aea/cli/issue_certificates.py | 12 ++++----- aea/configurations/base.py | 4 ++- .../connection-custom_config.json | 5 +++- aea/crypto/helpers.py | 12 +++++++++ docs/cli-vs-programmatic-aeas.md | 24 ++++++++++++++++- tests/conftest.py | 2 +- ...rivate_key.txt => fetchai_private_key.txt} | 0 tests/data/hashes.csv | 2 +- tests/test_cli/test_launch_end_to_end.py | 23 ++++++++++++++++ .../programmatic_aea.py | 24 ++++++++++++++++- .../test_cli_vs_programmatic_aea.py | 3 ++- .../test_orm_integration.py | 19 ++++++------- .../test_skill_guide/test_skill_guide.py | 19 ++++++------- .../test_skills_integration/test_tac.py | 27 +++++++++++++++++-- 14 files changed, 143 insertions(+), 33 deletions(-) rename tests/data/dummy_aea/{fetch_private_key.txt => fetchai_private_key.txt} (100%) diff --git a/aea/cli/issue_certificates.py b/aea/cli/issue_certificates.py index 728edf9e1c..266bf6c9fb 100644 --- a/aea/cli/issue_certificates.py +++ b/aea/cli/issue_certificates.py @@ -32,9 +32,10 @@ from aea.cli.utils.package_utils import get_package_path_unified from aea.configurations.base import ConnectionConfig, PublicId from aea.configurations.constants import CONNECTION +from aea.crypto.helpers import make_certificate from aea.crypto.registries import crypto_registry from aea.exceptions import enforce -from aea.helpers.base import CertRequest, ensure_dir +from aea.helpers.base import CertRequest @click.command() @@ -84,12 +85,11 @@ def _process_certificate( raise ClickException( f"Cannot find private key with id '{ledger_id}'. Please use `aea generate-key {key_identifier}` and `aea add-key {key_identifier}` to add a private key with id '{key_identifier}'." ) - crypto = crypto_registry.make(ledger_id, private_key_path=crypto_private_key_path) message = cert_request.get_message(public_key) - signature = crypto.sign_message(message).encode("ascii").hex() - click.echo(f"Generated signature: '{signature}'") - ensure_dir(os.path.dirname(os.path.join(ctx.cwd, output_path))) - Path(output_path).write_bytes(signature.encode("ascii")) + cert = make_certificate( + ledger_id, crypto_private_key_path, message, os.path.join(ctx.cwd, output_path) + ) + click.echo(f"Generated signature: '{cert}'") def _process_connection(ctx: Context, connection_id: PublicId): diff --git a/aea/configurations/base.py b/aea/configurations/base.py index eca468f924..6971958717 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -518,7 +518,9 @@ class ConnectionConfig(ComponentConfiguration): package_type = PackageType.CONNECTION schema = "connection-config_schema.json" - FIELDS_ALLOWED_TO_UPDATE: FrozenSet[str] = frozenset(["config", "is_abstract"]) + FIELDS_ALLOWED_TO_UPDATE: FrozenSet[str] = frozenset( + ["config", "cert_requests", "is_abstract"] + ) def __init__( self, diff --git a/aea/configurations/schemas/configurable_parts/connection-custom_config.json b/aea/configurations/schemas/configurable_parts/connection-custom_config.json index 6f62cd8828..bda88e055f 100644 --- a/aea/configurations/schemas/configurable_parts/connection-custom_config.json +++ b/aea/configurations/schemas/configurable_parts/connection-custom_config.json @@ -17,7 +17,10 @@ "type": "object" }, "is_abstract": { - "$ref": "skill-config_schema.json#/properties/is_abstract" + "$ref": "connection-config_schema.json#/properties/is_abstract" + }, + "cert_requests": { + "$ref": "connection-config_schema.json#/properties/cert_requests" } } } diff --git a/aea/crypto/helpers.py b/aea/crypto/helpers.py index a3186939c3..52a3c381c3 100644 --- a/aea/crypto/helpers.py +++ b/aea/crypto/helpers.py @@ -26,6 +26,7 @@ from aea.configurations.base import AgentConfig from aea.configurations.constants import PRIVATE_KEY_PATH_SCHEMA from aea.crypto.registries import crypto_registry, make_crypto, make_faucet_api +from aea.helpers.base import ensure_dir from aea.helpers.env_vars import is_env_variable @@ -146,3 +147,14 @@ def private_key_verify_or_create( repr(config_private_key_path), identifier, ) ) + + +def make_certificate( + ledger_id: str, crypto_private_key_path: str, message: bytes, output_path: str +) -> str: + """Create certificate.""" + crypto = crypto_registry.make(ledger_id, private_key_path=crypto_private_key_path) + signature = crypto.sign_message(message).encode("ascii").hex() + ensure_dir(os.path.dirname(output_path)) + Path(output_path).write_bytes(signature.encode("ascii")) + return signature diff --git a/docs/cli-vs-programmatic-aeas.md b/docs/cli-vs-programmatic-aeas.md index 898565a9b1..0ad59031f1 100644 --- a/docs/cli-vs-programmatic-aeas.md +++ b/docs/cli-vs-programmatic-aeas.md @@ -74,8 +74,13 @@ from aea.aea import AEA from aea.aea_builder import AEABuilder from aea.configurations.base import ConnectionConfig from aea.crypto.fetchai import FetchAICrypto -from aea.crypto.helpers import PRIVATE_KEY_PATH_SCHEMA, create_private_key +from aea.crypto.helpers import ( + PRIVATE_KEY_PATH_SCHEMA, + create_private_key, + make_certificate, +) from aea.crypto.wallet import Wallet +from aea.helpers.base import CertRequest from aea.identity.base import Identity from aea.protocols.base import Protocol from aea.registries.resources import Resources @@ -179,6 +184,22 @@ def run(): resources.add_connection(ledger_api_connection) # Add the P2P connection + cert_path = ".source/conn_cert.txt" + cert_request = CertRequest( + **{ + "identifier": "acn", + "ledger_id": FetchAICrypto.identifier, + "not_after": "2022-01-01", + "not_before": "2021-01-01", + "public_key": "fetchai", + "save_path": cert_path, + }, + ) + public_key = wallet.connection_cryptos.public_keys.get(FetchAICrypto.identifier) + message = cert_request.get_message(public_key) + make_certificate( + FetchAICrypto.identifier, FETCHAI_PRIVATE_KEY_FILE, message, cert_path + ) configuration = ConnectionConfig( connection_id=P2PLibp2pConnection.connection_id, delegate_uri="127.0.0.1:11001", @@ -188,6 +209,7 @@ def run(): public_uri="127.0.0.1:9001", build_directory=os.getcwd(), build_entrypoint="check_dependencies.py", + cert_requests=[cert_request], ) configuration.directory = os.path.dirname( packages.fetchai.connections.p2p_libp2p.connection.__file__ diff --git a/tests/conftest.py b/tests/conftest.py index cce9b4af67..e6b8df3642 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -633,7 +633,7 @@ def update_default_ethereum_ledger_api(ethereum_testnet_config): @pytest.mark.integration @pytest.mark.ledger @pytest.fixture(scope="session") -@action_for_platform("Linux", skip=False) +# @action_for_platform("Linux", skip=False) def ganache( ganache_configuration, ganache_addr, diff --git a/tests/data/dummy_aea/fetch_private_key.txt b/tests/data/dummy_aea/fetchai_private_key.txt similarity index 100% rename from tests/data/dummy_aea/fetch_private_key.txt rename to tests/data/dummy_aea/fetchai_private_key.txt diff --git a/tests/data/hashes.csv b/tests/data/hashes.csv index 17ee712718..dbfeb31e3c 100644 --- a/tests/data/hashes.csv +++ b/tests/data/hashes.csv @@ -1,4 +1,4 @@ -dummy_author/agents/dummy_aea,QmRDbp8LFBXqRLF8sScUcgcTf5JhqbAQJJvCVgRVa8oyPs +dummy_author/agents/dummy_aea,QmfM8jdCnfg8duVMoVnM5qtMH9w3GRLyLkuCPZHbJMdtx8 dummy_author/skills/dummy_skill,QmSmzcjGNhiz2KpSE6wSi19AeesCVjzupJm1NpXYJVnJuZ fetchai/connections/dummy_connection,Qmep3ive7fVWJi965SSdJv6fdfzAKD1wmP9A1kYRoQjRNn fetchai/contracts/dummy_contract,QmXhMKRF5upco1efaC9oNcesStEMkPWKhBUv6CK9Vngk2S diff --git a/tests/test_cli/test_launch_end_to_end.py b/tests/test_cli/test_launch_end_to_end.py index c49e9acbb1..c6b978a70c 100644 --- a/tests/test_cli/test_launch_end_to_end.py +++ b/tests/test_cli/test_launch_end_to_end.py @@ -27,6 +27,7 @@ from aea.test_tools.test_cases import AEATestCaseMany from tests.common.pexpect_popen import PexpectWrapper +from tests.conftest import FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION class TestLaunchEndToEnd(AEATestCaseMany): @@ -142,6 +143,28 @@ def test_end_to_end(self): self.run_cli_command( "build", cwd=search_agent_name, ) + self.set_agent_context(registration_agent_name) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) + self.add_private_key( + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True + ) + self.generate_private_key() + self.add_private_key() + self.unset_agent_context() + self.run_cli_command( + "issue-certificates", cwd=registration_agent_name, + ) + self.set_agent_context(search_agent_name) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) + self.add_private_key( + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True + ) + self.generate_private_key() + self.add_private_key() + self.unset_agent_context() + self.run_cli_command( + "issue-certificates", cwd=search_agent_name, + ) proc = PexpectWrapper( # nosec [ diff --git a/tests/test_docs/test_cli_vs_programmatic_aeas/programmatic_aea.py b/tests/test_docs/test_cli_vs_programmatic_aeas/programmatic_aea.py index ccf0713fe7..46434c43ce 100644 --- a/tests/test_docs/test_cli_vs_programmatic_aeas/programmatic_aea.py +++ b/tests/test_docs/test_cli_vs_programmatic_aeas/programmatic_aea.py @@ -28,8 +28,13 @@ from aea.aea_builder import AEABuilder from aea.configurations.base import ConnectionConfig from aea.crypto.fetchai import FetchAICrypto -from aea.crypto.helpers import PRIVATE_KEY_PATH_SCHEMA, create_private_key +from aea.crypto.helpers import ( + PRIVATE_KEY_PATH_SCHEMA, + create_private_key, + make_certificate, +) from aea.crypto.wallet import Wallet +from aea.helpers.base import CertRequest from aea.identity.base import Identity from aea.protocols.base import Protocol from aea.registries.resources import Resources @@ -133,6 +138,22 @@ def run(): resources.add_connection(ledger_api_connection) # Add the P2P connection + cert_path = ".source/conn_cert.txt" + cert_request = CertRequest( + **{ + "identifier": "acn", + "ledger_id": FetchAICrypto.identifier, + "not_after": "2022-01-01", + "not_before": "2021-01-01", + "public_key": "fetchai", + "save_path": cert_path, + }, + ) + public_key = wallet.connection_cryptos.public_keys.get(FetchAICrypto.identifier) + message = cert_request.get_message(public_key) + make_certificate( + FetchAICrypto.identifier, FETCHAI_PRIVATE_KEY_FILE, message, cert_path + ) configuration = ConnectionConfig( connection_id=P2PLibp2pConnection.connection_id, delegate_uri="127.0.0.1:11001", @@ -142,6 +163,7 @@ def run(): public_uri="127.0.0.1:9001", build_directory=os.getcwd(), build_entrypoint="check_dependencies.py", + cert_requests=[cert_request], ) configuration.directory = os.path.dirname( packages.fetchai.connections.p2p_libp2p.connection.__file__ diff --git a/tests/test_docs/test_cli_vs_programmatic_aeas/test_cli_vs_programmatic_aea.py b/tests/test_docs/test_cli_vs_programmatic_aeas/test_cli_vs_programmatic_aea.py index 08d167d8a5..f755f171ed 100644 --- a/tests/test_docs/test_cli_vs_programmatic_aeas/test_cli_vs_programmatic_aea.py +++ b/tests/test_docs/test_cli_vs_programmatic_aeas/test_cli_vs_programmatic_aea.py @@ -95,6 +95,7 @@ def test_cli_programmatic_communication(self): self.nested_set_config(setting_path, location) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) weather_station_process = self.run_agent() check_strings = ( @@ -172,7 +173,7 @@ def _inject_location(self, location, dst_file_path): """Inject location into the weather client strategy.""" file = Path(dst_file_path) lines = file.read_text().splitlines() - line_insertion_position = 180 # line below: `strategy._is_ledger_tx = False` + line_insertion_position = 204 # line below: `strategy._is_ledger_tx = False` lines.insert( line_insertion_position, " from packages.fetchai.skills.generic_buyer.strategy import Location", diff --git a/tests/test_docs/test_orm_integration/test_orm_integration.py b/tests/test_docs/test_orm_integration/test_orm_integration.py index 8aba52962a..da084186ed 100644 --- a/tests/test_docs/test_orm_integration/test_orm_integration.py +++ b/tests/test_docs/test_orm_integration/test_orm_integration.py @@ -30,12 +30,11 @@ from packages.fetchai.connections.p2p_libp2p.connection import LIBP2P_SUCCESS_MESSAGE from tests.conftest import ( - COSMOS, - COSMOS_PRIVATE_KEY_FILE_CONNECTION, FETCHAI, FETCHAI_PRIVATE_KEY_FILE, + FETCHAI_PRIVATE_KEY_FILE_CONNECTION, MAX_FLAKY_RERUNS_INTEGRATION, - NON_FUNDED_COSMOS_PRIVATE_KEY_1, + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, NON_GENESIS_CONFIG, ROOT_DIR, wait_for_localhost_ports_to_close, @@ -170,16 +169,16 @@ def test_orm_integration_docs_example(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file( - NON_FUNDED_COSMOS_PRIVATE_KEY_1, COSMOS_PRIVATE_KEY_FILE_CONNECTION + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, FETCHAI_PRIVATE_KEY_FILE_CONNECTION ) setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) + self.set_config(setting_path, FETCHAI) # replace location setting_path = "skills.thermometer.models.strategy.args.location" @@ -203,10 +202,10 @@ def test_orm_integration_docs_example(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) # fund key @@ -225,6 +224,7 @@ def test_orm_integration_docs_example(self): # Fire the sub-processes and the threads. self.set_agent_context(seller_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) seller_aea_process = self.run_agent() check_strings = ( @@ -242,6 +242,7 @@ def test_orm_integration_docs_example(self): self.set_agent_context(buyer_aea_name) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) buyer_aea_process = self.run_agent() check_strings = ( diff --git a/tests/test_docs/test_skill_guide/test_skill_guide.py b/tests/test_docs/test_skill_guide/test_skill_guide.py index 381d7231a2..beaafa5b45 100644 --- a/tests/test_docs/test_skill_guide/test_skill_guide.py +++ b/tests/test_docs/test_skill_guide/test_skill_guide.py @@ -34,12 +34,11 @@ from tests.conftest import ( AUTHOR, - COSMOS, - COSMOS_PRIVATE_KEY_FILE_CONNECTION, FETCHAI, FETCHAI_PRIVATE_KEY_FILE, + FETCHAI_PRIVATE_KEY_FILE_CONNECTION, MAX_FLAKY_RERUNS_INTEGRATION, - NON_FUNDED_COSMOS_PRIVATE_KEY_1, + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, NON_GENESIS_CONFIG, ROOT_DIR, wait_for_localhost_ports_to_close, @@ -84,16 +83,16 @@ def test_update_skill_and_run(self): self.set_agent_context(simple_service_registration_aea) # add non-funded key self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) self.replace_private_key_in_file( - NON_FUNDED_COSMOS_PRIVATE_KEY_1, COSMOS_PRIVATE_KEY_FILE_CONNECTION + NON_FUNDED_FETCHAI_PRIVATE_KEY_1, FETCHAI_PRIVATE_KEY_FILE_CONNECTION ) setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" - self.set_config(setting_path, COSMOS) + self.set_config(setting_path, FETCHAI) default_routing = { "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0", @@ -153,10 +152,10 @@ def test_update_skill_and_run(self): # add keys self.generate_private_key(FETCHAI) - self.generate_private_key(COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION) + self.generate_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION) self.add_private_key(FETCHAI, FETCHAI_PRIVATE_KEY_FILE) self.add_private_key( - COSMOS, COSMOS_PRIVATE_KEY_FILE_CONNECTION, connection=True + FETCHAI, FETCHAI_PRIVATE_KEY_FILE_CONNECTION, connection=True ) # fund key @@ -175,6 +174,7 @@ def test_update_skill_and_run(self): # run agents self.set_agent_context(simple_service_registration_aea) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) simple_service_registration_aea_process = self.run_agent() check_strings = ( @@ -197,6 +197,7 @@ def test_update_skill_and_run(self): self.set_agent_context(search_aea) self.run_cli_command("build", cwd=self._get_cwd()) + self.run_cli_command("issue-certificates", cwd=self._get_cwd()) search_aea_process = self.run_agent() check_strings = ( diff --git a/tests/test_packages/test_skills_integration/test_tac.py b/tests/test_packages/test_skills_integration/test_tac.py index b3607ba5d4..0cf756ff78 100644 --- a/tests/test_packages/test_skills_integration/test_tac.py +++ b/tests/test_packages/test_skills_integration/test_tac.py @@ -19,6 +19,7 @@ """This test module contains the integration test for the tac skills.""" import datetime +import json import uuid from random import uniform @@ -381,6 +382,18 @@ def test_tac(self): self.set_config(setting_path, ETHEREUM) setting_path = "vendor.fetchai.skills.tac_control.is_abstract" self.set_config(setting_path, True, "bool") + setting_path = "vendor.fetchai.connections.p2p_libp2p.cert_requests" + cert_requests = [ + { + "identifier": "acn", + "ledger_id": "ethereum", + "not_after": "2022-01-01", + "not_before": "2021-01-01", + "public_key": "ethereum", + "save_path": ".source/conn_cert.txt", + } + ] + self.set_config(setting_path, json.dumps(cert_requests), type_="list") # replace location setting_path = ( @@ -449,8 +462,18 @@ def test_tac(self): self.nested_set_config(setting_path, config) setting_path = "vendor.fetchai.connections.p2p_libp2p.config.ledger_id" self.set_config(setting_path, FETCHAI) - setting_path = "vendor.fetchai.connections.soef.config.chain_identifier" - self.set_config(setting_path, ETHEREUM) + setting_path = "vendor.fetchai.connections.p2p_libp2p.cert_requests" + cert_requests = [ + { + "identifier": "acn", + "ledger_id": "ethereum", + "not_after": "2022-01-01", + "not_before": "2021-01-01", + "public_key": "ethereum", + "save_path": ".source/conn_cert.txt", + } + ] + self.set_config(setting_path, json.dumps(cert_requests), type_="list") # replace location setting_path = ( From 9d1a3b12b7fef26917f8995ed0f6166fcab2105d Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 31 Dec 2020 16:15:53 +0000 Subject: [PATCH 110/204] fix paths --- docs/cli-vs-programmatic-aeas.md | 2 +- packages/fetchai/connections/p2p_libp2p/connection.yaml | 2 +- .../fetchai/connections/p2p_libp2p_client/connection.yaml | 4 ++-- packages/hashes.csv | 4 ++-- .../test_cli_vs_programmatic_aeas/programmatic_aea.py | 2 +- .../test_cli_vs_programmatic_aea.py | 4 ++-- .../test_docs/test_orm_integration/test_orm_integration.py | 6 +++--- tests/test_packages/test_skills_integration/test_tac.py | 4 ++-- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/cli-vs-programmatic-aeas.md b/docs/cli-vs-programmatic-aeas.md index 0ad59031f1..06600da643 100644 --- a/docs/cli-vs-programmatic-aeas.md +++ b/docs/cli-vs-programmatic-aeas.md @@ -184,7 +184,7 @@ def run(): resources.add_connection(ledger_api_connection) # Add the P2P connection - cert_path = ".source/conn_cert.txt" + cert_path = ".certs/conn_cert.txt" cert_request = CertRequest( **{ "identifier": "acn", diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index e0fb6206b4..a534e8f820 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -55,7 +55,7 @@ cert_requests: not_after: '2022-01-01' not_before: '2021-01-01' public_key: fetchai - save_path: conn_cert.txt + save_path: .certs/conn_cert.txt excluded_protocols: [] restricted_to_protocols: [] dependencies: {} diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml index 8dd23f5489..ce84613833 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml @@ -30,13 +30,13 @@ cert_requests: not_after: '2022-01-01' not_before: '2021-01-01' public_key: 0217a59bd805c310aca4febe0e99ce22ee3712ae085dc1e5630430b1e15a584bb7 - save_path: acn_fetch_ai_11000.txt + save_path: .certs/acn_fetch_ai_11000.txt - identifier: acn ledger_id: fetchai not_after: '2022-01-01' not_before: '2021-01-01' public_key: 03fa7cfae1037cba5218f0f5743802eced8de3247c55ecebaae46c7d3679e3f91d - save_path: acn_fetch_ai_11001.txt + save_path: .certs/acn_fetch_ai_11001.txt excluded_protocols: [] restricted_to_protocols: [] dependencies: {} diff --git a/packages/hashes.csv b/packages/hashes.csv index 64b0b457c0..473a83d0a7 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,8 +34,8 @@ fetchai/connections/http_server,QmQ8hNfEts6th9zfnocYHzyYw4dXkDNDj8zoKtov1wB49n fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 -fetchai/connections/p2p_libp2p,QmdzN27WT42mBXPUASYikJxUB3MQTE86DkuQT4VcKamVE5 -fetchai/connections/p2p_libp2p_client,QmScicGKTje9Zp2j9s5hzuqSP8EZoFYDkmqKfRrTAvaivA +fetchai/connections/p2p_libp2p,QmRgJVhG48t8WS89JRqUJ67m56Jm1GHZBwUT6mofSq5gfd +fetchai/connections/p2p_libp2p_client,Qmd8kf1oaemRf9sk87bjAypvuutjpF8MLwPfQqDdna4i48 fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T fetchai/connections/scaffold,QmaCupER4vedakPPLzn2Rw9YHPmqSPaZh5xnqi6xVxQHvn diff --git a/tests/test_docs/test_cli_vs_programmatic_aeas/programmatic_aea.py b/tests/test_docs/test_cli_vs_programmatic_aeas/programmatic_aea.py index 46434c43ce..ed76aeafc0 100644 --- a/tests/test_docs/test_cli_vs_programmatic_aeas/programmatic_aea.py +++ b/tests/test_docs/test_cli_vs_programmatic_aeas/programmatic_aea.py @@ -138,7 +138,7 @@ def run(): resources.add_connection(ledger_api_connection) # Add the P2P connection - cert_path = ".source/conn_cert.txt" + cert_path = ".certs/conn_cert.txt" cert_request = CertRequest( **{ "identifier": "acn", diff --git a/tests/test_docs/test_cli_vs_programmatic_aeas/test_cli_vs_programmatic_aea.py b/tests/test_docs/test_cli_vs_programmatic_aeas/test_cli_vs_programmatic_aea.py index f755f171ed..0cf8d5d49a 100644 --- a/tests/test_docs/test_cli_vs_programmatic_aeas/test_cli_vs_programmatic_aea.py +++ b/tests/test_docs/test_cli_vs_programmatic_aeas/test_cli_vs_programmatic_aea.py @@ -105,7 +105,7 @@ def test_cli_programmatic_communication(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - weather_station_process, check_strings, timeout=240, is_terminating=False + weather_station_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -124,7 +124,7 @@ def test_cli_programmatic_communication(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - weather_client_process, check_strings, timeout=240, is_terminating=False, + weather_client_process, check_strings, timeout=30, is_terminating=False, ) assert ( missing_strings == [] diff --git a/tests/test_docs/test_orm_integration/test_orm_integration.py b/tests/test_docs/test_orm_integration/test_orm_integration.py index da084186ed..e00db4493e 100644 --- a/tests/test_docs/test_orm_integration/test_orm_integration.py +++ b/tests/test_docs/test_orm_integration/test_orm_integration.py @@ -234,7 +234,7 @@ def test_orm_integration_docs_example(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - seller_aea_process, check_strings, timeout=240, is_terminating=False + seller_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -252,7 +252,7 @@ def test_orm_integration_docs_example(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - buyer_aea_process, check_strings, timeout=240, is_terminating=False, + buyer_aea_process, check_strings, timeout=30, is_terminating=False, ) assert ( missing_strings == [] @@ -272,7 +272,7 @@ def test_orm_integration_docs_example(self): "transaction confirmed, sending data=", ) missing_strings = self.missing_from_output( - seller_aea_process, check_strings, timeout=240, is_terminating=False + seller_aea_process, check_strings, timeout=120, is_terminating=False ) assert ( missing_strings == [] diff --git a/tests/test_packages/test_skills_integration/test_tac.py b/tests/test_packages/test_skills_integration/test_tac.py index 0cf756ff78..1a5e6a7771 100644 --- a/tests/test_packages/test_skills_integration/test_tac.py +++ b/tests/test_packages/test_skills_integration/test_tac.py @@ -390,7 +390,7 @@ def test_tac(self): "not_after": "2022-01-01", "not_before": "2021-01-01", "public_key": "ethereum", - "save_path": ".source/conn_cert.txt", + "save_path": "conn_cert.txt", } ] self.set_config(setting_path, json.dumps(cert_requests), type_="list") @@ -470,7 +470,7 @@ def test_tac(self): "not_after": "2022-01-01", "not_before": "2021-01-01", "public_key": "ethereum", - "save_path": ".source/conn_cert.txt", + "save_path": "conn_cert.txt", } ] self.set_config(setting_path, json.dumps(cert_requests), type_="list") From e6322c0ab594132aeb6c09aa048cd44bb786abd8 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Thu, 31 Dec 2020 15:25:27 +0000 Subject: [PATCH 111/204] Add full coverage --- .../fetchai/connections/p2p_libp2p/connection.py | 12 ++++++------ .../fetchai/connections/p2p_libp2p/connection.yaml | 2 +- .../connections/p2p_libp2p_client/connection.py | 12 ++++++------ .../connections/p2p_libp2p_client/connection.yaml | 2 +- packages/hashes.csv | 4 ++-- .../test_p2p_libp2p/test_public_dht.py | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index d291f46049..de4dc8b7b9 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -509,7 +509,7 @@ def __init__(self, **kwargs): # node will be run as a ClientDHT # requires entry peers to use as relay if entry_peers is None or len(entry_peers) == 0: - raise ValueError( + raise ValueError( # pragma: no cover "At least one Entry Peer should be provided when node is run in relayed mode" ) if delegate_uri is not None: # pragma: no cover @@ -519,7 +519,7 @@ def __init__(self, **kwargs): else: # node will be run as a full NodeDHT if uri is None: - raise ValueError( + raise ValueError( # pragma: no cover "Local Uri must be set when Public Uri is provided. " "Hint: they are the same for local host/network deployment" ) @@ -534,11 +534,11 @@ def __init__(self, **kwargs): cert_requests = self.configuration.cert_requests if cert_requests is None or len(cert_requests) != 1: - raise ValueError( + raise ValueError( # pragma: no cover "cert_requests field must be set and contain exactly one entry!" ) if not Path(cert_requests[0].save_path).is_file(): - raise Exception( + raise Exception( # pragma: no cover f"cert_request 'save_path' field {cert_requests[0].save_path} is not a file. " "Please ensure that 'issue-certificates' command is called beforehand" ) @@ -546,7 +546,7 @@ def __init__(self, **kwargs): signature, agent_public_key = signature_from_cert_request( cert_requests[0], key.public_key, self.address ) - except Exception as e: + except Exception as e: # pragma: no cover raise ValueError( f"Incorrect certificate from file {cert_requests[0].save_path} : {str(e)}" ) @@ -560,7 +560,7 @@ def __init__(self, **kwargs): ) try: record.check_validity(self.address, key.public_key) - except Exception as e: + except Exception as e: # pragma: no cover raise ValueError("Invalid Proof-of-Representation: {}".format(str(e))) # libp2p local node diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index a534e8f820..092327a3db 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -16,7 +16,7 @@ fingerprint: aea/pipe_unix.go: QmSzbAexrSUSA9ZE6VT6MmcWF5MhEcmceQjAbnfRoK7Ruv aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK check_dependencies.py: QmcZATMqEThMQBdsud1AwJEYtKq86VfPSXhnfdE7TSWcsc - connection.py: QmbZaNALbsN4j3YkEWiQzNzfpbuTeoTx6CEdaZZ8FL3S9V + connection.py: QmToAmEktB6M4iU8jHB648vtdNru9gGkMFTNH2Fy9rLFJo dht/dhtclient/dhtclient.go: QmZCgMowWw4ueCSsyYY12om3RMQA3Ydp98ti65Eqa5ccuz dht/dhtclient/dhtclient_test.go: QmbTvSmNU1uqjLQo9LdD7zWmDcLuzEacDeg7AG4VXhmriV dht/dhtclient/options.go: QmW3UjASUP6zGwK8b61hHQhVoLqBxQ13vZ1gcHuTmdrsDf diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.py b/packages/fetchai/connections/p2p_libp2p_client/connection.py index 39c70dd8c3..476c863a53 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.py +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.py @@ -102,12 +102,12 @@ def __init__(self, **kwargs): cert_requests = self.configuration.cert_requests if cert_requests is None or len(cert_requests) != len(nodes): - raise ValueError( + raise ValueError( # pragma: nocover "cert_requests field must be set and contain exactly as many entries as 'nodes'!" ) for cert_request in cert_requests: if not Path(cert_request.save_path).is_file(): - raise Exception( + raise Exception( # pragma: nocover "cert_request 'save_path' field is not a file. " "Please ensure that 'issue-certificates' command is called beforehand" ) @@ -136,7 +136,7 @@ def __init__(self, **kwargs): cert, nodes_public_keys[i], self.address ) signatures.append(signature) - except Exception as e: + except Exception as e: # pragma: nocover raise ValueError( f"Incorrect certificate from file {cert.save_path} " f"for node {nodes_uris[i]} : {str(e)}" @@ -161,7 +161,7 @@ def __init__(self, **kwargs): ) try: record.check_validity(self.address, public_key) - except Exception as e: + except Exception as e: # pragma: nocover raise ValueError( "Invalid Proof-of-Representation for node {}: {}".format( str(uri), str(e) @@ -242,12 +242,12 @@ async def _setup_connection(self): msg = AcnMessage() msg.ParseFromString(buf) payload = msg.WhichOneof("payload") - if payload != "status": + if payload != "status": # pragma: nocover raise Exception(f"Wrong response message from peer: {payload}") response = msg.status # pylint: disable=no-member if response.code != Status.SUCCESS: # pylint: disable=no-member - raise Exception( + raise Exception( # pragma: nocover "Registration to peer failed: {}".format( Status.ErrCode.Name(response.code) # pylint: disable=no-member ) diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml index ce84613833..4856ba79a7 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml @@ -12,7 +12,7 @@ fingerprint: __init__.py: QmT1FEHkPGMHV5oiVEfQHHr25N2qdZxydSNRJabJvYiTgf acn_message.proto: QmSwpCP72J94kt2MaLSPQdt8puqtLMVPp3Migwp4xRwadZ acn_message_pb2.py: QmbVqUUCHyQpfx7dHvW5e5j3RGcjNYqsDjuPV8eNcca2Nf - connection.py: QmVcc6cqVC4bx7te9yTgxzLn2RFWBsrYk6fBniSKoiMo4t + connection.py: QmPsDJ1jJ1qamuCzopygzunvszYPHtgbgTp4HBet4FN1Ya fingerprint_ignore_patterns: [] connections: [] protocols: [] diff --git a/packages/hashes.csv b/packages/hashes.csv index 473a83d0a7..29d0c2287c 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,8 +34,8 @@ fetchai/connections/http_server,QmQ8hNfEts6th9zfnocYHzyYw4dXkDNDj8zoKtov1wB49n fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 -fetchai/connections/p2p_libp2p,QmRgJVhG48t8WS89JRqUJ67m56Jm1GHZBwUT6mofSq5gfd -fetchai/connections/p2p_libp2p_client,Qmd8kf1oaemRf9sk87bjAypvuutjpF8MLwPfQqDdna4i48 +fetchai/connections/p2p_libp2p,QmZF328Yjdr6drv44C6KzrWes66DHfBWMPuCNPoA7dioA8 +fetchai/connections/p2p_libp2p_client,QmREbD3SiKrMdmzYbSwFX52M8wRmpPWcRs9XFJu3Cdsz5L fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T fetchai/connections/scaffold,QmaCupER4vedakPPLzn2Rw9YHPmqSPaZh5xnqi6xVxQHvn diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py index 59c359b86e..fdbbc75ad0 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py @@ -56,7 +56,7 @@ PUBLIC_DHT_DELEGATE_URIS = [PUBLIC_DHT_DELEGATE_URI_1, PUBLIC_DHT_DELEGATE_URI_2] PUBLIC_DHT_PUBLIC_KEYS = [PUBLIC_DHT_P2P_PUBLIC_KEY_1, PUBLIC_DHT_P2P_PUBLIC_KEY_2] AEA_DEFAULT_LAUNCH_TIMEOUT = 15 -AEA_LIBP2P_LAUNCH_TIMEOUT = 20 +AEA_LIBP2P_LAUNCH_TIMEOUT = 15 @pytest.mark.integration From 69927f253d1bee9238c312d51f9d885917a11c5a Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 31 Dec 2020 17:07:53 +0000 Subject: [PATCH 112/204] multiple doc updates for issue certs --- docs/car-park-skills.md | 4 ++-- docs/cli-vs-programmatic-aeas.md | 12 +++++++++- docs/erc1155-skills.md | 10 ++++++++ docs/generic-skills.md | 24 +++++++++++++++++-- docs/ml-skills.md | 24 +++++++++++++++++-- docs/tac-skills-contract.md | 11 ++++++--- docs/tac-skills.md | 12 +++++++++- docs/thermometer-skills.md | 24 +++++++++++++++++-- docs/weather-skills.md | 24 +++++++++++++++++-- .../md_files/bash-cli-vs-programmatic-aeas.md | 8 ++++++- .../md_files/bash-generic-skills.md | 15 +++++++++++- .../test_bash_yaml/md_files/bash-ml-skills.md | 15 +++++++++++- .../md_files/bash-tac-skills-contract.md | 7 ++++-- .../md_files/bash-tac-skills.md | 8 ++++++- .../md_files/bash-thermometer-skills.md | 15 +++++++++++- .../md_files/bash-weather-skills.md | 15 +++++++++++- .../test_skill_guide/test_skill_guide.py | 2 +- .../test_skills_integration/test_carpark.py | 10 ++++---- .../test_skills_integration/test_erc1155.py | 2 +- .../test_skills_integration/test_generic.py | 10 ++++---- .../test_skills_integration/test_weather.py | 10 ++++---- 21 files changed, 222 insertions(+), 40 deletions(-) diff --git a/docs/car-park-skills.md b/docs/car-park-skills.md index 83fedd1ddf..85ffa1829e 100644 --- a/docs/car-park-skills.md +++ b/docs/car-park-skills.md @@ -134,7 +134,7 @@ aea generate-key fetchai fetchai_connection_private_key.txt aea add-key fetchai fetchai_connection_private_key.txt --connection ``` -Finally, create certificates to associate the keys: +Finally, certify the key for use by the connections that request that: ``` bash aea issue-certificates ``` @@ -160,7 +160,7 @@ aea generate-key fetchai fetchai_connection_private_key.txt aea add-key fetchai fetchai_connection_private_key.txt --connection ``` -Finally, create certificates to associate the keys: +Finally, certify the key for use by the connections that request that: ``` bash aea issue-certificates ``` diff --git a/docs/cli-vs-programmatic-aeas.md b/docs/cli-vs-programmatic-aeas.md index 06600da643..e4a678e5c5 100644 --- a/docs/cli-vs-programmatic-aeas.md +++ b/docs/cli-vs-programmatic-aeas.md @@ -46,7 +46,17 @@ Add keys for the weather station. ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` + +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates ``` ### Run the weather station AEA diff --git a/docs/erc1155-skills.md b/docs/erc1155-skills.md index 00c287db4d..6583fa9d82 100644 --- a/docs/erc1155-skills.md +++ b/docs/erc1155-skills.md @@ -88,6 +88,11 @@ aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt --connection ``` +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates +``` + ### Create the client AEA In another terminal, fetch the AEA that will get some tokens from the deployer. @@ -155,6 +160,11 @@ aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt --connection ``` +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates +``` + ## Run Ganache Run the following command diff --git a/docs/generic-skills.md b/docs/generic-skills.md index 724110a86b..724c837640 100644 --- a/docs/generic-skills.md +++ b/docs/generic-skills.md @@ -134,7 +134,17 @@ First, create the private key for the seller AEA based on the network you want t ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` + +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates ``` ### Add keys and generate wealth for the buyer AEA @@ -145,7 +155,6 @@ First, create the private key for the buyer AEA based on the network you want to ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection ``` Then, create some wealth for your buyer based on the network you want to transact with. On the Fetch.ai `AgentLand` network: @@ -153,6 +162,17 @@ Then, create some wealth for your buyer based on the network you want to transac aea generate-wealth fetchai ``` +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates +``` + ### Update the skill configs The default skill configs assume that the transaction is settled against the fetchai ledger. diff --git a/docs/ml-skills.md b/docs/ml-skills.md index d4bb109a45..e030a5a7cc 100644 --- a/docs/ml-skills.md +++ b/docs/ml-skills.md @@ -133,7 +133,17 @@ First, create the private key for the data provider AEA based on the network you ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` + +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates ``` ### Add keys and generate wealth for the model trainer AEA @@ -144,7 +154,6 @@ First, create the private key for the model trainer AEA based on the network you ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection ``` Then, create some wealth for your model trainer based on the network you want to transact with. On the Fetch.ai `AgentLand` network: @@ -152,6 +161,17 @@ Then, create some wealth for your model trainer based on the network you want to aea generate-wealth fetchai ``` +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates +``` + ### Run both AEAs Run both AEAs from their respective terminals. diff --git a/docs/tac-skills-contract.md b/docs/tac-skills-contract.md index 84ae0de9d4..725c2908ec 100644 --- a/docs/tac-skills-contract.md +++ b/docs/tac-skills-contract.md @@ -224,10 +224,15 @@ aea build ### Add keys for all AEAs -Create the private key for the AEA for Fetch.ai `AgentLand`: +Create a private key used to secure the AEA's communications: ``` bash -aea generate-key fetchai -aea add-key fetchai fetchai_private_key.txt --connection +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates ``` ### Update the game parameters in the controller diff --git a/docs/tac-skills.md b/docs/tac-skills.md index 364f7b53bd..10512f0b0a 100644 --- a/docs/tac-skills.md +++ b/docs/tac-skills.md @@ -200,7 +200,17 @@ Create the private key for the AEA for Fetch.ai `AgentLand`: ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` + +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates ``` ### Update the game parameters in the controller diff --git a/docs/thermometer-skills.md b/docs/thermometer-skills.md index c85449aba6..6d5a0c14c9 100644 --- a/docs/thermometer-skills.md +++ b/docs/thermometer-skills.md @@ -128,7 +128,17 @@ First, create the private key for the thermometer AEA based on the network you w ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` + +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates ``` ### Add keys and generate wealth for the thermometer client AEA @@ -139,7 +149,6 @@ First, create the private key for the thermometer client AEA based on the networ ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection ``` Then, create some wealth for your thermometer client based on the network you want to transact with. On the Fetch.ai `testnet` network: @@ -147,6 +156,17 @@ Then, create some wealth for your thermometer client based on the network you wa aea generate-wealth fetchai ``` +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates +``` + ### Run both AEAs Run both AEAs from their respective terminals. diff --git a/docs/weather-skills.md b/docs/weather-skills.md index 59312da126..2ae2492366 100644 --- a/docs/weather-skills.md +++ b/docs/weather-skills.md @@ -133,7 +133,17 @@ First, create the private key for the weather station AEA based on the network y ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` + +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates ``` ### Add keys and generate wealth for the weather client AEA @@ -144,7 +154,6 @@ First, create the private key for the weather client AEA based on the network yo ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection ``` Then, create some wealth for your weather client based on the network you want to transact with. On the Fetch.ai `AgentLand` network: @@ -152,6 +161,17 @@ Then, create some wealth for your weather client based on the network you want t aea generate-wealth fetchai ``` +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates +``` + ### Run the AEAs Run both AEAs from their respective terminals. diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-cli-vs-programmatic-aeas.md b/tests/test_docs/test_bash_yaml/md_files/bash-cli-vs-programmatic-aeas.md index a5e9d38fd3..9e46390fc4 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-cli-vs-programmatic-aeas.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-cli-vs-programmatic-aeas.md @@ -11,7 +11,13 @@ aea config set vendor.fetchai.skills.weather_station.models.strategy.args.is_led ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates ``` ``` bash aea run diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills.md index c9f6cd34ef..75d4e2eac7 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills.md @@ -35,7 +35,13 @@ aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates ``` ``` bash aea generate-key fetchai @@ -46,6 +52,13 @@ aea add-key fetchai fetchai_private_key.txt --connection aea generate-wealth fetchai ``` ``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates +``` +``` bash cd my_seller_aea aea config set vendor.fetchai.skills.generic_seller.is_abstract false --type bool ``` diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md index 09aaed5d38..c8b9799c2d 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-ml-skills.md @@ -45,7 +45,13 @@ aea build ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates ``` ``` bash aea generate-key fetchai @@ -56,6 +62,13 @@ aea add-key fetchai fetchai_private_key.txt --connection aea generate-wealth fetchai ``` ``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates +``` +``` bash aea run ``` ``` bash diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills-contract.md b/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills-contract.md index f4a1a40807..1ad3568f1c 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills-contract.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills-contract.md @@ -94,8 +94,11 @@ aea install aea build ``` ``` bash -aea generate-key fetchai -aea add-key fetchai fetchai_private_key.txt --connection +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates ``` ``` bash aea config get vendor.fetchai.skills.tac_control_contract.models.parameters.args.registration_start_time diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md index 00e5c7f1fa..70d11dc6e7 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-tac-skills.md @@ -71,7 +71,13 @@ aea build ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates ``` ``` bash aea config get vendor.fetchai.skills.tac_control.models.parameters.args.registration_start_time diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-thermometer-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-thermometer-skills.md index 19971a3708..abddd1bd72 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-thermometer-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-thermometer-skills.md @@ -45,7 +45,13 @@ aea config set --type dict agent.default_routing \ ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates ``` ``` bash aea generate-key fetchai @@ -56,6 +62,13 @@ aea add-key fetchai fetchai_private_key.txt --connection aea generate-wealth fetchai ``` ``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates +``` +``` bash aea run ``` ``` bash diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-weather-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-weather-skills.md index 650877afde..9c24fbd6e5 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-weather-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-weather-skills.md @@ -45,7 +45,13 @@ aea config set --type dict agent.default_routing \ ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates ``` ``` bash aea generate-key fetchai @@ -56,6 +62,13 @@ aea add-key fetchai fetchai_private_key.txt --connection aea generate-wealth fetchai ``` ``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates +``` +``` bash aea run ``` ``` bash diff --git a/tests/test_docs/test_skill_guide/test_skill_guide.py b/tests/test_docs/test_skill_guide/test_skill_guide.py index beaafa5b45..e68a73cfc4 100644 --- a/tests/test_docs/test_skill_guide/test_skill_guide.py +++ b/tests/test_docs/test_skill_guide/test_skill_guide.py @@ -207,7 +207,7 @@ def test_update_skill_and_run(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - search_aea_process, check_strings, timeout=240, is_terminating=False + search_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] diff --git a/tests/test_packages/test_skills_integration/test_carpark.py b/tests/test_packages/test_skills_integration/test_carpark.py index 30a2db7fc6..71a65437d8 100644 --- a/tests/test_packages/test_skills_integration/test_carpark.py +++ b/tests/test_packages/test_skills_integration/test_carpark.py @@ -144,7 +144,7 @@ def test_carpark(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - carpark_aea_process, check_strings, timeout=240, is_terminating=False + carpark_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -160,7 +160,7 @@ def test_carpark(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - carpark_client_aea_process, check_strings, timeout=240, is_terminating=False + carpark_client_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -325,7 +325,7 @@ def test_carpark(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - carpark_aea_process, check_strings, timeout=240, is_terminating=False + carpark_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -341,7 +341,7 @@ def test_carpark(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - carpark_client_aea_process, check_strings, timeout=240, is_terminating=False + carpark_client_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -361,7 +361,7 @@ def test_carpark(self): "transaction confirmed, sending data=", ) missing_strings = self.missing_from_output( - carpark_aea_process, check_strings, timeout=240, is_terminating=False + carpark_aea_process, check_strings, timeout=120, is_terminating=False ) assert ( missing_strings == [] diff --git a/tests/test_packages/test_skills_integration/test_erc1155.py b/tests/test_packages/test_skills_integration/test_erc1155.py index e9480dc307..e15132acb7 100644 --- a/tests/test_packages/test_skills_integration/test_erc1155.py +++ b/tests/test_packages/test_skills_integration/test_erc1155.py @@ -168,7 +168,7 @@ def test_generic(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - deploy_aea_process, check_strings, timeout=240, is_terminating=False + deploy_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] diff --git a/tests/test_packages/test_skills_integration/test_generic.py b/tests/test_packages/test_skills_integration/test_generic.py index f12ab3ca19..7a768189f1 100644 --- a/tests/test_packages/test_skills_integration/test_generic.py +++ b/tests/test_packages/test_skills_integration/test_generic.py @@ -153,7 +153,7 @@ def test_generic(self, pytestconfig): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - seller_aea_process, check_strings, timeout=240, is_terminating=False + seller_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -169,7 +169,7 @@ def test_generic(self, pytestconfig): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - buyer_aea_process, check_strings, timeout=240, is_terminating=False + buyer_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -341,7 +341,7 @@ def test_generic(self, pytestconfig): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - seller_aea_process, check_strings, timeout=240, is_terminating=False + seller_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -357,7 +357,7 @@ def test_generic(self, pytestconfig): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - buyer_aea_process, check_strings, timeout=10, is_terminating=False + buyer_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -375,7 +375,7 @@ def test_generic(self, pytestconfig): "transaction confirmed, sending data=", ) missing_strings = self.missing_from_output( - seller_aea_process, check_strings, timeout=240, is_terminating=False + seller_aea_process, check_strings, timeout=120, is_terminating=False ) assert ( missing_strings == [] diff --git a/tests/test_packages/test_skills_integration/test_weather.py b/tests/test_packages/test_skills_integration/test_weather.py index fd8cd2a2e5..eab79a544d 100644 --- a/tests/test_packages/test_skills_integration/test_weather.py +++ b/tests/test_packages/test_skills_integration/test_weather.py @@ -145,7 +145,7 @@ def test_weather(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - weather_station_process, check_strings, timeout=240, is_terminating=False + weather_station_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -163,7 +163,7 @@ def test_weather(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - weather_client_process, check_strings, timeout=240, is_terminating=False, + weather_client_process, check_strings, timeout=30, is_terminating=False, ) assert ( missing_strings == [] @@ -323,7 +323,7 @@ def test_weather(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - weather_station_process, check_strings, timeout=240, is_terminating=False + weather_station_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -341,7 +341,7 @@ def test_weather(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - weather_client_process, check_strings, timeout=240, is_terminating=False, + weather_client_process, check_strings, timeout=30, is_terminating=False, ) assert ( missing_strings == [] @@ -359,7 +359,7 @@ def test_weather(self): "transaction confirmed, sending data=", ) missing_strings = self.missing_from_output( - weather_station_process, check_strings, timeout=240, is_terminating=False + weather_station_process, check_strings, timeout=120, is_terminating=False ) assert ( missing_strings == [] From a6c1a0281183089b7b41fcad0bdd769d3d9caeef Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Thu, 31 Dec 2020 18:46:46 +0000 Subject: [PATCH 113/204] update docs to include no outdated configs --- docs/aries-cloud-agent-demo.md | 40 +++++----- docs/cli-vs-programmatic-aeas.md | 2 +- docs/erc1155-skills.md | 34 +-------- docs/generic-skills-step-by-step.md | 63 +++++++--------- docs/generic-skills.md | 49 +++++++----- docs/oracle-demo.md | 14 +++- docs/orm-integration.md | 75 ++++++++++++++----- docs/p2p-connection.md | 20 ++--- docs/prometheus.md | 18 ++--- docs/protocol.md | 4 +- docs/quickstart.md | 6 +- docs/skill-guide.md | 24 +++++- .../md_files/bash-aries-cloud-agent-demo.md | 32 ++++---- .../md_files/bash-cli-how-to.md | 2 +- .../md_files/bash-erc1155-skills.md | 22 ++++-- .../bash-generic-skills-step-by-step.md | 33 +++++--- .../md_files/bash-generic-skills.md | 36 ++++++--- .../md_files/bash-orm-integration.md | 52 +++++++++---- .../md_files/bash-p2p-connection.md | 31 +++----- .../md_files/bash-quickstart.md | 6 +- .../md_files/bash-skill-guide.md | 16 +++- .../test_skill_guide/test_skill_guide.py | 2 +- .../test_skills_integration/test_erc1155.py | 2 +- .../test_skills_integration/test_ml_skills.py | 8 +- .../test_skills_integration/test_tac.py | 10 +-- .../test_thermometer.py | 6 +- 26 files changed, 362 insertions(+), 245 deletions(-) diff --git a/docs/aries-cloud-agent-demo.md b/docs/aries-cloud-agent-demo.md index 6ad7ce9506..1e4a97f600 100644 --- a/docs/aries-cloud-agent-demo.md +++ b/docs/aries-cloud-agent-demo.md @@ -231,17 +231,15 @@ aea config set vendor.fetchai.connections.webhook.config.webhook_url_path /webho #### Configure the `p2p_libp2p` connection: -(configuration file: `vendor/fetchai/connections/p2p_libp2p/connection.yaml`) - -Replace the `config` section with the following (note the changes in the URI ports): - -``` yaml -config: - delegate_uri: 127.0.0.1:11000 - entry_peers: [] - local_uri: 127.0.0.1:7000 - log_file: libp2p_node.log - public_uri: 127.0.0.1:7000 +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11000", + "entry_peers": [], + "local_uri": "127.0.0.1:7000", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:7000" +}' ``` ### Install the Dependencies and Run Alice_AEA: @@ -318,17 +316,15 @@ aea config set vendor.fetchai.connections.webhook.config.webhook_url_path /webho #### Configure the `p2p_libp2p` connection: -(configuration file: `vendor/fetchai/connections/p2p_libp2p/connection.yaml`) - -Replace the `config` section with the following (note the changes in the URI ports): - -``` yaml -config: - delegate_uri: 127.0.0.1:11001 - entry_peers: ['SOME_ADDRESS'] - local_uri: 127.0.0.1:7001 - log_file: libp2p_node.log - public_uri: 127.0.0.1:7001 +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:7001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:7001" +}' ``` where `SOME_ADDRESS` is **Alice_AEA's p2p address** as displayed in the third terminal. diff --git a/docs/cli-vs-programmatic-aeas.md b/docs/cli-vs-programmatic-aeas.md index e4a678e5c5..be0a51ff49 100644 --- a/docs/cli-vs-programmatic-aeas.md +++ b/docs/cli-vs-programmatic-aeas.md @@ -42,7 +42,7 @@ The `is_ledger_tx` will prevent the AEA to communicate with a ledger. ### Add keys -Add keys for the weather station. +Add a private key for the weather station. ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt diff --git a/docs/erc1155-skills.md b/docs/erc1155-skills.md index 6583fa9d82..0f9d2866fb 100644 --- a/docs/erc1155-skills.md +++ b/docs/erc1155-skills.md @@ -47,18 +47,6 @@ aea add skill fetchai/erc1155_deploy:0.19.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 -``` - -Then update the agent config (`aea-config.yaml`) with the default routing: -``` yaml -default_routing: - fetchai/contract_api:0.9.0: fetchai/ledger:0.11.0 - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` - -Or, run this command: -``` bash aea config set --type dict agent.default_routing \ '{ "fetchai/contract_api:0.9.0": "fetchai/ledger:0.11.0", @@ -76,7 +64,6 @@ aea config set agent.default_ledger ethereum

    Additionally, create the private key for the deployer AEA. Generate and add a key for Ethereum use: - ``` bash aea generate-key ethereum aea add-key ethereum ethereum_private_key.txt @@ -84,8 +71,8 @@ aea add-key ethereum ethereum_private_key.txt And one for the P2P connection: ``` bash -aea generate-key fetchai -aea add-key fetchai fetchai_private_key.txt --connection +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection ``` Finally, certify the key for use by the connections that request that: @@ -119,18 +106,6 @@ aea add skill fetchai/erc1155_client:0.18.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 -``` - -Then update the agent config (`aea-config.yaml`) with the default routing: -``` yaml -default_routing: - fetchai/contract_api:0.9.0: fetchai/ledger:0.11.0 - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` - -Or, run this command: -``` bash aea config set --type dict agent.default_routing \ '{ "fetchai/contract_api:0.9.0": "fetchai/ledger:0.11.0", @@ -148,7 +123,6 @@ aea config set agent.default_ledger ethereum
    Additionally, create the private key for the client AEA. Generate and add a key for Ethereum use: - ``` bash aea generate-key ethereum aea add-key ethereum ethereum_private_key.txt @@ -156,8 +130,8 @@ aea add-key ethereum ethereum_private_key.txt And one for the P2P connection: ``` bash -aea generate-key fetchai -aea add-key fetchai fetchai_private_key.txt --connection +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection ``` Finally, certify the key for use by the connections that request that: diff --git a/docs/generic-skills-step-by-step.md b/docs/generic-skills-step-by-step.md index 4a08bce460..bb506939c1 100644 --- a/docs/generic-skills-step-by-step.md +++ b/docs/generic-skills-step-by-step.md @@ -3148,41 +3148,34 @@ This will hash each file and save the hash in the fingerprint. This way, in the ## Run the AEAs - - ### Create private keys -Create the private key for each AEA: - +For each AEA, create the private key: ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` + +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates ``` ### Update the AEA configs -Both in `my_generic_seller/aea-config.yaml` and `my_generic_buyer/aea-config.yaml`, and -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 +In both AEAs run: +``` bash +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` ### Fund the buyer AEA @@ -3225,15 +3218,17 @@ aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` -Then, update the configuration of the buyer AEA's p2p connection (in `vendor/fetchai/connections/p2p_libp2p/connection.yaml`) replace the following: +Then, update the configuration of the buyer AEA's p2p connection: -``` yaml -config: - delegate_uri: 127.0.0.1:11001 - entry_peers: ['SOME_ADDRESS'] - local_uri: 127.0.0.1:9001 - log_file: libp2p_node.log - public_uri: 127.0.0.1:9001 +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' ``` where `SOME_ADDRESS` is replaced accordingly. diff --git a/docs/generic-skills.md b/docs/generic-skills.md index 724c837640..64ce63657e 100644 --- a/docs/generic-skills.md +++ b/docs/generic-skills.md @@ -79,13 +79,11 @@ aea add skill fetchai/generic_seller:0.18.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 -``` - -In `my_seller_aea/aea-config.yaml` add -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ```

    @@ -115,13 +113,11 @@ aea add skill fetchai/generic_buyer:0.18.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 -``` - -In `my_buyer_aea/aea-config.yaml` add -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ```

    @@ -251,19 +247,38 @@ aea run ``` Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) +This is the entry peer address for the local agent communication network created by the seller. -Then, update the configuration of the buyer AEA's p2p connection (in `vendor/fetchai/connections/p2p_libp2p/connection.yaml`) replace the following: + + +Then, in the buyer, run this command (replace `SOME_ADDRESS` with the correct value as described above): +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +This allows the buyer to connect to the same local agent communication network as the seller. Then run the buyer AEA: ``` bash diff --git a/docs/oracle-demo.md b/docs/oracle-demo.md index f5ee2bd059..e8946369fa 100644 --- a/docs/oracle-demo.md +++ b/docs/oracle-demo.md @@ -67,6 +67,16 @@ aea generate-key ethereum aea add-key ethereum ethereum_private_key.txt ``` +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates +``` The oracle AEAs require either a locally runnning test node or a connection to a remote testnet. @@ -80,12 +90,12 @@ docker run -p 8545:8545 trufflesuite/ganache-cli:latest --verbose --gasPrice=0 - ### Run the Oracle AEA Run the oracle agent. This will deploy a contract to the testnet, grant oracle permissions to the AEA's wallet address, and periodically update the contract with the latest price of FET (or whichever coin was specified). -```bash +``` bash aea run ``` After a few moments, you should see the following notices in the logs: -```bash +``` bash info: [coin_price_oracle] Oracle contract successfully deployed! ... info: [coin_price_oracle] Oracle role successfully granted! diff --git a/docs/orm-integration.md b/docs/orm-integration.md index dbd8e92227..6c6fdfbd1d 100644 --- a/docs/orm-integration.md +++ b/docs/orm-integration.md @@ -79,13 +79,11 @@ aea add skill fetchai/thermometer:0.17.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 -``` - -In `my_thermometer_aea/aea-config.yaml` add -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ```

    @@ -116,13 +114,11 @@ aea add skill fetchai/thermometer_client:0.17.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 -``` - -In `my_buyer_aea/aea-config.yaml` add -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ```

    @@ -135,7 +131,17 @@ First, create the private key for the thermometer AEA based on the network you w ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` + +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates ``` ### Add keys and generate wealth for the thermometer client AEA @@ -146,7 +152,17 @@ First, create the private key for the thermometer client AEA based on the networ ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` + +Then, create some wealth for your car data buyer based on the network you want to transact with. On the Fetch.ai `AgentLand` network: +``` bash +aea generate-wealth fetchai +``` + +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection ``` Then, create some wealth for your thermometer client based on the network you want to transact with. On the Fetch.ai `testnet` network: @@ -301,20 +317,39 @@ First, run the thermometer AEA: aea run ``` -Once you see a message of the form `To join its network use multiaddr: ['SOME_ADDRESS']` take note of the address. +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) +This is the entry peer address for the local agent communication network created by the thermometer AEA. -Then, update the configuration of the thermometer client AEA's p2p connection (in `vendor/fetchai/connections/p2p_libp2p/connection.yaml`) replace the following: + + +Then, in the thermometer client, run this command (replace `SOME_ADDRESS` with the correct value as described above): +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +This allows the thermometer client to connect to the same local agent communication network as the thermometer. Then run the thermometer client AEA: ``` bash diff --git a/docs/p2p-connection.md b/docs/p2p-connection.md index c576bece2c..84ff5ef6c6 100644 --- a/docs/p2p-connection.md +++ b/docs/p2p-connection.md @@ -25,15 +25,17 @@ aea add connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` -Provide the AEA with the information it needs to find the genesis by replacing the following block in `vendor/fetchai/connnections/p2p_libp2p/connection.yaml`: +Provide the AEA with the information it needs to find the genesis: -``` yaml -config: - delegate_uri: 127.0.0.1:11001 - entry_peers: MULTI_ADDRESSES - local_uri: 127.0.0.1:9001 - log_file: libp2p_node.log - public_uri: 127.0.0.1:9001 +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": MULTI_ADDRESSES, + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' ``` Here `MULTI_ADDRESSES` needs to be replaced with the list of multi addresses displayed in the log output of the genesis AEA. @@ -60,7 +62,7 @@ Explore the demo section for further examples. You can connect to the deployed public test network by adding one or multiple of the following addresses as the `libp2p_entry_peers`: -```yaml +``` yaml /dns4/acn.fetch.ai/tcp/9000/p2p/16Uiu2HAkw1ypeQYQbRFV5hKUxGRHocwU5ohmVmCnyJNg36tnPFdx /dns4/acn.fetch.ai/tcp/9001/p2p/16Uiu2HAmVWnopQAqq4pniYLw44VRvYxBUoRHqjz1Hh2SoCyjbyRW ``` diff --git a/docs/prometheus.md b/docs/prometheus.md index cdfac52b15..4964de8d7c 100644 --- a/docs/prometheus.md +++ b/docs/prometheus.md @@ -1,7 +1,7 @@ AEAs can create and update prometheus metrics for remote monitoring by sending messages to the prometheus connection `fetchai/prometheus:0.1.0`. To see this working in an agent, fetch and run the `coin_price_feed` agent and check `localhost:9090/metrics` to see the latest values of the metrics `num_retrievals` and `num_requests`: -```bash +``` bash aea fetch fetchai/coin_price_feed:0.2.0 cd coin_price_feed aea install @@ -15,7 +15,7 @@ To use this connection, add a model `prometheus_dialogues` to your skill to hand
    Click here for example -```python +``` python class PrometheusDialogues(Model, BasePrometheusDialogues): """The dialogues class keeps track of all prometheus dialogues.""" @@ -51,7 +51,7 @@ class PrometheusDialogues(Model, BasePrometheusDialogues):
    Then configure your metrics in the `skill.yaml` file. For example (from the `coin_price` skill): -```yaml +``` yaml models: prometheus_dialogues: args: @@ -69,7 +69,7 @@ models: ``` Add a metric `metric_name` of type `metric_type` {`Gauge`, `Counter`, ...} and description `description` by sending a message with performative `ADD_METRIC` to the prometheus connection: -```python +``` python def add_prometheus_metric( self, metric_name: str, @@ -107,14 +107,14 @@ def add_prometheus_metric( self.context.outbox.put_message(message=message, context=envelope_context) ``` where `PROM_CONNECTION_ID` should be imported to your skill as follows: -```python +``` python from packages.fetchai.connections.prometheus.connection import ( PUBLIC_ID as PROM_CONNECTION_ID, ) ``` Update metric `metric_name` with update function `update_func` {`inc`, `set`, `observe`, ...} and value `value` by sending a message with performative `UPDATE_METRIC` to the prometheus connection: -```python +``` python def update_prometheus_metric( self, metric_name: str, update_func: str, value: float, labels: Dict[str, str], ) -> None: @@ -149,7 +149,7 @@ def update_prometheus_metric( ``` Initialize the metrics from the configuration file in the behaviour setup: -```python +``` python def setup(self) -> None: """Implement the setup of the behaviour""" prom_dialogues = cast(PrometheusDialogues, self.context.prometheus_dialogues) @@ -163,7 +163,7 @@ def setup(self) -> None: Then call the `update_prometheus_metric` function from the appropriate places. For example, the following code in `handlers.py` for the `coin_price` skill updates the number of http requests served: -```python +``` python if self.context.prometheus_dialogues.enabled: self.context.behaviours.coin_price_behaviour.update_prometheus_metric( "num_requests", "inc", 1.0, {} @@ -175,7 +175,7 @@ Finally, you can add a `PrometheusHandler` to your skill to process response mes
    Click here for example -```python +``` python class PrometheusHandler(Handler): """This class handles responses from the prometheus server.""" diff --git a/docs/protocol.md b/docs/protocol.md index 831063c662..cf12926660 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -147,7 +147,7 @@ We show some example messages below: my_dialogue_reference = "a_unique_register_service_dialogue_reference" ``` and a description of the service we would like to register, for instance -```python +``` python from aea.helpers.search.models import Description my_service_data = {"country": "UK", "city": "Cambridge"} @@ -157,7 +157,7 @@ my_service_description = Description( ) ``` where we use, for instance -```python +``` python from aea.helpers.search.generic import GenericDataModel data_model_name = "location" diff --git a/docs/quickstart.md b/docs/quickstart.md index ecd75ac296..fb347062a1 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -50,19 +50,19 @@ To use the image you will first have to pull it and than run it with your curren To pull: -```bash +``` bash docker pull fetchai/aea-user:latest ``` To run the image on Linux and MacOs: -```bash +``` bash docker run -it -v $(pwd):/agents --workdir=/agents fetchai/aea-user:latest ``` And on Windows: -```bash +``` bash docker run -it -v %cd%:/agents --workdir=/agents fetchai/aea-user:latest ``` diff --git a/docs/skill-guide.md b/docs/skill-guide.md index b95f032272..031a42082a 100644 --- a/docs/skill-guide.md +++ b/docs/skill-guide.md @@ -438,7 +438,17 @@ We first create the private key for the service provider AEA based on the networ ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` + +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates ``` Then we run the aea: @@ -869,7 +879,17 @@ First, create the private key for the search AEA based on the network you want t ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` + +Next, create a private key used to secure the AEA's communications: +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` + +Finally, certify the key for use by the connections that request that: +``` bash +aea issue-certificates ``` Then, in the search AEA, run this command (replace `SOME_ADDRESS` with the correct value as described above): diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-aries-cloud-agent-demo.md b/tests/test_docs/test_bash_yaml/md_files/bash-aries-cloud-agent-demo.md index a173eb3438..d5545334b9 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-aries-cloud-agent-demo.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-aries-cloud-agent-demo.md @@ -39,13 +39,15 @@ aea config set --type int vendor.fetchai.connections.webhook.config.webhook_port ``` bash aea config set vendor.fetchai.connections.webhook.config.webhook_url_path /webhooks/topic/{topic}/ ``` -``` yaml -config: - delegate_uri: 127.0.0.1:11000 - entry_peers: [] - local_uri: 127.0.0.1:7000 - log_file: libp2p_node.log - public_uri: 127.0.0.1:7000 +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11000", + "entry_peers": [], + "local_uri": "127.0.0.1:7000", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:7000" +}' ``` ``` bash aea install @@ -79,13 +81,15 @@ aea config set --type int vendor.fetchai.connections.webhook.config.webhook_port ``` bash aea config set vendor.fetchai.connections.webhook.config.webhook_url_path /webhooks/topic/{topic}/ ``` -``` yaml -config: - delegate_uri: 127.0.0.1:11001 - entry_peers: ['SOME_ADDRESS'] - local_uri: 127.0.0.1:7001 - log_file: libp2p_node.log - public_uri: 127.0.0.1:7001 +``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:7001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:7001" +}' ``` ``` bash aea install diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-cli-how-to.md b/tests/test_docs/test_bash_yaml/md_files/bash-cli-how-to.md index d125e1fbee..d30ad741a6 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-cli-how-to.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-cli-how-to.md @@ -7,7 +7,7 @@ pip install aea[all] ``` bash pip install aea[all] --force --no-cache-dir ``` -```bash +``` bash aea ``` ``` bash diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-erc1155-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-erc1155-skills.md index 318d41ff2b..691c2653d7 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-erc1155-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-erc1155-skills.md @@ -14,6 +14,12 @@ aea add skill fetchai/erc1155_deploy:0.19.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/contract_api:0.9.0": "fetchai/ledger:0.11.0", + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` ``` bash aea config set agent.default_ledger ethereum @@ -23,8 +29,11 @@ aea generate-key ethereum aea add-key ethereum ethereum_private_key.txt ``` ``` bash -aea generate-key fetchai -aea add-key fetchai fetchai_private_key.txt --connection +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates ``` ``` bash aea fetch fetchai/erc1155_client:0.20.0 @@ -42,8 +51,6 @@ aea add skill fetchai/erc1155_client:0.18.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 -``` -``` bash aea config set --type dict agent.default_routing \ '{ "fetchai/contract_api:0.9.0": "fetchai/ledger:0.11.0", @@ -59,8 +66,11 @@ aea generate-key ethereum aea add-key ethereum ethereum_private_key.txt ``` ``` bash -aea generate-key fetchai -aea add-key fetchai fetchai_private_key.txt --connection +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates ``` ``` bash docker run -p 8545:8545 trufflesuite/ganache-cli:latest --verbose --gasPrice=0 --gasLimit=0x1fffffffffffff --account="$(cat erc1155_deployer/ethereum_private_key.txt),1000000000000000000000" --account="$(cat erc1155_client/ethereum_private_key.txt),1000000000000000000000" diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills-step-by-step.md b/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills-step-by-step.md index 6d67e9f480..da2270d95e 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills-step-by-step.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills-step-by-step.md @@ -42,7 +42,20 @@ aea fingerprint skill fetchai/generic_buyer:0.1.0 ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates +``` +``` bash +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` ``` bash aea generate-wealth fetchai --sync @@ -68,6 +81,16 @@ aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +``` bash aea run ``` ``` bash @@ -232,14 +255,6 @@ models: dependencies: {} ``` ``` yaml -addr: ${OEF_ADDR: 127.0.0.1} -``` -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml config: delegate_uri: 127.0.0.1:11001 entry_peers: ['SOME_ADDRESS'] diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills.md b/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills.md index 75d4e2eac7..85733b28a9 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-generic-skills.md @@ -14,6 +14,11 @@ aea add skill fetchai/generic_seller:0.18.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` ``` bash aea fetch fetchai/generic_buyer:0.17.0 --alias my_buyer_aea @@ -31,6 +36,11 @@ aea add skill fetchai/generic_buyer:0.18.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` ``` bash aea generate-key fetchai @@ -70,6 +80,16 @@ aea config set vendor.fetchai.skills.generic_buyer.is_abstract false --type bool aea run ``` ``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +``` bash aea run ``` ``` bash @@ -78,16 +98,6 @@ aea delete my_seller_aea aea delete my_buyer_aea ``` ``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml models: ... strategy: @@ -131,9 +141,13 @@ models: class_name: GenericStrategy ``` ``` yaml +--- +public_id: fetchai/p2p_libp2p:0.13.0 +type: connection config: delegate_uri: 127.0.0.1:11001 - entry_peers: ['SOME_ADDRESS'] + entry_peers: + - SOME_ADDRESS local_uri: 127.0.0.1:9001 log_file: libp2p_node.log public_uri: 127.0.0.1:9001 diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-orm-integration.md b/tests/test_docs/test_bash_yaml/md_files/bash-orm-integration.md index fccc093912..31f98f8176 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-orm-integration.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-orm-integration.md @@ -14,6 +14,11 @@ aea add skill fetchai/thermometer:0.17.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` ``` bash aea fetch fetchai/thermometer_client:0.18.0 --alias my_thermometer_client @@ -31,16 +36,33 @@ aea add skill fetchai/thermometer_client:0.17.0 aea install aea build aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea config set --type dict agent.default_routing \ +'{ + "fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0", + "fetchai/oef_search:0.11.0": "fetchai/soef:0.14.0" +}' ``` ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates ``` ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates ``` ``` bash aea generate-wealth fetchai @@ -59,6 +81,16 @@ aea fingerprint skill {YOUR_AUTHOR_HANDLE}/thermometer:0.1.0 aea run ``` ``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": ["SOME_ADDRESS"], + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +``` bash aea run ``` ``` bash @@ -67,16 +99,6 @@ aea delete my_thermometer_aea aea delete my_thermometer_client ``` ``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml -default_routing: - fetchai/ledger_api:0.8.0: fetchai/ledger:0.11.0 - fetchai/oef_search:0.11.0: fetchai/soef:0.14.0 -``` -``` yaml models: ... strategy: @@ -122,9 +144,13 @@ models: class_name: Strategy ``` ``` yaml +--- +public_id: fetchai/p2p_libp2p:0.13.0 +type: connection config: delegate_uri: 127.0.0.1:11001 - entry_peers: ['SOME_ADDRESS'] + entry_peers: + - SOME_ADDRESS local_uri: 127.0.0.1:9001 log_file: libp2p_node.log public_uri: 127.0.0.1:9001 diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-p2p-connection.md b/tests/test_docs/test_bash_yaml/md_files/bash-p2p-connection.md index 3a9360da5a..06990f1474 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-p2p-connection.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-p2p-connection.md @@ -12,6 +12,16 @@ aea add connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 ``` ``` bash +aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ +'{ + "delegate_uri": "127.0.0.1:11001", + "entry_peers": MULTI_ADDRESSES, + "local_uri": "127.0.0.1:9001", + "log_file": "libp2p_node.log", + "public_uri": "127.0.0.1:9001" +}' +``` +``` bash aea run --connections fetchai/p2p_libp2p:0.13.0 ``` ``` bash @@ -45,27 +55,6 @@ aea run --connections fetchai/p2p_libp2p:0.13.0 --entry-peers-maddrs ... ``` ``` yaml -config: - delegate_uri: 127.0.0.1:11001 - entry_peers: MULTI_ADDRESSES - local_uri: 127.0.0.1:9001 - log_file: libp2p_node.log - public_uri: 127.0.0.1:9001 -``` -``` yaml -default_routing: - ? "fetchai/oef_search:0.11.0" - : "fetchai/oef:0.14.0" -``` -``` yaml -config: - delegate_uri: 127.0.0.1:11001 - entry_peers: MULTI_ADDRESSES - local_uri: 127.0.0.1:9001 - log_file: libp2p_node.log - public_uri: 127.0.0.1:9001 -``` -```yaml /dns4/acn.fetch.ai/tcp/9000/p2p/16Uiu2HAkw1ypeQYQbRFV5hKUxGRHocwU5ohmVmCnyJNg36tnPFdx /dns4/acn.fetch.ai/tcp/9001/p2p/16Uiu2HAmVWnopQAqq4pniYLw44VRvYxBUoRHqjz1Hh2SoCyjbyRW ``` diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-quickstart.md b/tests/test_docs/test_bash_yaml/md_files/bash-quickstart.md index 1d3c5ee3ca..36323194f0 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-quickstart.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-quickstart.md @@ -1,10 +1,10 @@ -```bash +``` bash docker pull fetchai/aea-user:latest ``` - ```bash + ``` bash docker run -it -v $(pwd):/agents --workdir=/agents fetchai/aea-user:latest ``` - ```bash + ``` bash docker run -it -v %cd%:/agents --workdir=/agents fetchai/aea-user:latest ``` ``` bash diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-skill-guide.md b/tests/test_docs/test_bash_yaml/md_files/bash-skill-guide.md index c4e6c7c3dc..7d6c5bc953 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-skill-guide.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-skill-guide.md @@ -25,7 +25,13 @@ aea fetch fetchai/simple_service_registration:0.18.0 && cd simple_service_regist ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates ``` ``` bash aea run @@ -33,7 +39,13 @@ aea run ``` bash aea generate-key fetchai aea add-key fetchai fetchai_private_key.txt -aea add-key fetchai fetchai_private_key.txt --connection +``` +``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates ``` ``` bash aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ diff --git a/tests/test_docs/test_skill_guide/test_skill_guide.py b/tests/test_docs/test_skill_guide/test_skill_guide.py index e68a73cfc4..0c42dacbd0 100644 --- a/tests/test_docs/test_skill_guide/test_skill_guide.py +++ b/tests/test_docs/test_skill_guide/test_skill_guide.py @@ -186,7 +186,7 @@ def test_update_skill_and_run(self): missing_strings = self.missing_from_output( simple_service_registration_aea_process, check_strings, - timeout=240, + timeout=30, is_terminating=False, ) assert ( diff --git a/tests/test_packages/test_skills_integration/test_erc1155.py b/tests/test_packages/test_skills_integration/test_erc1155.py index e15132acb7..b664fdd11c 100644 --- a/tests/test_packages/test_skills_integration/test_erc1155.py +++ b/tests/test_packages/test_skills_integration/test_erc1155.py @@ -207,7 +207,7 @@ def test_generic(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - client_aea_process, check_strings, timeout=240, is_terminating=False + client_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] diff --git a/tests/test_packages/test_skills_integration/test_ml_skills.py b/tests/test_packages/test_skills_integration/test_ml_skills.py index 69f9886c8b..4e93b6c483 100644 --- a/tests/test_packages/test_skills_integration/test_ml_skills.py +++ b/tests/test_packages/test_skills_integration/test_ml_skills.py @@ -150,7 +150,7 @@ def test_ml_skills(self, pytestconfig): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - data_provider_aea_process, check_strings, timeout=240, is_terminating=False + data_provider_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -170,7 +170,7 @@ def test_ml_skills(self, pytestconfig): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - model_trainer_aea_process, check_strings, timeout=240, is_terminating=False + model_trainer_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -335,7 +335,7 @@ def test_ml_skills(self, pytestconfig): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - data_provider_aea_process, check_strings, timeout=240, is_terminating=False + data_provider_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -355,7 +355,7 @@ def test_ml_skills(self, pytestconfig): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - model_trainer_aea_process, check_strings, timeout=240, is_terminating=False + model_trainer_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] diff --git a/tests/test_packages/test_skills_integration/test_tac.py b/tests/test_packages/test_skills_integration/test_tac.py index 1a5e6a7771..3fe2cd4367 100644 --- a/tests/test_packages/test_skills_integration/test_tac.py +++ b/tests/test_packages/test_skills_integration/test_tac.py @@ -207,7 +207,7 @@ def test_tac(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - tac_controller_process, check_strings, timeout=240, is_terminating=False + tac_controller_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -231,7 +231,7 @@ def test_tac(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - tac_aea_one_process, check_strings, timeout=240, is_terminating=False + tac_aea_one_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -244,7 +244,7 @@ def test_tac(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - tac_aea_two_process, check_strings, timeout=240, is_terminating=False + tac_aea_two_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -552,7 +552,7 @@ def test_tac(self): "found the TAC controller. Registering...", ) missing_strings = self.missing_from_output( - tac_aea_one_process, check_strings, timeout=240, is_terminating=False + tac_aea_one_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -568,7 +568,7 @@ def test_tac(self): "found the TAC controller. Registering...", ) missing_strings = self.missing_from_output( - tac_aea_two_process, check_strings, timeout=240, is_terminating=False + tac_aea_two_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] diff --git a/tests/test_packages/test_skills_integration/test_thermometer.py b/tests/test_packages/test_skills_integration/test_thermometer.py index 28d1a6b717..a8f7615a4f 100644 --- a/tests/test_packages/test_skills_integration/test_thermometer.py +++ b/tests/test_packages/test_skills_integration/test_thermometer.py @@ -142,7 +142,7 @@ def test_thermometer(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - thermometer_aea_process, check_strings, timeout=240, is_terminating=False + thermometer_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -327,7 +327,7 @@ def test_thermometer(self): LIBP2P_SUCCESS_MESSAGE, ) missing_strings = self.missing_from_output( - thermometer_aea_process, check_strings, timeout=240, is_terminating=False + thermometer_aea_process, check_strings, timeout=30, is_terminating=False ) assert ( missing_strings == [] @@ -347,7 +347,7 @@ def test_thermometer(self): missing_strings = self.missing_from_output( thermometer_client_aea_process, check_strings, - timeout=240, + timeout=30, is_terminating=False, ) assert ( From eaa7609914345d051050789ba7d2b6fb449b8e60 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Thu, 31 Dec 2020 20:24:09 +0000 Subject: [PATCH 114/204] Add missing file --- aea/crypto/ethereum.py | 2 +- tests/test_helpers/test_acn.py | 132 +++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 tests/test_helpers/test_acn.py diff --git a/aea/crypto/ethereum.py b/aea/crypto/ethereum.py index 595c55b8b4..f83344d587 100644 --- a/aea/crypto/ethereum.py +++ b/aea/crypto/ethereum.py @@ -377,7 +377,7 @@ def recover_verifying_keys_from_message( :param is_deprecated_mode: if the deprecated signing was used :return: the recovered public keys """ - raise NotImplementedError + raise NotImplementedError # pragma: no cover @staticmethod def get_hash(message: bytes) -> str: diff --git a/tests/test_helpers/test_acn.py b/tests/test_helpers/test_acn.py new file mode 100644 index 0000000000..da7516db04 --- /dev/null +++ b/tests/test_helpers/test_acn.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the tests for acn helper module.""" + +import pytest + +from aea.configurations.constants import DEFAULT_LEDGER +from aea.crypto.registries import make_crypto +from aea.helpers.acn.agent_record import AgentRecord, signature_from_cert_request +from aea.helpers.base import CertRequest + +from tests.conftest import _process_cert + + +def test_signature_from_cert_request_errors(): + agent_key_1 = make_crypto(DEFAULT_LEDGER) + agent_key_2 = make_crypto(DEFAULT_LEDGER) + + peer_public_key = make_crypto(DEFAULT_LEDGER).public_key + + cert_path = "test_acn_cert.txt" + + cert = CertRequest( + peer_public_key, + "test_service", + DEFAULT_LEDGER, + "2021-01-01", + "2022-01-01", + cert_path, + ) + _process_cert(agent_key_1, cert) + + # success + _, signer_public_key = signature_from_cert_request( + cert, peer_public_key, agent_key_1.address + ) + assert signer_public_key == agent_key_1.public_key + + # error: wrong signer + with pytest.raises(Exception): + signature_from_cert_request(cert, peer_public_key, agent_key_2.address) + + +def test_agent_record_errors(): + agent_key_1 = make_crypto(DEFAULT_LEDGER) + agent_key_2 = make_crypto(DEFAULT_LEDGER) + + peer_public_key_1 = make_crypto(DEFAULT_LEDGER).public_key + peer_public_key_2 = make_crypto(DEFAULT_LEDGER).public_key + + cert_path = "test_acn_cert.txt" + service_id = "test_acn_service" + + cert = CertRequest( + peer_public_key_1, + "test_service", + DEFAULT_LEDGER, + "2021-01-01", + "2022-01-01", + cert_path, + ) + _process_cert(agent_key_1, cert) + signature, _ = signature_from_cert_request( + cert, peer_public_key_1, agent_key_1.address + ) + + # success + agent_record = AgentRecord( + agent_key_1.address, + agent_key_1.public_key, + peer_public_key_1, + signature, + service_id, + ) + agent_record.check_validity(agent_key_1.address, peer_public_key_1) + assert ( + agent_record.address == agent_key_1.address + and agent_record.public_key == agent_key_1.public_key + and agent_record.peer_public_key == peer_public_key_1 + and agent_record.signature == signature + and agent_record.service_id == service_id + ) + + # error: wrong agent address + with pytest.raises(Exception): + agent_record.check_validity(agent_key_2.address, peer_public_key_1) + + # error: wrong peer + with pytest.raises(Exception): + agent_record.check_validity(agent_key_1.address, peer_public_key_2) + + # error: agent address and public key don't match + agent_record = AgentRecord( + agent_key_2.address, + agent_key_1.public_key, + peer_public_key_1, + signature, + service_id, + ) + with pytest.raises(Exception): + agent_record.check_validity(agent_key_2.address, peer_public_key_1) + + # error: invalid signature + _process_cert(agent_key_2, cert) + signature, _ = signature_from_cert_request( + cert, peer_public_key_1, agent_key_2.address + ) + agent_record = AgentRecord( + agent_key_1.address, + agent_key_1.public_key, + peer_public_key_1, + signature, + service_id, + ) + with pytest.raises(Exception): + agent_record.check_validity(agent_key_1.address, peer_public_key_1) From 2e845e4fe4b8a1949ed3077bf7d8de39c99dd240 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Thu, 31 Dec 2020 20:36:17 +0000 Subject: [PATCH 115/204] Address flake8 --- aea/helpers/acn/agent_record.py | 2 +- tests/test_helpers/test_acn.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/aea/helpers/acn/agent_record.py b/aea/helpers/acn/agent_record.py index 65b1862a82..3509e38b30 100644 --- a/aea/helpers/acn/agent_record.py +++ b/aea/helpers/acn/agent_record.py @@ -110,7 +110,7 @@ def service_id(self) -> str: """Get record service id""" return self._service_id - def __str__(self): + def __str__(self): # pragma: no cover """Get string representation.""" return f"(address={self.address}, public_key={self.public_key}, peer_public_key={self.peer_public_key}, signature={self.signature})" diff --git a/tests/test_helpers/test_acn.py b/tests/test_helpers/test_acn.py index da7516db04..2deb0db026 100644 --- a/tests/test_helpers/test_acn.py +++ b/tests/test_helpers/test_acn.py @@ -29,6 +29,7 @@ def test_signature_from_cert_request_errors(): + """Test signature and public key proper retrieval from a CertRequest""" agent_key_1 = make_crypto(DEFAULT_LEDGER) agent_key_2 = make_crypto(DEFAULT_LEDGER) @@ -58,6 +59,7 @@ def test_signature_from_cert_request_errors(): def test_agent_record_errors(): + """Test AgentRecord check_validity""" agent_key_1 = make_crypto(DEFAULT_LEDGER) agent_key_2 = make_crypto(DEFAULT_LEDGER) From 7fff2ab032f069583ac83caba4407f0efbd020f0 Mon Sep 17 00:00:00 2001 From: Lokman Rahmani Date: Thu, 31 Dec 2020 21:36:43 +0000 Subject: [PATCH 116/204] Fix missing coverage --- aea/helpers/acn/agent_record.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aea/helpers/acn/agent_record.py b/aea/helpers/acn/agent_record.py index 3509e38b30..eaf7e74157 100644 --- a/aea/helpers/acn/agent_record.py +++ b/aea/helpers/acn/agent_record.py @@ -46,7 +46,7 @@ def signature_from_cert_request( public_keys = FetchAIHelper.recover_verifying_keys_from_message( cert.get_message(message), signature ) - if len(public_keys) == 0: + if len(public_keys) == 0: # pragma: no cover raise Exception("Malformed signature") addresses = [ FetchAIHelper.get_address_from_public_key(public_key) From 2771b6084bb7d3564a1755b1d3039f397607b8f2 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Fri, 1 Jan 2021 12:27:12 +0300 Subject: [PATCH 117/204] local registry sync --- aea/cli/core.py | 3 +- aea/cli/local_registry_sync.py | 140 +++++++++++++++++++ aea/configurations/base.py | 5 + aea/configurations/constants.py | 1 + aea/configurations/loader.py | 14 +- tests/test_cli/test_local_registry_update.py | 88 ++++++++++++ tests/test_cli/test_misc.py | 73 +++++----- 7 files changed, 280 insertions(+), 44 deletions(-) create mode 100644 aea/cli/local_registry_sync.py create mode 100644 tests/test_cli/test_local_registry_update.py diff --git a/aea/cli/core.py b/aea/cli/core.py index 16655e4335..2e1413a470 100644 --- a/aea/cli/core.py +++ b/aea/cli/core.py @@ -17,7 +17,6 @@ # limitations under the License. # # ------------------------------------------------------------------------------ - """Core definitions for the AEA command-line tool.""" import click @@ -45,6 +44,7 @@ from aea.cli.issue_certificates import issue_certificates from aea.cli.launch import launch from aea.cli.list import list_command as _list +from aea.cli.local_registry_sync import local_registry_sync from aea.cli.login import login from aea.cli.logout import logout from aea.cli.publish import publish @@ -151,5 +151,6 @@ def _init_gui() -> None: cli.add_command(run) cli.add_command(scaffold) cli.add_command(search) +cli.add_command(local_registry_sync) cli.add_command(transfer) cli.add_command(upgrade) diff --git a/aea/cli/local_registry_sync.py b/aea/cli/local_registry_sync.py new file mode 100644 index 0000000000..0dc92dfc5e --- /dev/null +++ b/aea/cli/local_registry_sync.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""Implementation of the 'aea sync-local-registry' subcommand.""" +import os +import shutil +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Generator, Tuple, Union + +import click + +from aea.cli.fingerprint import determine_package_type_for_directory +from aea.cli.registry.add import fetch_package +from aea.cli.registry.utils import get_package_meta +from aea.cli.utils.context import Context +from aea.cli.utils.decorators import pass_ctx +from aea.configurations.data_types import PackageId, PackageType, PublicId +from aea.configurations.loader import load_component_configuration + + +PACKAGES_DIRS = [i.to_plural() for i in PackageType] + + +@click.command() +@pass_ctx +def local_registry_sync(ctx: Context): + """Upgrade the local package registry.""" + do_local_registry_update(ctx.cwd) + + +def do_local_registry_update(base_dir: Union[str, Path]) -> None: + """ + Perform local registry update. + + :param base_dir: root directory of the local registry. + + :return: None + """ + for package_id, package_dir in enlist_packages(base_dir): + current_public_id = package_id.public_id + latest_public_id = get_package_latest_public_id(package_id) + if not ( # pylint: disable=superfluous-parens + current_public_id < latest_public_id + ): + click.echo(f"{package_id} is the latest version.") + continue + click.echo(f"Updating to {latest_public_id}.") + replace_package(str(package_id.package_type), latest_public_id, package_dir) + + +def replace_package( + package_type: str, public_id: PublicId, package_dir: Union[Path, str] +) -> None: + """ + Download, extract and replace exists package. + + :param package_type: str. + :param public_id: pacakge bulic id to download + :param: package_dir: target package dir + + :return: None + """ + with TemporaryDirectory() as tmp_dir: + new_package_dir = os.path.join(tmp_dir, public_id.name) + os.mkdir(new_package_dir) + fetch_package( + package_type, public_id=public_id, cwd=tmp_dir, dest=new_package_dir + ) + shutil.rmtree(package_dir) + shutil.move(new_package_dir, package_dir) + + +def get_package_latest_public_id(package_id: PackageId) -> PublicId: + """ + Get package latest package id from the remote repo. + + :param package_id: id of the package to check + + :return: package id of the latest package in remote repo + """ + package_meta = get_package_meta( + str(package_id.package_type), package_id.public_id.to_latest() + ) + return PublicId.from_str(package_meta["public_id"]) + + +def enlist_packages( + base_dir: Union[Path, str] +) -> Generator[Tuple[PackageId, Union[Path, str]], None, None]: + """ + Generate list of the packages in local repo directory. + + :param base_dir: path or str of the local repo. + + :return: generator with Tuple of package_id and package diurectory. + """ + for author in os.listdir(base_dir): + author_dir = os.path.join(base_dir, author) + if not os.path.isdir(author_dir): + continue # pragma: nocover + for package_type_plural in os.listdir(author_dir): + if package_type_plural not in PACKAGES_DIRS: + continue # pragma: nocover + package_type_dir = os.path.join(author_dir, package_type_plural) + if not os.path.isdir(package_type_dir): + continue # pragma: nocover + for package_name in os.listdir(package_type_dir): + package_dir = os.path.join(package_type_dir, package_name) + if not os.path.isdir(package_dir): + continue # pragma: nocover + try: + package_type = determine_package_type_for_directory( + Path(package_dir) + ) + if package_type.to_plural() != package_type_plural: + # incorrect package placing + continue # pragma: nocover + config = load_component_configuration( + package_type, Path(package_dir), skip_consistency_check=True + ) + yield (config.package_id, package_dir) + except ValueError: # pragma: nocover + # can not determine package type, not a package? just skip + pass # pragma: nocover diff --git a/aea/configurations/base.py b/aea/configurations/base.py index eca468f924..04bebbcac1 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -287,6 +287,11 @@ def directory(self, directory: Path) -> None: raise ValueError("Directory already set") self._directory = directory + @property + def package_id(self) -> PackageId: + """Get package id.""" + return PackageId(package_type=self.package_type, public_id=self.public_id) + @staticmethod def _parse_aea_version_specifier(aea_version_specifiers: str) -> SpecifierSet: try: diff --git a/aea/configurations/constants.py b/aea/configurations/constants.py index b2577b0e1f..ab284435ac 100644 --- a/aea/configurations/constants.py +++ b/aea/configurations/constants.py @@ -85,4 +85,5 @@ DEFAULT_PROTOCOL_CONFIG_FILE: PROTOCOL, DEFAULT_CONNECTION_CONFIG_FILE: CONNECTION, DEFAULT_CONTRACT_CONFIG_FILE: CONTRACT, + DEFAULT_AEA_CONFIG_FILE: AGENT, } # type: Dict[str, str] diff --git a/aea/configurations/loader.py b/aea/configurations/loader.py index 2ce50cc38d..521160facf 100644 --- a/aea/configurations/loader.py +++ b/aea/configurations/loader.py @@ -308,18 +308,20 @@ def from_package_type( def load_component_configuration( - component_type: ComponentType, + component_type: Union[ComponentType, PackageType], directory: Path, skip_consistency_check: bool = False, ) -> "ComponentConfiguration": """ Load configuration and check that it is consistent against the directory. - :param component_type: the component type. + :param component_type: the component or package type. :param directory: the root of the package :param skip_consistency_check: if True, the consistency check are skipped. :return: the configuration object. """ + if isinstance(component_type, ComponentType): + component_type = component_type.to_configuration_type() configuration_object = _load_configuration_object(component_type, directory) if not skip_consistency_check: configuration_object._check_configuration_consistency( # pylint: disable=protected-access @@ -329,7 +331,7 @@ def load_component_configuration( def _load_configuration_object( - component_type: ComponentType, directory: Path + package_type: PackageType, directory: Path ) -> ComponentConfiguration: """ Load the configuration object, without consistency checks. @@ -339,9 +341,7 @@ def _load_configuration_object( :return: the configuration object. :raises FileNotFoundError: if the configuration file is not found. """ - configuration_loader = ConfigLoader.from_configuration_type( - component_type.to_configuration_type() - ) + configuration_loader = ConfigLoader.from_configuration_type(package_type) configuration_filename = ( configuration_loader.configuration_class.default_configuration_filename ) @@ -352,7 +352,7 @@ def _load_configuration_object( except FileNotFoundError: raise FileNotFoundError( "{} configuration not found: {}".format( - component_type.value.capitalize(), configuration_filepath + package_type.value.capitalize(), configuration_filepath ) ) return configuration_object diff --git a/tests/test_cli/test_local_registry_update.py b/tests/test_cli/test_local_registry_update.py new file mode 100644 index 0000000000..3b5a41b6ed --- /dev/null +++ b/tests/test_cli/test_local_registry_update.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This test module contains the tests for the `aea local-registry-sync""" +import os +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from aea.cli.core import cli +from aea.cli.local_registry_sync import enlist_packages +from aea.cli.registry.add import fetch_package +from aea.configurations.data_types import PackageId, PackageType, PublicId +from aea.helpers.base import cd +from aea.test_tools.click_testing import CliRunner + + +def test_local_registry_update(): + """Test local-registry-sync cli command.""" + PACKAGES = [ + PackageId(PackageType.CONNECTION, PublicId("fetchai", "local", "0.11.0")), + PackageId(PackageType.AGENT, PublicId("fetchai", "my_first_aea", "0.10.0")), + ] + with TemporaryDirectory() as tmp_dir: + for package_id in PACKAGES: + package_dir = os.path.join( + tmp_dir, + package_id.public_id.author, + str(package_id.package_type.to_plural()), + package_id.public_id.name, + ) + os.makedirs(package_dir) + fetch_package( + str(package_id.package_type), + public_id=package_id.public_id, + cwd=tmp_dir, + dest=package_dir, + ) + + assert set(PACKAGES) == set([i[0] for i in enlist_packages(tmp_dir)]) + + runner = CliRunner() + with cd(tmp_dir): + # check intention to upgrade + with patch( + "aea.cli.local_registry_sync.replace_package" + ) as replace_package_mock: + result = runner.invoke( + cli, "local-registry-sync", catch_exceptions=False + ) + assert result.exit_code == 0, result.stdout + assert replace_package_mock.call_count == len(PACKAGES) + + # do actual upgrade + result = runner.invoke(cli, "local-registry-sync", catch_exceptions=False) + assert result.exit_code == 0, result.stdout + + # check next update will do nothing + with patch( + "aea.cli.local_registry_sync.replace_package" + ) as replace_package_mock: + result = runner.invoke( + cli, "local-registry-sync", catch_exceptions=False + ) + assert result.exit_code == 0, result.stdout + assert replace_package_mock.call_count == 0 + + def sort_(packages): + return sorted(packages, key=lambda x: str(x)) + + new_packages = [i[0] for i in enlist_packages(tmp_dir)] + + for new_package, old_package in zip(sort_(new_packages), sort_(PACKAGES)): + assert new_package.public_id > old_package.public_id diff --git a/tests/test_cli/test_misc.py b/tests/test_cli/test_misc.py index f6aed6bdb3..fa42e1838b 100644 --- a/tests/test_cli/test_misc.py +++ b/tests/test_cli/test_misc.py @@ -60,41 +60,42 @@ def test_flag_help(): --help Show this message and exit. Commands: - add Add a package to the agent. - add-key Add a private key to the wallet of the agent. - build Build the agent and its components. - config Read or modify a configuration of the agent. - create Create a new agent. - delete Delete an agent. - eject Eject a vendor package of the agent. - fetch Fetch an agent from the registry. - fingerprint Fingerprint a non-vendor package of the agent. - freeze Get the dependencies of the agent. - generate Generate a package for the agent. - generate-key Generate a private key and place it in a file. - generate-wealth Generate wealth for the agent on a test network. - get-address Get the address associated with a private key of the... - get-multiaddress Get the multiaddress associated with a private key or... - get-wealth Get the wealth associated with the private key of the... - gui Run the CLI GUI. - init Initialize your AEA configurations. - install Install the dependencies of the agent. - interact Interact with the running agent via the stub connection. - issue-certificates Issue certificates for connections that require them. - launch Launch many agents at the same time. - list List the installed packages of the agent. - login Login to the registry account. - logout Logout from the registry account. - publish Publish the agent to the registry. - push Push a non-vendor package of the agent to the registry. - register Create a new registry account. - remove Remove a package from the agent. - remove-key Remove a private key from the wallet of the agent. - reset_password Reset the password of the registry account. - run Run the agent. - scaffold Scaffold a package for the agent. - search Search for packages in the registry. - transfer Transfer wealth associated with a private key of the... - upgrade Upgrade the packages of the agent. + add Add a package to the agent. + add-key Add a private key to the wallet of the agent. + build Build the agent and its components. + config Read or modify a configuration of the agent. + create Create a new agent. + delete Delete an agent. + eject Eject a vendor package of the agent. + fetch Fetch an agent from the registry. + fingerprint Fingerprint a non-vendor package of the agent. + freeze Get the dependencies of the agent. + generate Generate a package for the agent. + generate-key Generate a private key and place it in a file. + generate-wealth Generate wealth for the agent on a test network. + get-address Get the address associated with a private key of the... + get-multiaddress Get the multiaddress associated with a private key or... + get-wealth Get the wealth associated with the private key of the... + gui Run the CLI GUI. + init Initialize your AEA configurations. + install Install the dependencies of the agent. + interact Interact with the running agent via the stub... + issue-certificates Issue certificates for connections that require them. + launch Launch many agents at the same time. + list List the installed packages of the agent. + local-registry-sync Upgrade the local package registry. + login Login to the registry account. + logout Logout from the registry account. + publish Publish the agent to the registry. + push Push a non-vendor package of the agent to the registry. + register Create a new registry account. + remove Remove a package from the agent. + remove-key Remove a private key from the wallet of the agent. + reset_password Reset the password of the registry account. + run Run the agent. + scaffold Scaffold a package for the agent. + search Search for packages in the registry. + transfer Transfer wealth associated with a private key of the... + upgrade Upgrade the packages of the agent. """ ) From 66e4a7167f49dd73e6daab7ed7c97ae571087acb Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Fri, 1 Jan 2021 13:46:02 +0300 Subject: [PATCH 118/204] protocols updated cause year changed --- packages/fetchai/protocols/contract_api/__init__.py | 2 +- packages/fetchai/protocols/contract_api/dialogues.py | 2 +- packages/fetchai/protocols/contract_api/message.py | 2 +- packages/fetchai/protocols/contract_api/protocol.yaml | 8 ++++---- packages/fetchai/protocols/contract_api/serialization.py | 2 +- packages/fetchai/protocols/default/__init__.py | 2 +- packages/fetchai/protocols/default/dialogues.py | 2 +- packages/fetchai/protocols/default/message.py | 2 +- packages/fetchai/protocols/default/protocol.yaml | 8 ++++---- packages/fetchai/protocols/default/serialization.py | 2 +- packages/fetchai/protocols/fipa/__init__.py | 2 +- packages/fetchai/protocols/fipa/dialogues.py | 2 +- packages/fetchai/protocols/fipa/message.py | 2 +- packages/fetchai/protocols/fipa/protocol.yaml | 8 ++++---- packages/fetchai/protocols/fipa/serialization.py | 2 +- packages/fetchai/protocols/gym/__init__.py | 2 +- packages/fetchai/protocols/gym/dialogues.py | 2 +- packages/fetchai/protocols/gym/message.py | 2 +- packages/fetchai/protocols/gym/protocol.yaml | 8 ++++---- packages/fetchai/protocols/gym/serialization.py | 2 +- packages/fetchai/protocols/http/__init__.py | 2 +- packages/fetchai/protocols/http/dialogues.py | 2 +- packages/fetchai/protocols/http/message.py | 2 +- packages/fetchai/protocols/http/protocol.yaml | 8 ++++---- packages/fetchai/protocols/http/serialization.py | 2 +- packages/fetchai/protocols/ledger_api/__init__.py | 2 +- packages/fetchai/protocols/ledger_api/dialogues.py | 2 +- packages/fetchai/protocols/ledger_api/message.py | 2 +- packages/fetchai/protocols/ledger_api/protocol.yaml | 8 ++++---- packages/fetchai/protocols/ledger_api/serialization.py | 2 +- packages/fetchai/protocols/ml_trade/__init__.py | 2 +- packages/fetchai/protocols/ml_trade/dialogues.py | 2 +- packages/fetchai/protocols/ml_trade/message.py | 2 +- packages/fetchai/protocols/ml_trade/protocol.yaml | 8 ++++---- packages/fetchai/protocols/ml_trade/serialization.py | 2 +- packages/fetchai/protocols/oef_search/__init__.py | 2 +- packages/fetchai/protocols/oef_search/dialogues.py | 2 +- packages/fetchai/protocols/oef_search/message.py | 2 +- packages/fetchai/protocols/oef_search/protocol.yaml | 8 ++++---- packages/fetchai/protocols/oef_search/serialization.py | 2 +- packages/fetchai/protocols/prometheus/__init__.py | 2 +- packages/fetchai/protocols/prometheus/dialogues.py | 2 +- packages/fetchai/protocols/prometheus/message.py | 2 +- packages/fetchai/protocols/prometheus/protocol.yaml | 8 ++++---- packages/fetchai/protocols/prometheus/serialization.py | 2 +- packages/fetchai/protocols/register/__init__.py | 2 +- packages/fetchai/protocols/register/dialogues.py | 2 +- packages/fetchai/protocols/register/message.py | 2 +- packages/fetchai/protocols/register/protocol.yaml | 8 ++++---- packages/fetchai/protocols/register/serialization.py | 2 +- packages/fetchai/protocols/signing/__init__.py | 2 +- packages/fetchai/protocols/signing/dialogues.py | 2 +- packages/fetchai/protocols/signing/message.py | 2 +- packages/fetchai/protocols/signing/protocol.yaml | 8 ++++---- packages/fetchai/protocols/signing/serialization.py | 2 +- packages/fetchai/protocols/state_update/__init__.py | 2 +- packages/fetchai/protocols/state_update/dialogues.py | 2 +- packages/fetchai/protocols/state_update/message.py | 2 +- packages/fetchai/protocols/state_update/protocol.yaml | 8 ++++---- packages/fetchai/protocols/state_update/serialization.py | 2 +- packages/fetchai/protocols/tac/__init__.py | 2 +- packages/fetchai/protocols/tac/dialogues.py | 2 +- packages/fetchai/protocols/tac/message.py | 2 +- packages/fetchai/protocols/tac/protocol.yaml | 8 ++++---- packages/fetchai/protocols/tac/serialization.py | 2 +- tests/data/generator/t_protocol/__init__.py | 2 +- tests/data/generator/t_protocol/dialogues.py | 2 +- tests/data/generator/t_protocol/message.py | 2 +- tests/data/generator/t_protocol/protocol.yaml | 8 ++++---- tests/data/generator/t_protocol/serialization.py | 2 +- tests/data/generator/t_protocol_no_ct/__init__.py | 2 +- tests/data/generator/t_protocol_no_ct/dialogues.py | 2 +- tests/data/generator/t_protocol_no_ct/message.py | 2 +- tests/data/generator/t_protocol_no_ct/protocol.yaml | 8 ++++---- tests/data/generator/t_protocol_no_ct/serialization.py | 2 +- 75 files changed, 120 insertions(+), 120 deletions(-) diff --git a/packages/fetchai/protocols/contract_api/__init__.py b/packages/fetchai/protocols/contract_api/__init__.py index b93cfc3722..ba0df54703 100644 --- a/packages/fetchai/protocols/contract_api/__init__.py +++ b/packages/fetchai/protocols/contract_api/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/contract_api/dialogues.py b/packages/fetchai/protocols/contract_api/dialogues.py index 5cd81f82bf..1ab32fbcb9 100644 --- a/packages/fetchai/protocols/contract_api/dialogues.py +++ b/packages/fetchai/protocols/contract_api/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/contract_api/message.py b/packages/fetchai/protocols/contract_api/message.py index e11f74f4e7..ea28cf03af 100644 --- a/packages/fetchai/protocols/contract_api/message.py +++ b/packages/fetchai/protocols/contract_api/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/contract_api/protocol.yaml b/packages/fetchai/protocols/contract_api/protocol.yaml index b052a9392b..ba2e5c62d0 100644 --- a/packages/fetchai/protocols/contract_api/protocol.yaml +++ b/packages/fetchai/protocols/contract_api/protocol.yaml @@ -7,13 +7,13 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmXmHpD7cKDzuvSu7dWXuNZhXp9weNhghauDopSPexSgif - __init__.py: QmcZFuqoBkEx1fYytpLr7142vtXf9qh8cpeRYVZcaWdmrD + __init__.py: QmY2WGKdVE1vExjbKz46U9dgxgEhy2anyQG12ax2NNNuot contract_api.proto: QmSZpXyFMf2MNgVud8iPinaaerx1CManffgHNMx8FcD8jY contract_api_pb2.py: QmYEVg28AVRiLDkbddDto51bumh38gRuouagn6wspDtZVN custom_types.py: QmT51f1pdiAMCnBkLJiYA2aKfreRqkyohq32RiM7vGfjVu - dialogues.py: QmaSSvNewnT3LrR7GgrAwnH4ifpceSAuRMopxu1pNL9x8c - message.py: QmSZZ7jcXHpjQpTSwhafgUTCab7g4gowaj9SvxeuHQDQqL - serialization.py: QmPNTw6vXbdw9GMUwCCGyoHNxopVE1ipcp5DriSn3kGiB8 + dialogues.py: QmNk7CJhkAJ97e4jVmUFDGPUJaXCk3PQp6pzHcFv9hpsg5 + message.py: QmQC1WtSV4mpRz7FwJz6MFzdFzaKHjzCgdURzeTFjCmGRp + serialization.py: QmX2gUAStbfjSotF1k8fsTSLzsVadyVctYNN9fw9jCzgd9 fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/contract_api/serialization.py b/packages/fetchai/protocols/contract_api/serialization.py index e653e24335..ad8694c7b0 100644 --- a/packages/fetchai/protocols/contract_api/serialization.py +++ b/packages/fetchai/protocols/contract_api/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/default/__init__.py b/packages/fetchai/protocols/default/__init__.py index 3a99aef056..4cd1553362 100644 --- a/packages/fetchai/protocols/default/__init__.py +++ b/packages/fetchai/protocols/default/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/default/dialogues.py b/packages/fetchai/protocols/default/dialogues.py index 4352e589b3..568e9f5f10 100644 --- a/packages/fetchai/protocols/default/dialogues.py +++ b/packages/fetchai/protocols/default/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/default/message.py b/packages/fetchai/protocols/default/message.py index 2c6834cb8a..33689259c3 100644 --- a/packages/fetchai/protocols/default/message.py +++ b/packages/fetchai/protocols/default/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/default/protocol.yaml b/packages/fetchai/protocols/default/protocol.yaml index a793a6aea9..b60aad0243 100644 --- a/packages/fetchai/protocols/default/protocol.yaml +++ b/packages/fetchai/protocols/default/protocol.yaml @@ -7,13 +7,13 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmdxJPsHzYGpbNMWgXR1so3A87ns1vevGtzeh1WR2LoZ6G - __init__.py: QmRWWkHpiBGcYVXSvnxZLFD9QCyKcdtvbZ6GhLHLUWFa6G + __init__.py: Qmcv4DkrYV8HETwDvounAzdvXjFydt4jzHHWhu5uTk1ha8 custom_types.py: QmRcgwDdTxkSHyfF9eoMtsb5P5GJDm4oyLq5W6ZBko1MFU default.proto: QmNMoGYhsiiqQxZGGECeczvmKWnYYxyJ6TNTpvb679c4J1 default_pb2.py: QmSbakk8PG4VWSNcDG8mwoSzMgyjKbR9ZbmZjMPnCVmM2e - dialogues.py: QmUf8x7AWjzf4YF2pnjyPZryKwzgoEMeEfvL4ypdy889Y7 - message.py: Qmec2cSx4HC6Yzj3J6Zof7VUQTDRNEyDRHzSBMnCAhMqE3 - serialization.py: QmTJZno4tZPCFFa5ksDmvHuuNyigngCoR2kvSZdMTp3ukb + dialogues.py: QmXfP6bCy49A24RJYnzKZ6HBsf41hLyfLnV6VtaWgjBCWN + message.py: QmUZCG6QsTPTbwzbkKej2brJHvtv7Rn7HXMZ6NwpTD5cxH + serialization.py: QmTc271Ymt2tjqjgeR9ASTFujmNFi9WQKM2G7LWPrqSpK2 fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/default/serialization.py b/packages/fetchai/protocols/default/serialization.py index 5d96aa4c41..b342c18b58 100644 --- a/packages/fetchai/protocols/default/serialization.py +++ b/packages/fetchai/protocols/default/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/fipa/__init__.py b/packages/fetchai/protocols/fipa/__init__.py index 84ca80bc01..a36f6b41c5 100644 --- a/packages/fetchai/protocols/fipa/__init__.py +++ b/packages/fetchai/protocols/fipa/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/fipa/dialogues.py b/packages/fetchai/protocols/fipa/dialogues.py index aea14d5a4d..92d7df0e3a 100644 --- a/packages/fetchai/protocols/fipa/dialogues.py +++ b/packages/fetchai/protocols/fipa/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/fipa/message.py b/packages/fetchai/protocols/fipa/message.py index 1d8885b55f..2eb22f9ebc 100644 --- a/packages/fetchai/protocols/fipa/message.py +++ b/packages/fetchai/protocols/fipa/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/fipa/protocol.yaml b/packages/fetchai/protocols/fipa/protocol.yaml index fe511865b8..f4367213b8 100644 --- a/packages/fetchai/protocols/fipa/protocol.yaml +++ b/packages/fetchai/protocols/fipa/protocol.yaml @@ -7,13 +7,13 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmS3NhEEhPHKb2w1GFVRygmCY8sA2uyTDZApWi1Z6B2onT - __init__.py: QmR6pcWX14FsQip4eYJRNeiQdrNMPj6y4m6Tsgd6hd7yU6 + __init__.py: QmcqbJT7w5RhQVBx5W55iMwa1AgxM4LFu5RRPLmWAAxiKe custom_types.py: Qmf72KRbkNsxxAHwMtkmJc5TRL23fU7AuzJAdSTftckwJQ - dialogues.py: QmUGcpRpjx44DCXqi7RmD9YB7R7AoCHqeGjgTqyJRWvqBi + dialogues.py: QmYioX35NA2PAQNUDaNwu6K6PF1YpnBEwDY6DPqubcUy6B fipa.proto: QmZZV4YhcQXMSqWFsiyqb9b6c67NFmrsB4sbDre49GRHNp fipa_pb2.py: QmUdsdHFuXFqNbm373dugcEZ6WcNiAZRNkNLN81yuvfjh9 - message.py: QmbtqMXBdaRvsGg7RqpWvtGjguFU1nAyg2BBmiBFxAHf4d - serialization.py: QmaYjDAEyCe3jLktynmBqwaSquSz4YVj1rVpXuhZGK56Yi + message.py: QmRTcjGDyfsB54SjJeTh89Yz2g6FrmNMLvCfCTHTmA4knt + serialization.py: QmTRbLt1n8b7c9Yuy4G6WkL9y7iHw6g3GcjNBxU3aKYXtS fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/fipa/serialization.py b/packages/fetchai/protocols/fipa/serialization.py index 93c99bacfc..ab4e745214 100644 --- a/packages/fetchai/protocols/fipa/serialization.py +++ b/packages/fetchai/protocols/fipa/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/gym/__init__.py b/packages/fetchai/protocols/gym/__init__.py index 6e25f7d246..2a140125af 100644 --- a/packages/fetchai/protocols/gym/__init__.py +++ b/packages/fetchai/protocols/gym/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/gym/dialogues.py b/packages/fetchai/protocols/gym/dialogues.py index 7b2c3ee018..863aa21403 100644 --- a/packages/fetchai/protocols/gym/dialogues.py +++ b/packages/fetchai/protocols/gym/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/gym/message.py b/packages/fetchai/protocols/gym/message.py index f5b855f388..3077b7f5d9 100644 --- a/packages/fetchai/protocols/gym/message.py +++ b/packages/fetchai/protocols/gym/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/gym/protocol.yaml b/packages/fetchai/protocols/gym/protocol.yaml index 67260474ee..5f529f662f 100644 --- a/packages/fetchai/protocols/gym/protocol.yaml +++ b/packages/fetchai/protocols/gym/protocol.yaml @@ -7,13 +7,13 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmPjZperegz1BdXFdjguECvZQnYeGzPyYf2qG3WoQK98Lp - __init__.py: QmQvogZ6FVrp15UX2GZ2YKqZASS9gamA72MGt79oieE2tq + __init__.py: QmeyBLjo4xgz4mXvg9dwwwQCapYsbX4cgBXcpwvegg4zWZ custom_types.py: QmT3VKT86xZKR11RR1vQ3myRpmVZNdzY6ELd8HG9U2ngwa - dialogues.py: QmPEFYLc966VzBV4UQb53HEtizY6D9RWP54q2gzkzmbnk9 + dialogues.py: QmUx8de3kabB45rkbpddSpYHfmCnw74QjCjeuP7hC522Yb gym.proto: QmbrGMjAwLXxg4vZTTsdNkbsudhJbSbvkG2mag9RP6ejEg gym_pb2.py: QmPE79TZQjxqxCydj3t2gdPUeFwDXAR3mtWDqKEQfPvQe2 - message.py: QmPX8Y6odg32uAb8UX3yubVw8G7NkTUGMbc32idNdTrcDK - serialization.py: QmT2d4sLcJ96Yf2GEBoKqL3oq4pE518yQvK5WbeHaDXMSQ + message.py: QmPPzVnWong31qYJDak4tRmybbTDwbuSgFthrshJj2VW8b + serialization.py: QmQoL5VtkJCGgceAowdnRE2AvXcVSJJiGNP3dPaiBeQxXF fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/gym/serialization.py b/packages/fetchai/protocols/gym/serialization.py index d005731d68..5c0c5c647c 100644 --- a/packages/fetchai/protocols/gym/serialization.py +++ b/packages/fetchai/protocols/gym/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/http/__init__.py b/packages/fetchai/protocols/http/__init__.py index 9ec736425a..c6e0facc0c 100644 --- a/packages/fetchai/protocols/http/__init__.py +++ b/packages/fetchai/protocols/http/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/http/dialogues.py b/packages/fetchai/protocols/http/dialogues.py index f9e73ed227..e5ebfcf404 100644 --- a/packages/fetchai/protocols/http/dialogues.py +++ b/packages/fetchai/protocols/http/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/http/message.py b/packages/fetchai/protocols/http/message.py index 7672501c8d..af92b2abb4 100644 --- a/packages/fetchai/protocols/http/message.py +++ b/packages/fetchai/protocols/http/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/http/protocol.yaml b/packages/fetchai/protocols/http/protocol.yaml index 40087240f4..d0e05d8afd 100644 --- a/packages/fetchai/protocols/http/protocol.yaml +++ b/packages/fetchai/protocols/http/protocol.yaml @@ -7,12 +7,12 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmaxcPNm1aypK6tynS2xjBNn3ke8AhXTagtz3z5q9z272S - __init__.py: QmWzgWYrnS7PhjYrrx2mykLoaCbb7rDnVRcDqifsRukTy4 - dialogues.py: QmaZSG5heTuTYqeBcwRepCFXSgrzRrLCKysnEVQR5XCCzo + __init__.py: QmeXYyvvRiMhy3GRxPNU4EtHibVN26SvZT5r5RcxLGaE5D + dialogues.py: QmSBawWTSmg42xnVFZ2HHhQ7XjmhZab7UQ3iMDwwFUshLd http.proto: QmZdfqJYikfp8bcCzL2hLntDnfE6r5GKkcvwCBnhYLEkRD http_pb2.py: QmPbNBKxZjY3tGUXt4o4RTc2RFRErXi4ML3MEk56v9fMDK - message.py: QmPjVEnvwssNHq3QieTVbgBNtdXPYH21tghtks6L5vzVjH - serialization.py: QmbhfaMqjyBJW484pxkT1Sec4p8PRLkfETkqocS6Bj6W6g + message.py: QmYeHUEob5SCX8m5c3M2HvFaHgf8dWe6PGGUG6hYriG43d + serialization.py: QmUfoq7RSFuuRNGHBt8oDymPjEk17U4AjbaZgSkQ9bbUcz fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/http/serialization.py b/packages/fetchai/protocols/http/serialization.py index 976fd3e111..e77c7f20b2 100644 --- a/packages/fetchai/protocols/http/serialization.py +++ b/packages/fetchai/protocols/http/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ledger_api/__init__.py b/packages/fetchai/protocols/ledger_api/__init__.py index 99f0907397..93700bfe28 100644 --- a/packages/fetchai/protocols/ledger_api/__init__.py +++ b/packages/fetchai/protocols/ledger_api/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ledger_api/dialogues.py b/packages/fetchai/protocols/ledger_api/dialogues.py index dfe4d017a6..f1449c1d09 100644 --- a/packages/fetchai/protocols/ledger_api/dialogues.py +++ b/packages/fetchai/protocols/ledger_api/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ledger_api/message.py b/packages/fetchai/protocols/ledger_api/message.py index 519a639e23..61a1bd8adc 100644 --- a/packages/fetchai/protocols/ledger_api/message.py +++ b/packages/fetchai/protocols/ledger_api/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ledger_api/protocol.yaml b/packages/fetchai/protocols/ledger_api/protocol.yaml index fee74384c0..026eb7230a 100644 --- a/packages/fetchai/protocols/ledger_api/protocol.yaml +++ b/packages/fetchai/protocols/ledger_api/protocol.yaml @@ -7,13 +7,13 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmNqqzJepaeagpSLMztqFSLNurUdvo69SpS689RneNWJVn - __init__.py: QmX6ta6j6ust7qhVk1kZygzZK3gTTg7hSCBbSMKUxJgWgG + __init__.py: QmRWoVtn2moDouQ7gsdMKxNp3Pi5Z2EFLzhCkvowLZgt6i custom_types.py: QmSsjSb6NzKgqExpG26TfdEyJALFSosLYU5NcLxNVeDjWc - dialogues.py: QmcuATXqitgySbLujTF2s6B79qaKVCMQgSb5iGDrDfiezQ + dialogues.py: QmRKQLnkoJ5PsSoS9r3MCLnQ4E1ordxETkJcgcuXrjSx92 ledger_api.proto: QmTZbaYwaY57uQyYEz6B5qNrCkAJMDnokAzfWBX8BrRBEi ledger_api_pb2.py: QmbSRQTh59Zjy1kT1Rhra5WhxqwVTe5Xv5E7n4QC6v9qcp - message.py: QmVHm2yrWf2YDuJgG3zp4W5NY9H99ktuFnz4diskgwgj8s - serialization.py: Qmbkyy4aNmAGAM7PReTMv9HbpwDQJmHnMsf4L46gWavBTd + message.py: QmT1aL9Zj2obye4GDgrh8C3ZjoTUAoJmcFQJmEWvrHneRu + serialization.py: QmefYH5vyu8RJBx1rcAoYgszhc5t3xXuFWEvLQPs1JtKrJ fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/ledger_api/serialization.py b/packages/fetchai/protocols/ledger_api/serialization.py index 422471ed0c..f138c1ccf4 100644 --- a/packages/fetchai/protocols/ledger_api/serialization.py +++ b/packages/fetchai/protocols/ledger_api/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ml_trade/__init__.py b/packages/fetchai/protocols/ml_trade/__init__.py index bb602e2a47..a1d916b5ef 100644 --- a/packages/fetchai/protocols/ml_trade/__init__.py +++ b/packages/fetchai/protocols/ml_trade/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ml_trade/dialogues.py b/packages/fetchai/protocols/ml_trade/dialogues.py index 3f89804c13..0074c3b53c 100644 --- a/packages/fetchai/protocols/ml_trade/dialogues.py +++ b/packages/fetchai/protocols/ml_trade/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ml_trade/message.py b/packages/fetchai/protocols/ml_trade/message.py index 280768b87a..b773c7d5c1 100644 --- a/packages/fetchai/protocols/ml_trade/message.py +++ b/packages/fetchai/protocols/ml_trade/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ml_trade/protocol.yaml b/packages/fetchai/protocols/ml_trade/protocol.yaml index 95a98e39b2..6d16a69fee 100644 --- a/packages/fetchai/protocols/ml_trade/protocol.yaml +++ b/packages/fetchai/protocols/ml_trade/protocol.yaml @@ -7,13 +7,13 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmccSUTpVzDpaAqpGPGXWY5aG9Dp1yStX4Sn2tqCjsKVEC - __init__.py: QmcCS9uUQTTS2w85dTNiN5rQ14wyBhmBkr7pPPPcbLphcn + __init__.py: QmRnYbHxrQvb86cdheENeUhP7PswboKgugHaVMpcdPzaqR custom_types.py: QmPa6mxbN8WShsniQxJACfzAPRjGzYLbUFGoVU4N9DewUw - dialogues.py: Qmbras4sV8Fz7cnXwXMJLM5G4Y6rvSWvXakfTjDbyddbY5 - message.py: QmYXwvWBCQkfPKGB8cbNPeomxS248K4CvgW8djNuXCq7Qa + dialogues.py: QmUZ3BiJYY2bNiqrvwPZyNsjZ3v9Rvj8EW1UkH52XxjAqd + message.py: QmQzujKk8d4KztAK8Wug9FWhKdpeotiftCGX5aXmGGvPDA ml_trade.proto: QmNvSW9EUYiyu5SLhCVVaAdzLi1yjM94mnUeNSBzu2GRYH ml_trade_pb2.py: QmcrA4D7C9LFPcj6QKe8uUxYdHfHpRmZ5Dm4U5dKpT9nPh - serialization.py: QmNnEyqVdHuXXQLjAvbyibmdRbWBZG8tNEQy32s7SAVtcE + serialization.py: Qman9RVkzmHhnfU9VFbQ921nPWSuWdASvJSgfJqzsczEam fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/ml_trade/serialization.py b/packages/fetchai/protocols/ml_trade/serialization.py index 3b3c34ffd6..7bb84baf07 100644 --- a/packages/fetchai/protocols/ml_trade/serialization.py +++ b/packages/fetchai/protocols/ml_trade/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/oef_search/__init__.py b/packages/fetchai/protocols/oef_search/__init__.py index f130fc2c1b..681bacceaa 100644 --- a/packages/fetchai/protocols/oef_search/__init__.py +++ b/packages/fetchai/protocols/oef_search/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/oef_search/dialogues.py b/packages/fetchai/protocols/oef_search/dialogues.py index 7e56cc5c53..2579c15314 100644 --- a/packages/fetchai/protocols/oef_search/dialogues.py +++ b/packages/fetchai/protocols/oef_search/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/oef_search/message.py b/packages/fetchai/protocols/oef_search/message.py index 9b3b5f4132..66840af9c3 100644 --- a/packages/fetchai/protocols/oef_search/message.py +++ b/packages/fetchai/protocols/oef_search/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/oef_search/protocol.yaml b/packages/fetchai/protocols/oef_search/protocol.yaml index b5f5f8bac2..f86384d43b 100644 --- a/packages/fetchai/protocols/oef_search/protocol.yaml +++ b/packages/fetchai/protocols/oef_search/protocol.yaml @@ -7,13 +7,13 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: Qmdh6FwhH48U94pvhNfK3X5vwBgjKJzNofmd9FDyNEq4nS - __init__.py: Qmdr5ks5X4YtnpH6yKUcNu9uouyv3EGmrKFhyvNH7ZBjvT + __init__.py: QmZLjUfh8xW6dLaUgHZmTpX82giavcPQHwSKyx8X6ao3Xx custom_types.py: QmWbpGmegvEkiQCEEkCAjQoVFppKt23VXVXaBC1KbhCLbD - dialogues.py: QmVRDcsDPZEh5QcbaveAKHqSG7VZNvJPCjYbayfujsjbHf - message.py: QmUpvMGcR45d3e51PFCaGNwcKNwCjGeqgmoC6pZVCe9u1k + dialogues.py: QmSs2nefhrSMXE13QUT9U1q3JVSjzr3dPgsHBqYkCw2fUB + message.py: QmaCZb3pkF9QdpJnXVZb9pBFsUskXRajq8SdQ7XVwWjzfS oef_search.proto: QmdBhNnBMnUDiX9UD7eLy6HHKyUvmPrhys44fr1EjF4fnR oef_search_pb2.py: QmQbMm1pWABNZ9aPdcsJvX5icS1qrTi2CntePCVbtv5777 - serialization.py: QmcMQLbz6fkvZeqUXyE9WwH4TEJ3Dzy6pV4txLAVw9sdwb + serialization.py: QmNvuE9y9Xbi661GJmGMHt13kRaz9pWWRv9LXAYJMYTrwR fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/oef_search/serialization.py b/packages/fetchai/protocols/oef_search/serialization.py index 3d0348c1f7..ac14bcd8e3 100644 --- a/packages/fetchai/protocols/oef_search/serialization.py +++ b/packages/fetchai/protocols/oef_search/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/prometheus/__init__.py b/packages/fetchai/protocols/prometheus/__init__.py index 4eecec584f..e849e6eaf9 100644 --- a/packages/fetchai/protocols/prometheus/__init__.py +++ b/packages/fetchai/protocols/prometheus/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/prometheus/dialogues.py b/packages/fetchai/protocols/prometheus/dialogues.py index b58bb0ba52..6778e07f8d 100644 --- a/packages/fetchai/protocols/prometheus/dialogues.py +++ b/packages/fetchai/protocols/prometheus/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/prometheus/message.py b/packages/fetchai/protocols/prometheus/message.py index fc35dbfc9e..d7c889878a 100644 --- a/packages/fetchai/protocols/prometheus/message.py +++ b/packages/fetchai/protocols/prometheus/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/prometheus/protocol.yaml b/packages/fetchai/protocols/prometheus/protocol.yaml index f07fb1dd6f..bfb81792c0 100644 --- a/packages/fetchai/protocols/prometheus/protocol.yaml +++ b/packages/fetchai/protocols/prometheus/protocol.yaml @@ -7,12 +7,12 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmZb4NVBU8G417AcXM4Lj427MZR2D6guCvBeMiB7uhQp2m - __init__.py: QmYcCsWNz4qmSUBorY9N4XgyXvdYrrzNGr6oP3iQP9wUM5 - dialogues.py: QmSg5DmxpnvN9g2ThzmB4N6fJ2SeQ63iGa2ByGrLQD6BKF - message.py: QmXKpTPb7HLUdL2D5ycwGSLFchBSwj4Qc4mSc2Jrawhzjg + __init__.py: QmQvMGLitiy95SKYYSZQdfUH2UjpZ1ueTJooVuBG19P7QE + dialogues.py: QmYQYkByWC3sBnQPnDEzFh98ULycx1jzqqs6hLC196YZg5 + message.py: QmTQQeP2rv2QNG3AKCoMW2uP4rMNyUsG9BreSScptkUFJm prometheus.proto: QmUmvB2uvp8gdWzvUoxDJyc4Lvi4bFuaSdqMTMyk6hBhgF prometheus_pb2.py: QmXwmam9ZPyR3yf6UTmgMNVuvxTAZzgx6hPohzVPmKq452 - serialization.py: QmUgeNdHPqtCTyRM5K42a8rtzr1RYP94YiV7qycGDxdYzg + serialization.py: QmaWVbKwubB4pymQpQQg3yXfjeEdxrehxcHDkb8o4SeXun fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/prometheus/serialization.py b/packages/fetchai/protocols/prometheus/serialization.py index 17a5dd696f..52e86f2984 100644 --- a/packages/fetchai/protocols/prometheus/serialization.py +++ b/packages/fetchai/protocols/prometheus/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/register/__init__.py b/packages/fetchai/protocols/register/__init__.py index 435c5af99b..fc318e5f60 100644 --- a/packages/fetchai/protocols/register/__init__.py +++ b/packages/fetchai/protocols/register/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/register/dialogues.py b/packages/fetchai/protocols/register/dialogues.py index 717e465967..c0d52fa7c3 100644 --- a/packages/fetchai/protocols/register/dialogues.py +++ b/packages/fetchai/protocols/register/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/register/message.py b/packages/fetchai/protocols/register/message.py index c350a2ed1e..b3eeed3705 100644 --- a/packages/fetchai/protocols/register/message.py +++ b/packages/fetchai/protocols/register/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/register/protocol.yaml b/packages/fetchai/protocols/register/protocol.yaml index f85b36d763..0cd4c2f53f 100644 --- a/packages/fetchai/protocols/register/protocol.yaml +++ b/packages/fetchai/protocols/register/protocol.yaml @@ -7,12 +7,12 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmNRAfWZWLmEwBZ5Lq9fZr1ujWryiL4opuykzPUgiWVkkw - __init__.py: QmVqqvCtZJ6RNioinbGtGXqXQdxW38jgRjMyBpLhK6HTAz - dialogues.py: QmcUiHEdDbJLD14bb4RSYkXFJdtxJX74hd52QX6q4oGiSQ - message.py: QmanYNfvNFW9tjSbwAvQNS2V4JjBkmPRrbNf6SAfsr9RrZ + __init__.py: QmZgAVL7zheA4UnmaYBBaFetMyhFBVyTvcLF5HMvTicGvF + dialogues.py: QmfBmUxThNF1Qgz4bSrvPPPVYqxvdRtaMuXpQUQ5XVTPQC + message.py: QmP3x5QqxtHQFy4rRvLeiXAqYuaX4bwYGNgtbhk7DNqeDw register.proto: QmRuQ3XkDM668dsSSKrKcwt8SZqttT6bmVARgbDvs7b5tp register_pb2.py: QmVhxx411rnYPhbf7ipjFNLQgxUNkfRif3wk8XLz4she9p - serialization.py: QmcDqszzn1juRwmiLQAh1ZttB7ScS4zSizNo5qVUbMtXpy + serialization.py: QmP3BiPfJZ8JYdPLdGRWUA6dpJRdsAvt5mLuoXDMMBBzDX fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/register/serialization.py b/packages/fetchai/protocols/register/serialization.py index 3b73ba57f5..223285acf7 100644 --- a/packages/fetchai/protocols/register/serialization.py +++ b/packages/fetchai/protocols/register/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/signing/__init__.py b/packages/fetchai/protocols/signing/__init__.py index 194ca24a25..3f6550ff81 100644 --- a/packages/fetchai/protocols/signing/__init__.py +++ b/packages/fetchai/protocols/signing/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/signing/dialogues.py b/packages/fetchai/protocols/signing/dialogues.py index bc884340b0..0c8e470113 100644 --- a/packages/fetchai/protocols/signing/dialogues.py +++ b/packages/fetchai/protocols/signing/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/signing/message.py b/packages/fetchai/protocols/signing/message.py index 03e39a53f4..94704a599f 100644 --- a/packages/fetchai/protocols/signing/message.py +++ b/packages/fetchai/protocols/signing/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/signing/protocol.yaml b/packages/fetchai/protocols/signing/protocol.yaml index 037bdb8750..6f661710ed 100644 --- a/packages/fetchai/protocols/signing/protocol.yaml +++ b/packages/fetchai/protocols/signing/protocol.yaml @@ -7,11 +7,11 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmdSbVEY9rf812ZDoFjbPYBG15N4XaVVoHsEc1SXkwxPpp - __init__.py: QmUcmsCfop25uvwR6XQR3GGaJ63pbDsEPQPJeXM7nASuQW + __init__.py: QmPVnWg24cNfoLcZfqtRePdE2exyoKjRL2cfdz5qF7yZjg custom_types.py: Qmc7sAyCQbAaVs5dZf9hFkTrB2BG8VAioWzbyKBAybrQ1J - dialogues.py: QmRZWRnWTJYDtdQAxdUY2MRcFciE6uSrrkb2F4css5tv9K - message.py: QmeJA4NF3dtrCvj7yoAkQ2ZVwUGhudoAxR2xAcvuqyJdn2 - serialization.py: QmVMoDnWyDYchf8MMbBQNSh9FPXev2dXqoMCrSXFWjkLYQ + dialogues.py: Qmamd1T5ZDF8ExQznV9bXSoSC8s9rTgbkDoBRVTAzcARXB + message.py: QmRt1VStX7mEHp8724H6pyaVzoMucYKJ4HzdpsC1wfpthj + serialization.py: QmQY5rs9C3XYErfJ2fFtgqLqJiGx12hGHG5hmFp3ePMhFW signing.proto: QmZN9CmcfXCBiMQd9GTG81LadsbVQQ7j5pLFxEiQsQ4Sqk signing_pb2.py: QmSeawGUWMvFFgDrzuYrVMfWhb5UadXbPcyhzX2rLTdCTK fingerprint_ignore_patterns: [] diff --git a/packages/fetchai/protocols/signing/serialization.py b/packages/fetchai/protocols/signing/serialization.py index 8d5f3a416e..52909ab500 100644 --- a/packages/fetchai/protocols/signing/serialization.py +++ b/packages/fetchai/protocols/signing/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/state_update/__init__.py b/packages/fetchai/protocols/state_update/__init__.py index 785aa79709..c4434f981c 100644 --- a/packages/fetchai/protocols/state_update/__init__.py +++ b/packages/fetchai/protocols/state_update/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/state_update/dialogues.py b/packages/fetchai/protocols/state_update/dialogues.py index a14f685ec9..c71dfe0338 100644 --- a/packages/fetchai/protocols/state_update/dialogues.py +++ b/packages/fetchai/protocols/state_update/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/state_update/message.py b/packages/fetchai/protocols/state_update/message.py index 22a5693f6f..4945723d27 100644 --- a/packages/fetchai/protocols/state_update/message.py +++ b/packages/fetchai/protocols/state_update/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/state_update/protocol.yaml b/packages/fetchai/protocols/state_update/protocol.yaml index 1e767a1bed..3dd6814d7e 100644 --- a/packages/fetchai/protocols/state_update/protocol.yaml +++ b/packages/fetchai/protocols/state_update/protocol.yaml @@ -7,10 +7,10 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmUKL4QkkPJCkKNiw8oF7dbKVAmSs9YkZc5RHaq8ziCYaD - __init__.py: Qmd7GvLL2hzSbbk5JNcHyRDNH7yvhS3SaxitBbQcvA9i7U - dialogues.py: QmTaHNZ5L4T8HoRNgQJ3nxUfKXCXLPAeAiopDy21ybs8o9 - message.py: QmZkz31xtjHCywQaQHDNZXRkTMKtxqpYfdSePTeAdeSW41 - serialization.py: QmPQuxNbWMemgAJ9Lwzzi8K95aTES2DVSVzhJzsN1Njo1E + __init__.py: QmYQASXkhmnZbo5Vxh6mVbcaCkBP46UCZzQHQFtWYvBELL + dialogues.py: QmU4GFxG3DL5F515aF3z2GamzieU6XXWqzRy7V5oA7HnjV + message.py: QmWC9RCfMMNQUA6YLnSyHi6uScuVQGW5XSzibHKtvU4cDY + serialization.py: QmQrpKMDr9E4gtoknmjVenVnhGL8jbkgEmWzsSB8YYx4oh state_update.proto: QmVZjMwsoSC5UYWDVCv2Sp8ELS9TG6sCR5gHqBWAKayRY7 state_update_pb2.py: QmdnTuC7FLetYMPWBk1Nv3TccghovZYddW6BGXcHUQ9nSh fingerprint_ignore_patterns: [] diff --git a/packages/fetchai/protocols/state_update/serialization.py b/packages/fetchai/protocols/state_update/serialization.py index c46759acce..6773b71c48 100644 --- a/packages/fetchai/protocols/state_update/serialization.py +++ b/packages/fetchai/protocols/state_update/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/tac/__init__.py b/packages/fetchai/protocols/tac/__init__.py index 0dd910252c..d7682129cc 100644 --- a/packages/fetchai/protocols/tac/__init__.py +++ b/packages/fetchai/protocols/tac/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/tac/dialogues.py b/packages/fetchai/protocols/tac/dialogues.py index 9d26068c3d..a8dd60a151 100644 --- a/packages/fetchai/protocols/tac/dialogues.py +++ b/packages/fetchai/protocols/tac/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/tac/message.py b/packages/fetchai/protocols/tac/message.py index 2a462d2996..055498f12e 100644 --- a/packages/fetchai/protocols/tac/message.py +++ b/packages/fetchai/protocols/tac/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/tac/protocol.yaml b/packages/fetchai/protocols/tac/protocol.yaml index e300b3f221..4f64b7b1ee 100644 --- a/packages/fetchai/protocols/tac/protocol.yaml +++ b/packages/fetchai/protocols/tac/protocol.yaml @@ -8,11 +8,11 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmbuKk4QvBmziXByv3nq46o3pmapyf7gMPRQDLRBB6Sric - __init__.py: QmSAC7PGra9fig8RhhF1j3XEVpgie9UZNNYPc2AB9Kx9xJ + __init__.py: QmRuxxNxJgmwtf4eMqVWpFKB2hrpT48GFLUmqRpi1Xicaj custom_types.py: QmXQATfnvuCpt4FicF4QcqCcLj9PQNsSHjCBvVQknWpyaN - dialogues.py: QmSYjYBpE3YZxPuXE42DK7fo47kgF8dqWT9ySYb9qcB51T - message.py: QmcGUgrk2aJffnyyyotv3WpyPph25MarjXjmNhyny32L6V - serialization.py: Qmani4DjtVVNYF1DDUSWAnQu7pC2v1DPvEW6ZQwmp2nMSA + dialogues.py: QmRjWdedEm43231zPpSad7R97BZ567qNSjjjtPnabrziRd + message.py: QmejFGHZAmBW7NTJrSsYA3ZJynGLiBn3DCgUkEjcE7MWpe + serialization.py: QmVHNaPqLYsNe5PTA8qFSWaHonmfkcnxcMbBdjLexSmCH7 tac.proto: QmUXk2kwqp1vo22oZdvLbWKirojeqkXdGSmiz6r14bMqSE tac_pb2.py: QmVHkh5GctFUU36wiVZZfZYYoQxT4uZP8eUzAtgKgUBxn6 fingerprint_ignore_patterns: [] diff --git a/packages/fetchai/protocols/tac/serialization.py b/packages/fetchai/protocols/tac/serialization.py index 559b5a6626..29fcb7a366 100644 --- a/packages/fetchai/protocols/tac/serialization.py +++ b/packages/fetchai/protocols/tac/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol/__init__.py b/tests/data/generator/t_protocol/__init__.py index 56de0d0a42..6e3181c92b 100644 --- a/tests/data/generator/t_protocol/__init__.py +++ b/tests/data/generator/t_protocol/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol/dialogues.py b/tests/data/generator/t_protocol/dialogues.py index 2aeb20997a..596bf59c2b 100644 --- a/tests/data/generator/t_protocol/dialogues.py +++ b/tests/data/generator/t_protocol/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol/message.py b/tests/data/generator/t_protocol/message.py index 4d9eeeab32..96f72f05a1 100644 --- a/tests/data/generator/t_protocol/message.py +++ b/tests/data/generator/t_protocol/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol/protocol.yaml b/tests/data/generator/t_protocol/protocol.yaml index 692604f6e6..d727cad719 100644 --- a/tests/data/generator/t_protocol/protocol.yaml +++ b/tests/data/generator/t_protocol/protocol.yaml @@ -6,11 +6,11 @@ description: A protocol for testing purposes. license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: - __init__.py: QmQy21g5sVYfmy4vSYuEFyPnobM4SA1dEouz5deXNssPWx + __init__.py: QmVEtFTkpwEHfaijkMso9P6JMJCSepSvUKeqcRJLVJMyji custom_types.py: QmWg8HFav8w9tfZfMrTG5Uo7QpexvYKKkhpGPD18233pLw - dialogues.py: Qmaq3wGigpyL2xZwCgqv1CN6tNy1uayzym9ybwjvZRYm5C - message.py: QmYGdkkon76MfdhgBvHrpuog3yCpgZhgDyKXBh4ZRUN7GS - serialization.py: QmYJLfQ49vsqRSpC7GV1ukc3PnkVBuRMM3GBtGoCgDEv5z + dialogues.py: QmWAdikDRJWTG7HUXsCsZRg4Wxnf8cMr5KujpyC4M75gnB + message.py: Qmf4TsCLKT2PudSxm8eUCQVKPfvwUmpSt3bDssEaqPZp1J + serialization.py: QmPBxFuGJ9EZdwT1mP1wZqffYCsQYo7t2kxvSSWnG6msFp t_protocol.proto: QmWdNaAJ9Mkf2SHF1RSZrsk2a5jZyXZtCD7XU5PHLCph5z t_protocol_pb2.py: QmamuU3UsGEFKWPSbt5Qeo31kXSCLpTD81TkuX44njkYgh fingerprint_ignore_patterns: [] diff --git a/tests/data/generator/t_protocol/serialization.py b/tests/data/generator/t_protocol/serialization.py index 2c6b422f86..f0f61e1f1c 100644 --- a/tests/data/generator/t_protocol/serialization.py +++ b/tests/data/generator/t_protocol/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol_no_ct/__init__.py b/tests/data/generator/t_protocol_no_ct/__init__.py index 2518f2ff75..c79ce27d80 100644 --- a/tests/data/generator/t_protocol_no_ct/__init__.py +++ b/tests/data/generator/t_protocol_no_ct/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol_no_ct/dialogues.py b/tests/data/generator/t_protocol_no_ct/dialogues.py index 67d0f9b1cd..bd800706e8 100644 --- a/tests/data/generator/t_protocol_no_ct/dialogues.py +++ b/tests/data/generator/t_protocol_no_ct/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol_no_ct/message.py b/tests/data/generator/t_protocol_no_ct/message.py index 8f07d60d6c..c36dd9ccd9 100644 --- a/tests/data/generator/t_protocol_no_ct/message.py +++ b/tests/data/generator/t_protocol_no_ct/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol_no_ct/protocol.yaml b/tests/data/generator/t_protocol_no_ct/protocol.yaml index 515aa32085..03541a26cf 100644 --- a/tests/data/generator/t_protocol_no_ct/protocol.yaml +++ b/tests/data/generator/t_protocol_no_ct/protocol.yaml @@ -6,10 +6,10 @@ description: A protocol for testing purposes. license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: - __init__.py: QmaaZ7Je2PRTkcnqy8oLR58yBDVpcRQ4BcaRe3sd3fug3Z - dialogues.py: QmXCaVN6cAjHk7cMjRbD2GHxEosKvCp7nFGKMqXf8FThQQ - message.py: QmXB1CxBaQtNqtYRcJFRruFuvytxA4KQtBXpo3jTsdwPrA - serialization.py: QmPX4KzaEfK9JwbbHAmmB1rZp1guJTW5PgPan7ZGAp36DH + __init__.py: QmWsBeSVmG4FZWq4mGEzvERUDmDh4jC59UhYQWFvdYvcPo + dialogues.py: Qmeq7m8vf1LW5WeehNG8qnoGoRstQrABw2vdQh5tmB3KxX + message.py: QmWFjERqT1xxKNEMwwnJx5JXNVtYzxJL1nLLu65nkfWdk5 + serialization.py: QmcxwDYsX1r8Yyy13Ci8vpZFtrpD7RaNXd2MJHCPuXCvBG t_protocol_no_ct.proto: Qmc8KkKnWZ9utBxrbEyWhVDRdut87DkFvmHP3SYUg4J3EU t_protocol_no_ct_pb2.py: QmdCYBHYVZ49fdZ5CHkL72Zt5xQ9H1b3KFyTx9MvnPDnFw fingerprint_ignore_patterns: [] diff --git a/tests/data/generator/t_protocol_no_ct/serialization.py b/tests/data/generator/t_protocol_no_ct/serialization.py index 3e2752017a..b94350afa9 100644 --- a/tests/data/generator/t_protocol_no_ct/serialization.py +++ b/tests/data/generator/t_protocol_no_ct/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From e6e2dec22bcd7a2bba2cb3261059760871eb60c9 Mon Sep 17 00:00:00 2001 From: Yuri Turchenkov Date: Fri, 1 Jan 2021 14:53:46 +0300 Subject: [PATCH 119/204] Update aea/cli/local_registry_sync.py Co-authored-by: David Minarsch --- aea/cli/local_registry_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aea/cli/local_registry_sync.py b/aea/cli/local_registry_sync.py index 0dc92dfc5e..db48d845c8 100644 --- a/aea/cli/local_registry_sync.py +++ b/aea/cli/local_registry_sync.py @@ -60,7 +60,7 @@ def do_local_registry_update(base_dir: Union[str, Path]) -> None: ): click.echo(f"{package_id} is the latest version.") continue - click.echo(f"Updating to {latest_public_id}.") + click.echo(f"Updating {current_public_id} to {latest_public_id}.") replace_package(str(package_id.package_type), latest_public_id, package_dir) From e1044a64dc04231f6ccd5591b11a2fee31b3b120 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Fri, 1 Jan 2021 12:15:46 +0000 Subject: [PATCH 120/204] fix hashes --- packages/hashes.csv | 26 +++++++++++++------------- tests/data/hashes.csv | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/hashes.csv b/packages/hashes.csv index ed9c086945..5570dd18f3 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -49,20 +49,20 @@ fetchai/contracts/oracle,QmbmDutosj3Zn5qLxvuCK81D1obww1xAMb1iTpchuL9Y3s fetchai/contracts/oracle_client,Qmcv9ctDViLGVxaPBNtaB6M2o6bRQDekJM1ue9XZxA7naB fetchai/contracts/scaffold,QmR8eqeu9i2auL7XjkvbyV4DMf3cG5hEy3eDfVEeeMMrD5 fetchai/contracts/staking_erc20,QmQfqoSKY1CeyWvrvZdEB2ftLXKboXvku7be2t3cNH3t2H -fetchai/protocols/contract_api,QmTRqbP6JrXC76VAx9WsUwgDHYKGbfiib1WGDzXmC8dbF1 -fetchai/protocols/default,QmXmrDnJzARnRyhCMhPXy12Bu8Zq85ZrjbxUUbcNJvYVjm -fetchai/protocols/fipa,QmdZevSTKavH7wCVXeZyiQ5y3sDAkhEQ3SiwhaiXWYgnDJ -fetchai/protocols/gym,QmW8fqeoXvsgdjZzZt6Wfj6vfCJBEweAUFdUk1dqNyBa1m -fetchai/protocols/http,QmYF5MEFZbJUo2xTLJwCt6cTkADbytUqfeWvsGzvuy6hJF -fetchai/protocols/ledger_api,QmWjvwNBpomTDCPEfA86dhHihr2HkCpnveCVZDWU7x5m42 -fetchai/protocols/ml_trade,QmezUacB3b2N31wcs6zxCgurFWubEFJXrfPdaT368LVhTX -fetchai/protocols/oef_search,QmQXGHb6GRgeeeYGy5KVPz6bkaWxo3S6tm3fHqoNJ3Zoij -fetchai/protocols/prometheus,QmR9KCLCWMMwJRkGV313WGxYwk49bLtg6SxoAavYb3C97h -fetchai/protocols/register,QmNwCo2HYKjZzHbjPfmAZnYUBFVURbbFR2JZAye9Wg2WmY +fetchai/protocols/contract_api,QmNQ18rgSAuUWH1KMCMmudi3EXKZmZGEp9GBqZXxavKQMu +fetchai/protocols/default,QmQebB6xck4FfPRpUCsDsQV4sYJoiL3jNeCXwEHkY4XyNK +fetchai/protocols/fipa,QmeduYjwZKP1PQNqQi7T8UP2NLLdPduL854xtjEpR6DDoo +fetchai/protocols/gym,QmSPb74yYvvwACaHQQZ6FUiu55kEeq7XwnDLcrcmasp6mD +fetchai/protocols/http,QmRMF4HUX58VLQusBTDtRpB4Vn2SKoFse3XkZ2CM22Kd58 +fetchai/protocols/ledger_api,QmYhCs8F79TjGJtVC67WZ2GA1a63yChsosXfdkXP3rtFKt +fetchai/protocols/ml_trade,QmRxSdyvHggeCdNBELhUohmvFnTZ6uUBaJ1SLzoLJ4yJ9t +fetchai/protocols/oef_search,QmU5hhGcrtna6EjBJyAatMbM9sAnLuNG39GfnMELXngGQY +fetchai/protocols/prometheus,QmTC8MW6s1xXKnz9TwcxA2Ffw7aPqgZoG4McvxzkeQuRd9 +fetchai/protocols/register,QmWm6SeMSTqRr1wtY1RY2RfoiHBjkJpTnnRTimvtcHS3H3 fetchai/protocols/scaffold,QmUEd8eCvcdPXvV37tZr3pQE7oTFtBZbLkf6GPTMwF8Aiu -fetchai/protocols/signing,QmcTcVkMJ7uCnPF4CnchTdp7k1iz4ADt5xXV3emvzF9iQb -fetchai/protocols/state_update,Qma95xUrwSX7XGPRyfWYPziJLx8kEpYL6j5gCxFYZyLARQ -fetchai/protocols/tac,QmTWucqbXue3H332ivNHMG5eZkSSPxz1aYNtPs3q8YEff5 +fetchai/protocols/signing,QmTVRQBUtxJKhFBQL92qDRF81tCrZxJccLkK69diFiTeD5 +fetchai/protocols/state_update,QmRn5XnwEUU77tEzTTNNS933Fdj5orxZVJgn3pGef822uH +fetchai/protocols/tac,QmXeSbSFunSy7VeNCSq5C7ciwDY5B3ufKd1afKB31zz8AL fetchai/skills/aries_alice,QmdQEHTqFhEkFB9ZALnD19DwskyfLD39du3ukAPYpXYMhv fetchai/skills/aries_faber,QmeveDnkEFYnUmfJN8cdXJGokrtQD47qxyYvtjEt5Qx9oL fetchai/skills/carpark_client,QmWabiH48yVv4hNaufU3jtCqe3GSEsmhMe5DEPeciTYTP3 diff --git a/tests/data/hashes.csv b/tests/data/hashes.csv index 0cdb6108e6..4897fe3484 100644 --- a/tests/data/hashes.csv +++ b/tests/data/hashes.csv @@ -2,7 +2,7 @@ dummy_author/agents/dummy_aea,QmPWxJVM9xnkRFtB41Wr1Xi2NGKpeRgDXUzbAzA8rferGh dummy_author/skills/dummy_skill,QmSmzcjGNhiz2KpSE6wSi19AeesCVjzupJm1NpXYJVnJuZ fetchai/connections/dummy_connection,QmYn34pVcTpKazRLUpgafNW917ndLZidq9nDFFiRW7YzbT fetchai/contracts/dummy_contract,QmXhMKRF5upco1efaC9oNcesStEMkPWKhBUv6CK9Vngk2S -fetchai/protocols/t_protocol,QmNhzDCznorQpySrvKt4F4sQFXjzF1ZecEd5inixiNavDH -fetchai/protocols/t_protocol_no_ct,QmWwMwHxvuPZJaMh44Bd1d5aBzYiogep8TJswTKkRwLfBu +fetchai/protocols/t_protocol,QmWjV39cYymZmHhNj55zfoGMY9Sa768UykXLhuYiJbD56v +fetchai/protocols/t_protocol_no_ct,QmcKpRB2A89wiMvA9f6wfRZH5K3EmTHP3qrkQazmub1zsu fetchai/skills/dependencies_skill,QmSZtiXSrzj8mnYztRHKM6A8fYwUqhrTLL317r2JqXu8Ga fetchai/skills/exception_skill,QmboGWn3yLp3kLdoCYHYEQ92W4u6LGrzdvaiZ4WaigcCqB From 782899f8e17c07c11ce8ada8d6ff97c43a266925 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Sat, 2 Jan 2021 14:04:51 +0000 Subject: [PATCH 121/204] address pr comments --- aea/cli/local_registry_sync.py | 27 ++++++++------ aea/configurations/base.py | 2 +- aea/configurations/data_types.py | 4 +-- aea/configurations/loader.py | 37 +++++++++++++++----- tests/test_cli/test_fingerprint.py | 4 +-- tests/test_cli/test_local_registry_update.py | 8 +++-- tests/test_configurations/test_base.py | 1 + tests/test_exceptions.py | 26 +++++++++++++- 8 files changed, 81 insertions(+), 28 deletions(-) diff --git a/aea/cli/local_registry_sync.py b/aea/cli/local_registry_sync.py index db48d845c8..57f320e798 100644 --- a/aea/cli/local_registry_sync.py +++ b/aea/cli/local_registry_sync.py @@ -30,8 +30,9 @@ from aea.cli.registry.utils import get_package_meta from aea.cli.utils.context import Context from aea.cli.utils.decorators import pass_ctx +from aea.cli.utils.loggers import logger from aea.configurations.data_types import PackageId, PackageType, PublicId -from aea.configurations.loader import load_component_configuration +from aea.configurations.loader import load_package_configuration PACKAGES_DIRS = [i.to_plural() for i in PackageType] @@ -41,10 +42,13 @@ @pass_ctx def local_registry_sync(ctx: Context): """Upgrade the local package registry.""" - do_local_registry_update(ctx.cwd) + skip_consistency_check = ctx.config["skip_consistency_check"] + do_local_registry_update(ctx.cwd, skip_consistency_check) -def do_local_registry_update(base_dir: Union[str, Path]) -> None: +def do_local_registry_update( + base_dir: Union[str, Path], skip_consistency_check: bool = True +) -> None: """ Perform local registry update. @@ -52,7 +56,7 @@ def do_local_registry_update(base_dir: Union[str, Path]) -> None: :return: None """ - for package_id, package_dir in enlist_packages(base_dir): + for package_id, package_dir in enlist_packages(base_dir, skip_consistency_check): current_public_id = package_id.public_id latest_public_id = get_package_latest_public_id(package_id) if not ( # pylint: disable=superfluous-parens @@ -101,7 +105,7 @@ def get_package_latest_public_id(package_id: PackageId) -> PublicId: def enlist_packages( - base_dir: Union[Path, str] + base_dir: Union[Path, str], skip_consistency_check: bool = True ) -> Generator[Tuple[PackageId, Union[Path, str]], None, None]: """ Generate list of the packages in local repo directory. @@ -131,10 +135,13 @@ def enlist_packages( if package_type.to_plural() != package_type_plural: # incorrect package placing continue # pragma: nocover - config = load_component_configuration( - package_type, Path(package_dir), skip_consistency_check=True + config = load_package_configuration( + package_type, + Path(package_dir), + skip_consistency_check=skip_consistency_check, ) yield (config.package_id, package_dir) - except ValueError: # pragma: nocover - # can not determine package type, not a package? just skip - pass # pragma: nocover + except ValueError as e: # pragma: nocover + logger.error( # pragma: nocover + f"Error with package_dir={package_dir}: {e}" + ) diff --git a/aea/configurations/base.py b/aea/configurations/base.py index 04bebbcac1..a1e8cd9afe 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -492,7 +492,7 @@ def check_fingerprint(self, directory: Path) -> None: if not directory.exists() or not directory.is_dir(): raise ValueError("Directory {} is not valid.".format(directory)) _compare_fingerprints( - self, directory, False, self.component_type.to_configuration_type() + self, directory, False, self.component_type.to_package_type() ) def check_aea_version(self): diff --git a/aea/configurations/data_types.py b/aea/configurations/data_types.py index 7ac75957f3..f6638a908d 100644 --- a/aea/configurations/data_types.py +++ b/aea/configurations/data_types.py @@ -163,7 +163,7 @@ class ComponentType(Enum): SKILL = SKILL CONTRACT = CONTRACT - def to_configuration_type(self) -> PackageType: + def to_package_type(self) -> PackageType: """Get package type for component type.""" return PackageType(self.value) @@ -565,7 +565,7 @@ def __init__(self, component_type: Union[ComponentType, str], public_id: PublicI :param public_id: the public id. """ component_type = ComponentType(component_type) - super().__init__(component_type.to_configuration_type(), public_id) + super().__init__(component_type.to_package_type(), public_id) @property def component_type(self) -> ComponentType: diff --git a/aea/configurations/loader.py b/aea/configurations/loader.py index 521160facf..e7468368b5 100644 --- a/aea/configurations/loader.py +++ b/aea/configurations/loader.py @@ -308,22 +308,41 @@ def from_package_type( def load_component_configuration( - component_type: Union[ComponentType, PackageType], + component_type: ComponentType, directory: Path, skip_consistency_check: bool = False, -) -> "ComponentConfiguration": +) -> ComponentConfiguration: """ Load configuration and check that it is consistent against the directory. - :param component_type: the component or package type. + :param component_type: the component type. :param directory: the root of the package :param skip_consistency_check: if True, the consistency check are skipped. :return: the configuration object. """ - if isinstance(component_type, ComponentType): - component_type = component_type.to_configuration_type() - configuration_object = _load_configuration_object(component_type, directory) - if not skip_consistency_check: + package_type = component_type.to_package_type() + configuration_object = load_package_configuration( + package_type, directory, skip_consistency_check + ) + configuration_object = cast(ComponentConfiguration, configuration_object) + return configuration_object + + +def load_package_configuration( + package_type: PackageType, directory: Path, skip_consistency_check: bool = False, +) -> PackageConfiguration: + """ + Load configuration and check that it is consistent against the directory. + + :param package_type: the package type. + :param directory: the root of the package + :param skip_consistency_check: if True, the consistency check are skipped. + :return: the configuration object. + """ + configuration_object = _load_configuration_object(package_type, directory) + if not skip_consistency_check and isinstance( + configuration_object, ComponentConfiguration + ): configuration_object._check_configuration_consistency( # pylint: disable=protected-access directory ) @@ -332,11 +351,11 @@ def load_component_configuration( def _load_configuration_object( package_type: PackageType, directory: Path -) -> ComponentConfiguration: +) -> PackageConfiguration: """ Load the configuration object, without consistency checks. - :param component_type: the component type. + :param package_type: the package type. :param directory: the directory of the configuration. :return: the configuration object. :raises FileNotFoundError: if the configuration file is not found. diff --git a/tests/test_cli/test_fingerprint.py b/tests/test_cli/test_fingerprint.py index 325db9bd29..db60c18c4d 100644 --- a/tests/test_cli/test_fingerprint.py +++ b/tests/test_cli/test_fingerprint.py @@ -93,14 +93,14 @@ def test_by_path_exceptions(self, *mocks): """Test fingerprint by_path works raises exceptions.""" with pytest.raises( ClickException, - match=f"No package config file found in `.*`. Incorrect directory?", + match="No package config file found in `.*`. Incorrect directory?", ): with mock.patch("os.listdir", return_value=[]): self._run_fingerprint_by_path() with pytest.raises( ClickException, - match=f"Too many config files in the directory, only one has to present!", + match="Too many config files in the directory, only one has to present!", ): with mock.patch( "os.listdir", diff --git a/tests/test_cli/test_local_registry_update.py b/tests/test_cli/test_local_registry_update.py index 3b5a41b6ed..eb9896275f 100644 --- a/tests/test_cli/test_local_registry_update.py +++ b/tests/test_cli/test_local_registry_update.py @@ -60,13 +60,15 @@ def test_local_registry_update(): "aea.cli.local_registry_sync.replace_package" ) as replace_package_mock: result = runner.invoke( - cli, "local-registry-sync", catch_exceptions=False + cli, ["-s", "local-registry-sync"], catch_exceptions=False ) assert result.exit_code == 0, result.stdout assert replace_package_mock.call_count == len(PACKAGES) # do actual upgrade - result = runner.invoke(cli, "local-registry-sync", catch_exceptions=False) + result = runner.invoke( + cli, ["-s", "local-registry-sync"], catch_exceptions=False + ) assert result.exit_code == 0, result.stdout # check next update will do nothing @@ -74,7 +76,7 @@ def test_local_registry_update(): "aea.cli.local_registry_sync.replace_package" ) as replace_package_mock: result = runner.invoke( - cli, "local-registry-sync", catch_exceptions=False + cli, ["-s", "local-registry-sync"], catch_exceptions=False ) assert result.exit_code == 0, result.stdout assert replace_package_mock.call_count == 0 diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index eade52496b..10919cbfe6 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -842,6 +842,7 @@ def test_agent_config_to_json_with_optional_configurations(): agent_config.default_connection = "author/name:0.1.0" agent_config.default_ledger = DEFAULT_LEDGER agent_config.json + assert agent_config.package_id == PackageId.from_uri_path("agent/author/name/0.1.0") def test_protocol_specification_attributes(): diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 92fa0dbe0c..bc7e7b6ec6 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -42,7 +42,7 @@ def test_stop_runtime(): assert e.reraise == test -def test_parse_exception(): +def test_parse_exception_i(): """Test parse exception.""" def exception_raise(): @@ -61,3 +61,27 @@ def exception_raise(): 'raise ValueError("expected")\n\nValueError: expected\n', ] assert all([string in out for string in expected]) + + +def test_parse_exception_ii(): + """Test parse exception.""" + + def exception_raise(): + """A function that raises an exception.""" + raise AEAEnforceError("expected") + + try: + exception_raise() + except Exception as e: + out = parse_exception(e) + + expected = [ + "Traceback (most recent call last):\n\n", + 'test_exceptions.py", line ', + "in test_parse_exception_ii\n", + "exception_raise()\n\n", + ", line", + "in exception_raise\n", + 'raise AEAEnforceError("expected")\n\naea.exceptions.AEAEnforceError: expected\n', + ] + assert all([string in out for string in expected]) From d83598f014440ba7dadddd023eec14322936e81d Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Fri, 1 Jan 2021 12:27:12 +0300 Subject: [PATCH 122/204] local registry sync --- aea/cli/core.py | 3 +- aea/cli/local_registry_sync.py | 140 +++++++++++++++++++ aea/configurations/base.py | 5 + aea/configurations/constants.py | 1 + aea/configurations/loader.py | 14 +- tests/test_cli/test_local_registry_update.py | 88 ++++++++++++ tests/test_cli/test_misc.py | 73 +++++----- 7 files changed, 280 insertions(+), 44 deletions(-) create mode 100644 aea/cli/local_registry_sync.py create mode 100644 tests/test_cli/test_local_registry_update.py diff --git a/aea/cli/core.py b/aea/cli/core.py index 16655e4335..2e1413a470 100644 --- a/aea/cli/core.py +++ b/aea/cli/core.py @@ -17,7 +17,6 @@ # limitations under the License. # # ------------------------------------------------------------------------------ - """Core definitions for the AEA command-line tool.""" import click @@ -45,6 +44,7 @@ from aea.cli.issue_certificates import issue_certificates from aea.cli.launch import launch from aea.cli.list import list_command as _list +from aea.cli.local_registry_sync import local_registry_sync from aea.cli.login import login from aea.cli.logout import logout from aea.cli.publish import publish @@ -151,5 +151,6 @@ def _init_gui() -> None: cli.add_command(run) cli.add_command(scaffold) cli.add_command(search) +cli.add_command(local_registry_sync) cli.add_command(transfer) cli.add_command(upgrade) diff --git a/aea/cli/local_registry_sync.py b/aea/cli/local_registry_sync.py new file mode 100644 index 0000000000..0dc92dfc5e --- /dev/null +++ b/aea/cli/local_registry_sync.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""Implementation of the 'aea sync-local-registry' subcommand.""" +import os +import shutil +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Generator, Tuple, Union + +import click + +from aea.cli.fingerprint import determine_package_type_for_directory +from aea.cli.registry.add import fetch_package +from aea.cli.registry.utils import get_package_meta +from aea.cli.utils.context import Context +from aea.cli.utils.decorators import pass_ctx +from aea.configurations.data_types import PackageId, PackageType, PublicId +from aea.configurations.loader import load_component_configuration + + +PACKAGES_DIRS = [i.to_plural() for i in PackageType] + + +@click.command() +@pass_ctx +def local_registry_sync(ctx: Context): + """Upgrade the local package registry.""" + do_local_registry_update(ctx.cwd) + + +def do_local_registry_update(base_dir: Union[str, Path]) -> None: + """ + Perform local registry update. + + :param base_dir: root directory of the local registry. + + :return: None + """ + for package_id, package_dir in enlist_packages(base_dir): + current_public_id = package_id.public_id + latest_public_id = get_package_latest_public_id(package_id) + if not ( # pylint: disable=superfluous-parens + current_public_id < latest_public_id + ): + click.echo(f"{package_id} is the latest version.") + continue + click.echo(f"Updating to {latest_public_id}.") + replace_package(str(package_id.package_type), latest_public_id, package_dir) + + +def replace_package( + package_type: str, public_id: PublicId, package_dir: Union[Path, str] +) -> None: + """ + Download, extract and replace exists package. + + :param package_type: str. + :param public_id: pacakge bulic id to download + :param: package_dir: target package dir + + :return: None + """ + with TemporaryDirectory() as tmp_dir: + new_package_dir = os.path.join(tmp_dir, public_id.name) + os.mkdir(new_package_dir) + fetch_package( + package_type, public_id=public_id, cwd=tmp_dir, dest=new_package_dir + ) + shutil.rmtree(package_dir) + shutil.move(new_package_dir, package_dir) + + +def get_package_latest_public_id(package_id: PackageId) -> PublicId: + """ + Get package latest package id from the remote repo. + + :param package_id: id of the package to check + + :return: package id of the latest package in remote repo + """ + package_meta = get_package_meta( + str(package_id.package_type), package_id.public_id.to_latest() + ) + return PublicId.from_str(package_meta["public_id"]) + + +def enlist_packages( + base_dir: Union[Path, str] +) -> Generator[Tuple[PackageId, Union[Path, str]], None, None]: + """ + Generate list of the packages in local repo directory. + + :param base_dir: path or str of the local repo. + + :return: generator with Tuple of package_id and package diurectory. + """ + for author in os.listdir(base_dir): + author_dir = os.path.join(base_dir, author) + if not os.path.isdir(author_dir): + continue # pragma: nocover + for package_type_plural in os.listdir(author_dir): + if package_type_plural not in PACKAGES_DIRS: + continue # pragma: nocover + package_type_dir = os.path.join(author_dir, package_type_plural) + if not os.path.isdir(package_type_dir): + continue # pragma: nocover + for package_name in os.listdir(package_type_dir): + package_dir = os.path.join(package_type_dir, package_name) + if not os.path.isdir(package_dir): + continue # pragma: nocover + try: + package_type = determine_package_type_for_directory( + Path(package_dir) + ) + if package_type.to_plural() != package_type_plural: + # incorrect package placing + continue # pragma: nocover + config = load_component_configuration( + package_type, Path(package_dir), skip_consistency_check=True + ) + yield (config.package_id, package_dir) + except ValueError: # pragma: nocover + # can not determine package type, not a package? just skip + pass # pragma: nocover diff --git a/aea/configurations/base.py b/aea/configurations/base.py index 6971958717..051a8586be 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -287,6 +287,11 @@ def directory(self, directory: Path) -> None: raise ValueError("Directory already set") self._directory = directory + @property + def package_id(self) -> PackageId: + """Get package id.""" + return PackageId(package_type=self.package_type, public_id=self.public_id) + @staticmethod def _parse_aea_version_specifier(aea_version_specifiers: str) -> SpecifierSet: try: diff --git a/aea/configurations/constants.py b/aea/configurations/constants.py index b2577b0e1f..ab284435ac 100644 --- a/aea/configurations/constants.py +++ b/aea/configurations/constants.py @@ -85,4 +85,5 @@ DEFAULT_PROTOCOL_CONFIG_FILE: PROTOCOL, DEFAULT_CONNECTION_CONFIG_FILE: CONNECTION, DEFAULT_CONTRACT_CONFIG_FILE: CONTRACT, + DEFAULT_AEA_CONFIG_FILE: AGENT, } # type: Dict[str, str] diff --git a/aea/configurations/loader.py b/aea/configurations/loader.py index 2ce50cc38d..521160facf 100644 --- a/aea/configurations/loader.py +++ b/aea/configurations/loader.py @@ -308,18 +308,20 @@ def from_package_type( def load_component_configuration( - component_type: ComponentType, + component_type: Union[ComponentType, PackageType], directory: Path, skip_consistency_check: bool = False, ) -> "ComponentConfiguration": """ Load configuration and check that it is consistent against the directory. - :param component_type: the component type. + :param component_type: the component or package type. :param directory: the root of the package :param skip_consistency_check: if True, the consistency check are skipped. :return: the configuration object. """ + if isinstance(component_type, ComponentType): + component_type = component_type.to_configuration_type() configuration_object = _load_configuration_object(component_type, directory) if not skip_consistency_check: configuration_object._check_configuration_consistency( # pylint: disable=protected-access @@ -329,7 +331,7 @@ def load_component_configuration( def _load_configuration_object( - component_type: ComponentType, directory: Path + package_type: PackageType, directory: Path ) -> ComponentConfiguration: """ Load the configuration object, without consistency checks. @@ -339,9 +341,7 @@ def _load_configuration_object( :return: the configuration object. :raises FileNotFoundError: if the configuration file is not found. """ - configuration_loader = ConfigLoader.from_configuration_type( - component_type.to_configuration_type() - ) + configuration_loader = ConfigLoader.from_configuration_type(package_type) configuration_filename = ( configuration_loader.configuration_class.default_configuration_filename ) @@ -352,7 +352,7 @@ def _load_configuration_object( except FileNotFoundError: raise FileNotFoundError( "{} configuration not found: {}".format( - component_type.value.capitalize(), configuration_filepath + package_type.value.capitalize(), configuration_filepath ) ) return configuration_object diff --git a/tests/test_cli/test_local_registry_update.py b/tests/test_cli/test_local_registry_update.py new file mode 100644 index 0000000000..3b5a41b6ed --- /dev/null +++ b/tests/test_cli/test_local_registry_update.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This test module contains the tests for the `aea local-registry-sync""" +import os +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from aea.cli.core import cli +from aea.cli.local_registry_sync import enlist_packages +from aea.cli.registry.add import fetch_package +from aea.configurations.data_types import PackageId, PackageType, PublicId +from aea.helpers.base import cd +from aea.test_tools.click_testing import CliRunner + + +def test_local_registry_update(): + """Test local-registry-sync cli command.""" + PACKAGES = [ + PackageId(PackageType.CONNECTION, PublicId("fetchai", "local", "0.11.0")), + PackageId(PackageType.AGENT, PublicId("fetchai", "my_first_aea", "0.10.0")), + ] + with TemporaryDirectory() as tmp_dir: + for package_id in PACKAGES: + package_dir = os.path.join( + tmp_dir, + package_id.public_id.author, + str(package_id.package_type.to_plural()), + package_id.public_id.name, + ) + os.makedirs(package_dir) + fetch_package( + str(package_id.package_type), + public_id=package_id.public_id, + cwd=tmp_dir, + dest=package_dir, + ) + + assert set(PACKAGES) == set([i[0] for i in enlist_packages(tmp_dir)]) + + runner = CliRunner() + with cd(tmp_dir): + # check intention to upgrade + with patch( + "aea.cli.local_registry_sync.replace_package" + ) as replace_package_mock: + result = runner.invoke( + cli, "local-registry-sync", catch_exceptions=False + ) + assert result.exit_code == 0, result.stdout + assert replace_package_mock.call_count == len(PACKAGES) + + # do actual upgrade + result = runner.invoke(cli, "local-registry-sync", catch_exceptions=False) + assert result.exit_code == 0, result.stdout + + # check next update will do nothing + with patch( + "aea.cli.local_registry_sync.replace_package" + ) as replace_package_mock: + result = runner.invoke( + cli, "local-registry-sync", catch_exceptions=False + ) + assert result.exit_code == 0, result.stdout + assert replace_package_mock.call_count == 0 + + def sort_(packages): + return sorted(packages, key=lambda x: str(x)) + + new_packages = [i[0] for i in enlist_packages(tmp_dir)] + + for new_package, old_package in zip(sort_(new_packages), sort_(PACKAGES)): + assert new_package.public_id > old_package.public_id diff --git a/tests/test_cli/test_misc.py b/tests/test_cli/test_misc.py index f6aed6bdb3..fa42e1838b 100644 --- a/tests/test_cli/test_misc.py +++ b/tests/test_cli/test_misc.py @@ -60,41 +60,42 @@ def test_flag_help(): --help Show this message and exit. Commands: - add Add a package to the agent. - add-key Add a private key to the wallet of the agent. - build Build the agent and its components. - config Read or modify a configuration of the agent. - create Create a new agent. - delete Delete an agent. - eject Eject a vendor package of the agent. - fetch Fetch an agent from the registry. - fingerprint Fingerprint a non-vendor package of the agent. - freeze Get the dependencies of the agent. - generate Generate a package for the agent. - generate-key Generate a private key and place it in a file. - generate-wealth Generate wealth for the agent on a test network. - get-address Get the address associated with a private key of the... - get-multiaddress Get the multiaddress associated with a private key or... - get-wealth Get the wealth associated with the private key of the... - gui Run the CLI GUI. - init Initialize your AEA configurations. - install Install the dependencies of the agent. - interact Interact with the running agent via the stub connection. - issue-certificates Issue certificates for connections that require them. - launch Launch many agents at the same time. - list List the installed packages of the agent. - login Login to the registry account. - logout Logout from the registry account. - publish Publish the agent to the registry. - push Push a non-vendor package of the agent to the registry. - register Create a new registry account. - remove Remove a package from the agent. - remove-key Remove a private key from the wallet of the agent. - reset_password Reset the password of the registry account. - run Run the agent. - scaffold Scaffold a package for the agent. - search Search for packages in the registry. - transfer Transfer wealth associated with a private key of the... - upgrade Upgrade the packages of the agent. + add Add a package to the agent. + add-key Add a private key to the wallet of the agent. + build Build the agent and its components. + config Read or modify a configuration of the agent. + create Create a new agent. + delete Delete an agent. + eject Eject a vendor package of the agent. + fetch Fetch an agent from the registry. + fingerprint Fingerprint a non-vendor package of the agent. + freeze Get the dependencies of the agent. + generate Generate a package for the agent. + generate-key Generate a private key and place it in a file. + generate-wealth Generate wealth for the agent on a test network. + get-address Get the address associated with a private key of the... + get-multiaddress Get the multiaddress associated with a private key or... + get-wealth Get the wealth associated with the private key of the... + gui Run the CLI GUI. + init Initialize your AEA configurations. + install Install the dependencies of the agent. + interact Interact with the running agent via the stub... + issue-certificates Issue certificates for connections that require them. + launch Launch many agents at the same time. + list List the installed packages of the agent. + local-registry-sync Upgrade the local package registry. + login Login to the registry account. + logout Logout from the registry account. + publish Publish the agent to the registry. + push Push a non-vendor package of the agent to the registry. + register Create a new registry account. + remove Remove a package from the agent. + remove-key Remove a private key from the wallet of the agent. + reset_password Reset the password of the registry account. + run Run the agent. + scaffold Scaffold a package for the agent. + search Search for packages in the registry. + transfer Transfer wealth associated with a private key of the... + upgrade Upgrade the packages of the agent. """ ) From 6138ede0a8162b0f4fdf43b00b6d72e0f71f1be7 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Fri, 1 Jan 2021 13:46:02 +0300 Subject: [PATCH 123/204] protocols updated cause year changed --- packages/fetchai/protocols/contract_api/__init__.py | 2 +- packages/fetchai/protocols/contract_api/dialogues.py | 2 +- packages/fetchai/protocols/contract_api/message.py | 2 +- packages/fetchai/protocols/contract_api/protocol.yaml | 8 ++++---- packages/fetchai/protocols/contract_api/serialization.py | 2 +- packages/fetchai/protocols/default/__init__.py | 2 +- packages/fetchai/protocols/default/dialogues.py | 2 +- packages/fetchai/protocols/default/message.py | 2 +- packages/fetchai/protocols/default/protocol.yaml | 8 ++++---- packages/fetchai/protocols/default/serialization.py | 2 +- packages/fetchai/protocols/fipa/__init__.py | 2 +- packages/fetchai/protocols/fipa/dialogues.py | 2 +- packages/fetchai/protocols/fipa/message.py | 2 +- packages/fetchai/protocols/fipa/protocol.yaml | 8 ++++---- packages/fetchai/protocols/fipa/serialization.py | 2 +- packages/fetchai/protocols/gym/__init__.py | 2 +- packages/fetchai/protocols/gym/dialogues.py | 2 +- packages/fetchai/protocols/gym/message.py | 2 +- packages/fetchai/protocols/gym/protocol.yaml | 8 ++++---- packages/fetchai/protocols/gym/serialization.py | 2 +- packages/fetchai/protocols/http/__init__.py | 2 +- packages/fetchai/protocols/http/dialogues.py | 2 +- packages/fetchai/protocols/http/message.py | 2 +- packages/fetchai/protocols/http/protocol.yaml | 8 ++++---- packages/fetchai/protocols/http/serialization.py | 2 +- packages/fetchai/protocols/ledger_api/__init__.py | 2 +- packages/fetchai/protocols/ledger_api/dialogues.py | 2 +- packages/fetchai/protocols/ledger_api/message.py | 2 +- packages/fetchai/protocols/ledger_api/protocol.yaml | 8 ++++---- packages/fetchai/protocols/ledger_api/serialization.py | 2 +- packages/fetchai/protocols/ml_trade/__init__.py | 2 +- packages/fetchai/protocols/ml_trade/dialogues.py | 2 +- packages/fetchai/protocols/ml_trade/message.py | 2 +- packages/fetchai/protocols/ml_trade/protocol.yaml | 8 ++++---- packages/fetchai/protocols/ml_trade/serialization.py | 2 +- packages/fetchai/protocols/oef_search/__init__.py | 2 +- packages/fetchai/protocols/oef_search/dialogues.py | 2 +- packages/fetchai/protocols/oef_search/message.py | 2 +- packages/fetchai/protocols/oef_search/protocol.yaml | 8 ++++---- packages/fetchai/protocols/oef_search/serialization.py | 2 +- packages/fetchai/protocols/prometheus/__init__.py | 2 +- packages/fetchai/protocols/prometheus/dialogues.py | 2 +- packages/fetchai/protocols/prometheus/message.py | 2 +- packages/fetchai/protocols/prometheus/protocol.yaml | 8 ++++---- packages/fetchai/protocols/prometheus/serialization.py | 2 +- packages/fetchai/protocols/register/__init__.py | 2 +- packages/fetchai/protocols/register/dialogues.py | 2 +- packages/fetchai/protocols/register/message.py | 2 +- packages/fetchai/protocols/register/protocol.yaml | 8 ++++---- packages/fetchai/protocols/register/serialization.py | 2 +- packages/fetchai/protocols/signing/__init__.py | 2 +- packages/fetchai/protocols/signing/dialogues.py | 2 +- packages/fetchai/protocols/signing/message.py | 2 +- packages/fetchai/protocols/signing/protocol.yaml | 8 ++++---- packages/fetchai/protocols/signing/serialization.py | 2 +- packages/fetchai/protocols/state_update/__init__.py | 2 +- packages/fetchai/protocols/state_update/dialogues.py | 2 +- packages/fetchai/protocols/state_update/message.py | 2 +- packages/fetchai/protocols/state_update/protocol.yaml | 8 ++++---- packages/fetchai/protocols/state_update/serialization.py | 2 +- packages/fetchai/protocols/tac/__init__.py | 2 +- packages/fetchai/protocols/tac/dialogues.py | 2 +- packages/fetchai/protocols/tac/message.py | 2 +- packages/fetchai/protocols/tac/protocol.yaml | 8 ++++---- packages/fetchai/protocols/tac/serialization.py | 2 +- tests/data/generator/t_protocol/__init__.py | 2 +- tests/data/generator/t_protocol/dialogues.py | 2 +- tests/data/generator/t_protocol/message.py | 2 +- tests/data/generator/t_protocol/protocol.yaml | 8 ++++---- tests/data/generator/t_protocol/serialization.py | 2 +- tests/data/generator/t_protocol_no_ct/__init__.py | 2 +- tests/data/generator/t_protocol_no_ct/dialogues.py | 2 +- tests/data/generator/t_protocol_no_ct/message.py | 2 +- tests/data/generator/t_protocol_no_ct/protocol.yaml | 8 ++++---- tests/data/generator/t_protocol_no_ct/serialization.py | 2 +- 75 files changed, 120 insertions(+), 120 deletions(-) diff --git a/packages/fetchai/protocols/contract_api/__init__.py b/packages/fetchai/protocols/contract_api/__init__.py index b93cfc3722..ba0df54703 100644 --- a/packages/fetchai/protocols/contract_api/__init__.py +++ b/packages/fetchai/protocols/contract_api/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/contract_api/dialogues.py b/packages/fetchai/protocols/contract_api/dialogues.py index 5cd81f82bf..1ab32fbcb9 100644 --- a/packages/fetchai/protocols/contract_api/dialogues.py +++ b/packages/fetchai/protocols/contract_api/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/contract_api/message.py b/packages/fetchai/protocols/contract_api/message.py index e11f74f4e7..ea28cf03af 100644 --- a/packages/fetchai/protocols/contract_api/message.py +++ b/packages/fetchai/protocols/contract_api/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/contract_api/protocol.yaml b/packages/fetchai/protocols/contract_api/protocol.yaml index b052a9392b..ba2e5c62d0 100644 --- a/packages/fetchai/protocols/contract_api/protocol.yaml +++ b/packages/fetchai/protocols/contract_api/protocol.yaml @@ -7,13 +7,13 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmXmHpD7cKDzuvSu7dWXuNZhXp9weNhghauDopSPexSgif - __init__.py: QmcZFuqoBkEx1fYytpLr7142vtXf9qh8cpeRYVZcaWdmrD + __init__.py: QmY2WGKdVE1vExjbKz46U9dgxgEhy2anyQG12ax2NNNuot contract_api.proto: QmSZpXyFMf2MNgVud8iPinaaerx1CManffgHNMx8FcD8jY contract_api_pb2.py: QmYEVg28AVRiLDkbddDto51bumh38gRuouagn6wspDtZVN custom_types.py: QmT51f1pdiAMCnBkLJiYA2aKfreRqkyohq32RiM7vGfjVu - dialogues.py: QmaSSvNewnT3LrR7GgrAwnH4ifpceSAuRMopxu1pNL9x8c - message.py: QmSZZ7jcXHpjQpTSwhafgUTCab7g4gowaj9SvxeuHQDQqL - serialization.py: QmPNTw6vXbdw9GMUwCCGyoHNxopVE1ipcp5DriSn3kGiB8 + dialogues.py: QmNk7CJhkAJ97e4jVmUFDGPUJaXCk3PQp6pzHcFv9hpsg5 + message.py: QmQC1WtSV4mpRz7FwJz6MFzdFzaKHjzCgdURzeTFjCmGRp + serialization.py: QmX2gUAStbfjSotF1k8fsTSLzsVadyVctYNN9fw9jCzgd9 fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/contract_api/serialization.py b/packages/fetchai/protocols/contract_api/serialization.py index e653e24335..ad8694c7b0 100644 --- a/packages/fetchai/protocols/contract_api/serialization.py +++ b/packages/fetchai/protocols/contract_api/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/default/__init__.py b/packages/fetchai/protocols/default/__init__.py index 3a99aef056..4cd1553362 100644 --- a/packages/fetchai/protocols/default/__init__.py +++ b/packages/fetchai/protocols/default/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/default/dialogues.py b/packages/fetchai/protocols/default/dialogues.py index 4352e589b3..568e9f5f10 100644 --- a/packages/fetchai/protocols/default/dialogues.py +++ b/packages/fetchai/protocols/default/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/default/message.py b/packages/fetchai/protocols/default/message.py index 2c6834cb8a..33689259c3 100644 --- a/packages/fetchai/protocols/default/message.py +++ b/packages/fetchai/protocols/default/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/default/protocol.yaml b/packages/fetchai/protocols/default/protocol.yaml index a793a6aea9..b60aad0243 100644 --- a/packages/fetchai/protocols/default/protocol.yaml +++ b/packages/fetchai/protocols/default/protocol.yaml @@ -7,13 +7,13 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmdxJPsHzYGpbNMWgXR1so3A87ns1vevGtzeh1WR2LoZ6G - __init__.py: QmRWWkHpiBGcYVXSvnxZLFD9QCyKcdtvbZ6GhLHLUWFa6G + __init__.py: Qmcv4DkrYV8HETwDvounAzdvXjFydt4jzHHWhu5uTk1ha8 custom_types.py: QmRcgwDdTxkSHyfF9eoMtsb5P5GJDm4oyLq5W6ZBko1MFU default.proto: QmNMoGYhsiiqQxZGGECeczvmKWnYYxyJ6TNTpvb679c4J1 default_pb2.py: QmSbakk8PG4VWSNcDG8mwoSzMgyjKbR9ZbmZjMPnCVmM2e - dialogues.py: QmUf8x7AWjzf4YF2pnjyPZryKwzgoEMeEfvL4ypdy889Y7 - message.py: Qmec2cSx4HC6Yzj3J6Zof7VUQTDRNEyDRHzSBMnCAhMqE3 - serialization.py: QmTJZno4tZPCFFa5ksDmvHuuNyigngCoR2kvSZdMTp3ukb + dialogues.py: QmXfP6bCy49A24RJYnzKZ6HBsf41hLyfLnV6VtaWgjBCWN + message.py: QmUZCG6QsTPTbwzbkKej2brJHvtv7Rn7HXMZ6NwpTD5cxH + serialization.py: QmTc271Ymt2tjqjgeR9ASTFujmNFi9WQKM2G7LWPrqSpK2 fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/default/serialization.py b/packages/fetchai/protocols/default/serialization.py index 5d96aa4c41..b342c18b58 100644 --- a/packages/fetchai/protocols/default/serialization.py +++ b/packages/fetchai/protocols/default/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/fipa/__init__.py b/packages/fetchai/protocols/fipa/__init__.py index 84ca80bc01..a36f6b41c5 100644 --- a/packages/fetchai/protocols/fipa/__init__.py +++ b/packages/fetchai/protocols/fipa/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/fipa/dialogues.py b/packages/fetchai/protocols/fipa/dialogues.py index aea14d5a4d..92d7df0e3a 100644 --- a/packages/fetchai/protocols/fipa/dialogues.py +++ b/packages/fetchai/protocols/fipa/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/fipa/message.py b/packages/fetchai/protocols/fipa/message.py index 1d8885b55f..2eb22f9ebc 100644 --- a/packages/fetchai/protocols/fipa/message.py +++ b/packages/fetchai/protocols/fipa/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/fipa/protocol.yaml b/packages/fetchai/protocols/fipa/protocol.yaml index fe511865b8..f4367213b8 100644 --- a/packages/fetchai/protocols/fipa/protocol.yaml +++ b/packages/fetchai/protocols/fipa/protocol.yaml @@ -7,13 +7,13 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmS3NhEEhPHKb2w1GFVRygmCY8sA2uyTDZApWi1Z6B2onT - __init__.py: QmR6pcWX14FsQip4eYJRNeiQdrNMPj6y4m6Tsgd6hd7yU6 + __init__.py: QmcqbJT7w5RhQVBx5W55iMwa1AgxM4LFu5RRPLmWAAxiKe custom_types.py: Qmf72KRbkNsxxAHwMtkmJc5TRL23fU7AuzJAdSTftckwJQ - dialogues.py: QmUGcpRpjx44DCXqi7RmD9YB7R7AoCHqeGjgTqyJRWvqBi + dialogues.py: QmYioX35NA2PAQNUDaNwu6K6PF1YpnBEwDY6DPqubcUy6B fipa.proto: QmZZV4YhcQXMSqWFsiyqb9b6c67NFmrsB4sbDre49GRHNp fipa_pb2.py: QmUdsdHFuXFqNbm373dugcEZ6WcNiAZRNkNLN81yuvfjh9 - message.py: QmbtqMXBdaRvsGg7RqpWvtGjguFU1nAyg2BBmiBFxAHf4d - serialization.py: QmaYjDAEyCe3jLktynmBqwaSquSz4YVj1rVpXuhZGK56Yi + message.py: QmRTcjGDyfsB54SjJeTh89Yz2g6FrmNMLvCfCTHTmA4knt + serialization.py: QmTRbLt1n8b7c9Yuy4G6WkL9y7iHw6g3GcjNBxU3aKYXtS fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/fipa/serialization.py b/packages/fetchai/protocols/fipa/serialization.py index 93c99bacfc..ab4e745214 100644 --- a/packages/fetchai/protocols/fipa/serialization.py +++ b/packages/fetchai/protocols/fipa/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/gym/__init__.py b/packages/fetchai/protocols/gym/__init__.py index 6e25f7d246..2a140125af 100644 --- a/packages/fetchai/protocols/gym/__init__.py +++ b/packages/fetchai/protocols/gym/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/gym/dialogues.py b/packages/fetchai/protocols/gym/dialogues.py index 7b2c3ee018..863aa21403 100644 --- a/packages/fetchai/protocols/gym/dialogues.py +++ b/packages/fetchai/protocols/gym/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/gym/message.py b/packages/fetchai/protocols/gym/message.py index f5b855f388..3077b7f5d9 100644 --- a/packages/fetchai/protocols/gym/message.py +++ b/packages/fetchai/protocols/gym/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/gym/protocol.yaml b/packages/fetchai/protocols/gym/protocol.yaml index 67260474ee..5f529f662f 100644 --- a/packages/fetchai/protocols/gym/protocol.yaml +++ b/packages/fetchai/protocols/gym/protocol.yaml @@ -7,13 +7,13 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmPjZperegz1BdXFdjguECvZQnYeGzPyYf2qG3WoQK98Lp - __init__.py: QmQvogZ6FVrp15UX2GZ2YKqZASS9gamA72MGt79oieE2tq + __init__.py: QmeyBLjo4xgz4mXvg9dwwwQCapYsbX4cgBXcpwvegg4zWZ custom_types.py: QmT3VKT86xZKR11RR1vQ3myRpmVZNdzY6ELd8HG9U2ngwa - dialogues.py: QmPEFYLc966VzBV4UQb53HEtizY6D9RWP54q2gzkzmbnk9 + dialogues.py: QmUx8de3kabB45rkbpddSpYHfmCnw74QjCjeuP7hC522Yb gym.proto: QmbrGMjAwLXxg4vZTTsdNkbsudhJbSbvkG2mag9RP6ejEg gym_pb2.py: QmPE79TZQjxqxCydj3t2gdPUeFwDXAR3mtWDqKEQfPvQe2 - message.py: QmPX8Y6odg32uAb8UX3yubVw8G7NkTUGMbc32idNdTrcDK - serialization.py: QmT2d4sLcJ96Yf2GEBoKqL3oq4pE518yQvK5WbeHaDXMSQ + message.py: QmPPzVnWong31qYJDak4tRmybbTDwbuSgFthrshJj2VW8b + serialization.py: QmQoL5VtkJCGgceAowdnRE2AvXcVSJJiGNP3dPaiBeQxXF fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/gym/serialization.py b/packages/fetchai/protocols/gym/serialization.py index d005731d68..5c0c5c647c 100644 --- a/packages/fetchai/protocols/gym/serialization.py +++ b/packages/fetchai/protocols/gym/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/http/__init__.py b/packages/fetchai/protocols/http/__init__.py index 9ec736425a..c6e0facc0c 100644 --- a/packages/fetchai/protocols/http/__init__.py +++ b/packages/fetchai/protocols/http/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/http/dialogues.py b/packages/fetchai/protocols/http/dialogues.py index f9e73ed227..e5ebfcf404 100644 --- a/packages/fetchai/protocols/http/dialogues.py +++ b/packages/fetchai/protocols/http/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/http/message.py b/packages/fetchai/protocols/http/message.py index 7672501c8d..af92b2abb4 100644 --- a/packages/fetchai/protocols/http/message.py +++ b/packages/fetchai/protocols/http/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/http/protocol.yaml b/packages/fetchai/protocols/http/protocol.yaml index 40087240f4..d0e05d8afd 100644 --- a/packages/fetchai/protocols/http/protocol.yaml +++ b/packages/fetchai/protocols/http/protocol.yaml @@ -7,12 +7,12 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmaxcPNm1aypK6tynS2xjBNn3ke8AhXTagtz3z5q9z272S - __init__.py: QmWzgWYrnS7PhjYrrx2mykLoaCbb7rDnVRcDqifsRukTy4 - dialogues.py: QmaZSG5heTuTYqeBcwRepCFXSgrzRrLCKysnEVQR5XCCzo + __init__.py: QmeXYyvvRiMhy3GRxPNU4EtHibVN26SvZT5r5RcxLGaE5D + dialogues.py: QmSBawWTSmg42xnVFZ2HHhQ7XjmhZab7UQ3iMDwwFUshLd http.proto: QmZdfqJYikfp8bcCzL2hLntDnfE6r5GKkcvwCBnhYLEkRD http_pb2.py: QmPbNBKxZjY3tGUXt4o4RTc2RFRErXi4ML3MEk56v9fMDK - message.py: QmPjVEnvwssNHq3QieTVbgBNtdXPYH21tghtks6L5vzVjH - serialization.py: QmbhfaMqjyBJW484pxkT1Sec4p8PRLkfETkqocS6Bj6W6g + message.py: QmYeHUEob5SCX8m5c3M2HvFaHgf8dWe6PGGUG6hYriG43d + serialization.py: QmUfoq7RSFuuRNGHBt8oDymPjEk17U4AjbaZgSkQ9bbUcz fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/http/serialization.py b/packages/fetchai/protocols/http/serialization.py index 976fd3e111..e77c7f20b2 100644 --- a/packages/fetchai/protocols/http/serialization.py +++ b/packages/fetchai/protocols/http/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ledger_api/__init__.py b/packages/fetchai/protocols/ledger_api/__init__.py index 99f0907397..93700bfe28 100644 --- a/packages/fetchai/protocols/ledger_api/__init__.py +++ b/packages/fetchai/protocols/ledger_api/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ledger_api/dialogues.py b/packages/fetchai/protocols/ledger_api/dialogues.py index dfe4d017a6..f1449c1d09 100644 --- a/packages/fetchai/protocols/ledger_api/dialogues.py +++ b/packages/fetchai/protocols/ledger_api/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ledger_api/message.py b/packages/fetchai/protocols/ledger_api/message.py index 519a639e23..61a1bd8adc 100644 --- a/packages/fetchai/protocols/ledger_api/message.py +++ b/packages/fetchai/protocols/ledger_api/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ledger_api/protocol.yaml b/packages/fetchai/protocols/ledger_api/protocol.yaml index fee74384c0..026eb7230a 100644 --- a/packages/fetchai/protocols/ledger_api/protocol.yaml +++ b/packages/fetchai/protocols/ledger_api/protocol.yaml @@ -7,13 +7,13 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmNqqzJepaeagpSLMztqFSLNurUdvo69SpS689RneNWJVn - __init__.py: QmX6ta6j6ust7qhVk1kZygzZK3gTTg7hSCBbSMKUxJgWgG + __init__.py: QmRWoVtn2moDouQ7gsdMKxNp3Pi5Z2EFLzhCkvowLZgt6i custom_types.py: QmSsjSb6NzKgqExpG26TfdEyJALFSosLYU5NcLxNVeDjWc - dialogues.py: QmcuATXqitgySbLujTF2s6B79qaKVCMQgSb5iGDrDfiezQ + dialogues.py: QmRKQLnkoJ5PsSoS9r3MCLnQ4E1ordxETkJcgcuXrjSx92 ledger_api.proto: QmTZbaYwaY57uQyYEz6B5qNrCkAJMDnokAzfWBX8BrRBEi ledger_api_pb2.py: QmbSRQTh59Zjy1kT1Rhra5WhxqwVTe5Xv5E7n4QC6v9qcp - message.py: QmVHm2yrWf2YDuJgG3zp4W5NY9H99ktuFnz4diskgwgj8s - serialization.py: Qmbkyy4aNmAGAM7PReTMv9HbpwDQJmHnMsf4L46gWavBTd + message.py: QmT1aL9Zj2obye4GDgrh8C3ZjoTUAoJmcFQJmEWvrHneRu + serialization.py: QmefYH5vyu8RJBx1rcAoYgszhc5t3xXuFWEvLQPs1JtKrJ fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/ledger_api/serialization.py b/packages/fetchai/protocols/ledger_api/serialization.py index 422471ed0c..f138c1ccf4 100644 --- a/packages/fetchai/protocols/ledger_api/serialization.py +++ b/packages/fetchai/protocols/ledger_api/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ml_trade/__init__.py b/packages/fetchai/protocols/ml_trade/__init__.py index bb602e2a47..a1d916b5ef 100644 --- a/packages/fetchai/protocols/ml_trade/__init__.py +++ b/packages/fetchai/protocols/ml_trade/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ml_trade/dialogues.py b/packages/fetchai/protocols/ml_trade/dialogues.py index 3f89804c13..0074c3b53c 100644 --- a/packages/fetchai/protocols/ml_trade/dialogues.py +++ b/packages/fetchai/protocols/ml_trade/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ml_trade/message.py b/packages/fetchai/protocols/ml_trade/message.py index 280768b87a..b773c7d5c1 100644 --- a/packages/fetchai/protocols/ml_trade/message.py +++ b/packages/fetchai/protocols/ml_trade/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/ml_trade/protocol.yaml b/packages/fetchai/protocols/ml_trade/protocol.yaml index 95a98e39b2..6d16a69fee 100644 --- a/packages/fetchai/protocols/ml_trade/protocol.yaml +++ b/packages/fetchai/protocols/ml_trade/protocol.yaml @@ -7,13 +7,13 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmccSUTpVzDpaAqpGPGXWY5aG9Dp1yStX4Sn2tqCjsKVEC - __init__.py: QmcCS9uUQTTS2w85dTNiN5rQ14wyBhmBkr7pPPPcbLphcn + __init__.py: QmRnYbHxrQvb86cdheENeUhP7PswboKgugHaVMpcdPzaqR custom_types.py: QmPa6mxbN8WShsniQxJACfzAPRjGzYLbUFGoVU4N9DewUw - dialogues.py: Qmbras4sV8Fz7cnXwXMJLM5G4Y6rvSWvXakfTjDbyddbY5 - message.py: QmYXwvWBCQkfPKGB8cbNPeomxS248K4CvgW8djNuXCq7Qa + dialogues.py: QmUZ3BiJYY2bNiqrvwPZyNsjZ3v9Rvj8EW1UkH52XxjAqd + message.py: QmQzujKk8d4KztAK8Wug9FWhKdpeotiftCGX5aXmGGvPDA ml_trade.proto: QmNvSW9EUYiyu5SLhCVVaAdzLi1yjM94mnUeNSBzu2GRYH ml_trade_pb2.py: QmcrA4D7C9LFPcj6QKe8uUxYdHfHpRmZ5Dm4U5dKpT9nPh - serialization.py: QmNnEyqVdHuXXQLjAvbyibmdRbWBZG8tNEQy32s7SAVtcE + serialization.py: Qman9RVkzmHhnfU9VFbQ921nPWSuWdASvJSgfJqzsczEam fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/ml_trade/serialization.py b/packages/fetchai/protocols/ml_trade/serialization.py index 3b3c34ffd6..7bb84baf07 100644 --- a/packages/fetchai/protocols/ml_trade/serialization.py +++ b/packages/fetchai/protocols/ml_trade/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/oef_search/__init__.py b/packages/fetchai/protocols/oef_search/__init__.py index f130fc2c1b..681bacceaa 100644 --- a/packages/fetchai/protocols/oef_search/__init__.py +++ b/packages/fetchai/protocols/oef_search/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/oef_search/dialogues.py b/packages/fetchai/protocols/oef_search/dialogues.py index 7e56cc5c53..2579c15314 100644 --- a/packages/fetchai/protocols/oef_search/dialogues.py +++ b/packages/fetchai/protocols/oef_search/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/oef_search/message.py b/packages/fetchai/protocols/oef_search/message.py index 9b3b5f4132..66840af9c3 100644 --- a/packages/fetchai/protocols/oef_search/message.py +++ b/packages/fetchai/protocols/oef_search/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/oef_search/protocol.yaml b/packages/fetchai/protocols/oef_search/protocol.yaml index b5f5f8bac2..f86384d43b 100644 --- a/packages/fetchai/protocols/oef_search/protocol.yaml +++ b/packages/fetchai/protocols/oef_search/protocol.yaml @@ -7,13 +7,13 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: Qmdh6FwhH48U94pvhNfK3X5vwBgjKJzNofmd9FDyNEq4nS - __init__.py: Qmdr5ks5X4YtnpH6yKUcNu9uouyv3EGmrKFhyvNH7ZBjvT + __init__.py: QmZLjUfh8xW6dLaUgHZmTpX82giavcPQHwSKyx8X6ao3Xx custom_types.py: QmWbpGmegvEkiQCEEkCAjQoVFppKt23VXVXaBC1KbhCLbD - dialogues.py: QmVRDcsDPZEh5QcbaveAKHqSG7VZNvJPCjYbayfujsjbHf - message.py: QmUpvMGcR45d3e51PFCaGNwcKNwCjGeqgmoC6pZVCe9u1k + dialogues.py: QmSs2nefhrSMXE13QUT9U1q3JVSjzr3dPgsHBqYkCw2fUB + message.py: QmaCZb3pkF9QdpJnXVZb9pBFsUskXRajq8SdQ7XVwWjzfS oef_search.proto: QmdBhNnBMnUDiX9UD7eLy6HHKyUvmPrhys44fr1EjF4fnR oef_search_pb2.py: QmQbMm1pWABNZ9aPdcsJvX5icS1qrTi2CntePCVbtv5777 - serialization.py: QmcMQLbz6fkvZeqUXyE9WwH4TEJ3Dzy6pV4txLAVw9sdwb + serialization.py: QmNvuE9y9Xbi661GJmGMHt13kRaz9pWWRv9LXAYJMYTrwR fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/oef_search/serialization.py b/packages/fetchai/protocols/oef_search/serialization.py index 3d0348c1f7..ac14bcd8e3 100644 --- a/packages/fetchai/protocols/oef_search/serialization.py +++ b/packages/fetchai/protocols/oef_search/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/prometheus/__init__.py b/packages/fetchai/protocols/prometheus/__init__.py index 4eecec584f..e849e6eaf9 100644 --- a/packages/fetchai/protocols/prometheus/__init__.py +++ b/packages/fetchai/protocols/prometheus/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/prometheus/dialogues.py b/packages/fetchai/protocols/prometheus/dialogues.py index b58bb0ba52..6778e07f8d 100644 --- a/packages/fetchai/protocols/prometheus/dialogues.py +++ b/packages/fetchai/protocols/prometheus/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/prometheus/message.py b/packages/fetchai/protocols/prometheus/message.py index fc35dbfc9e..d7c889878a 100644 --- a/packages/fetchai/protocols/prometheus/message.py +++ b/packages/fetchai/protocols/prometheus/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/prometheus/protocol.yaml b/packages/fetchai/protocols/prometheus/protocol.yaml index f07fb1dd6f..bfb81792c0 100644 --- a/packages/fetchai/protocols/prometheus/protocol.yaml +++ b/packages/fetchai/protocols/prometheus/protocol.yaml @@ -7,12 +7,12 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmZb4NVBU8G417AcXM4Lj427MZR2D6guCvBeMiB7uhQp2m - __init__.py: QmYcCsWNz4qmSUBorY9N4XgyXvdYrrzNGr6oP3iQP9wUM5 - dialogues.py: QmSg5DmxpnvN9g2ThzmB4N6fJ2SeQ63iGa2ByGrLQD6BKF - message.py: QmXKpTPb7HLUdL2D5ycwGSLFchBSwj4Qc4mSc2Jrawhzjg + __init__.py: QmQvMGLitiy95SKYYSZQdfUH2UjpZ1ueTJooVuBG19P7QE + dialogues.py: QmYQYkByWC3sBnQPnDEzFh98ULycx1jzqqs6hLC196YZg5 + message.py: QmTQQeP2rv2QNG3AKCoMW2uP4rMNyUsG9BreSScptkUFJm prometheus.proto: QmUmvB2uvp8gdWzvUoxDJyc4Lvi4bFuaSdqMTMyk6hBhgF prometheus_pb2.py: QmXwmam9ZPyR3yf6UTmgMNVuvxTAZzgx6hPohzVPmKq452 - serialization.py: QmUgeNdHPqtCTyRM5K42a8rtzr1RYP94YiV7qycGDxdYzg + serialization.py: QmaWVbKwubB4pymQpQQg3yXfjeEdxrehxcHDkb8o4SeXun fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/prometheus/serialization.py b/packages/fetchai/protocols/prometheus/serialization.py index 17a5dd696f..52e86f2984 100644 --- a/packages/fetchai/protocols/prometheus/serialization.py +++ b/packages/fetchai/protocols/prometheus/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/register/__init__.py b/packages/fetchai/protocols/register/__init__.py index 435c5af99b..fc318e5f60 100644 --- a/packages/fetchai/protocols/register/__init__.py +++ b/packages/fetchai/protocols/register/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/register/dialogues.py b/packages/fetchai/protocols/register/dialogues.py index 717e465967..c0d52fa7c3 100644 --- a/packages/fetchai/protocols/register/dialogues.py +++ b/packages/fetchai/protocols/register/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/register/message.py b/packages/fetchai/protocols/register/message.py index c350a2ed1e..b3eeed3705 100644 --- a/packages/fetchai/protocols/register/message.py +++ b/packages/fetchai/protocols/register/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/register/protocol.yaml b/packages/fetchai/protocols/register/protocol.yaml index f85b36d763..0cd4c2f53f 100644 --- a/packages/fetchai/protocols/register/protocol.yaml +++ b/packages/fetchai/protocols/register/protocol.yaml @@ -7,12 +7,12 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmNRAfWZWLmEwBZ5Lq9fZr1ujWryiL4opuykzPUgiWVkkw - __init__.py: QmVqqvCtZJ6RNioinbGtGXqXQdxW38jgRjMyBpLhK6HTAz - dialogues.py: QmcUiHEdDbJLD14bb4RSYkXFJdtxJX74hd52QX6q4oGiSQ - message.py: QmanYNfvNFW9tjSbwAvQNS2V4JjBkmPRrbNf6SAfsr9RrZ + __init__.py: QmZgAVL7zheA4UnmaYBBaFetMyhFBVyTvcLF5HMvTicGvF + dialogues.py: QmfBmUxThNF1Qgz4bSrvPPPVYqxvdRtaMuXpQUQ5XVTPQC + message.py: QmP3x5QqxtHQFy4rRvLeiXAqYuaX4bwYGNgtbhk7DNqeDw register.proto: QmRuQ3XkDM668dsSSKrKcwt8SZqttT6bmVARgbDvs7b5tp register_pb2.py: QmVhxx411rnYPhbf7ipjFNLQgxUNkfRif3wk8XLz4she9p - serialization.py: QmcDqszzn1juRwmiLQAh1ZttB7ScS4zSizNo5qVUbMtXpy + serialization.py: QmP3BiPfJZ8JYdPLdGRWUA6dpJRdsAvt5mLuoXDMMBBzDX fingerprint_ignore_patterns: [] dependencies: protobuf: {} diff --git a/packages/fetchai/protocols/register/serialization.py b/packages/fetchai/protocols/register/serialization.py index 3b73ba57f5..223285acf7 100644 --- a/packages/fetchai/protocols/register/serialization.py +++ b/packages/fetchai/protocols/register/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/signing/__init__.py b/packages/fetchai/protocols/signing/__init__.py index 194ca24a25..3f6550ff81 100644 --- a/packages/fetchai/protocols/signing/__init__.py +++ b/packages/fetchai/protocols/signing/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/signing/dialogues.py b/packages/fetchai/protocols/signing/dialogues.py index bc884340b0..0c8e470113 100644 --- a/packages/fetchai/protocols/signing/dialogues.py +++ b/packages/fetchai/protocols/signing/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/signing/message.py b/packages/fetchai/protocols/signing/message.py index 03e39a53f4..94704a599f 100644 --- a/packages/fetchai/protocols/signing/message.py +++ b/packages/fetchai/protocols/signing/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/signing/protocol.yaml b/packages/fetchai/protocols/signing/protocol.yaml index 037bdb8750..6f661710ed 100644 --- a/packages/fetchai/protocols/signing/protocol.yaml +++ b/packages/fetchai/protocols/signing/protocol.yaml @@ -7,11 +7,11 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmdSbVEY9rf812ZDoFjbPYBG15N4XaVVoHsEc1SXkwxPpp - __init__.py: QmUcmsCfop25uvwR6XQR3GGaJ63pbDsEPQPJeXM7nASuQW + __init__.py: QmPVnWg24cNfoLcZfqtRePdE2exyoKjRL2cfdz5qF7yZjg custom_types.py: Qmc7sAyCQbAaVs5dZf9hFkTrB2BG8VAioWzbyKBAybrQ1J - dialogues.py: QmRZWRnWTJYDtdQAxdUY2MRcFciE6uSrrkb2F4css5tv9K - message.py: QmeJA4NF3dtrCvj7yoAkQ2ZVwUGhudoAxR2xAcvuqyJdn2 - serialization.py: QmVMoDnWyDYchf8MMbBQNSh9FPXev2dXqoMCrSXFWjkLYQ + dialogues.py: Qmamd1T5ZDF8ExQznV9bXSoSC8s9rTgbkDoBRVTAzcARXB + message.py: QmRt1VStX7mEHp8724H6pyaVzoMucYKJ4HzdpsC1wfpthj + serialization.py: QmQY5rs9C3XYErfJ2fFtgqLqJiGx12hGHG5hmFp3ePMhFW signing.proto: QmZN9CmcfXCBiMQd9GTG81LadsbVQQ7j5pLFxEiQsQ4Sqk signing_pb2.py: QmSeawGUWMvFFgDrzuYrVMfWhb5UadXbPcyhzX2rLTdCTK fingerprint_ignore_patterns: [] diff --git a/packages/fetchai/protocols/signing/serialization.py b/packages/fetchai/protocols/signing/serialization.py index 8d5f3a416e..52909ab500 100644 --- a/packages/fetchai/protocols/signing/serialization.py +++ b/packages/fetchai/protocols/signing/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/state_update/__init__.py b/packages/fetchai/protocols/state_update/__init__.py index 785aa79709..c4434f981c 100644 --- a/packages/fetchai/protocols/state_update/__init__.py +++ b/packages/fetchai/protocols/state_update/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/state_update/dialogues.py b/packages/fetchai/protocols/state_update/dialogues.py index a14f685ec9..c71dfe0338 100644 --- a/packages/fetchai/protocols/state_update/dialogues.py +++ b/packages/fetchai/protocols/state_update/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/state_update/message.py b/packages/fetchai/protocols/state_update/message.py index 22a5693f6f..4945723d27 100644 --- a/packages/fetchai/protocols/state_update/message.py +++ b/packages/fetchai/protocols/state_update/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/state_update/protocol.yaml b/packages/fetchai/protocols/state_update/protocol.yaml index 1e767a1bed..3dd6814d7e 100644 --- a/packages/fetchai/protocols/state_update/protocol.yaml +++ b/packages/fetchai/protocols/state_update/protocol.yaml @@ -7,10 +7,10 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmUKL4QkkPJCkKNiw8oF7dbKVAmSs9YkZc5RHaq8ziCYaD - __init__.py: Qmd7GvLL2hzSbbk5JNcHyRDNH7yvhS3SaxitBbQcvA9i7U - dialogues.py: QmTaHNZ5L4T8HoRNgQJ3nxUfKXCXLPAeAiopDy21ybs8o9 - message.py: QmZkz31xtjHCywQaQHDNZXRkTMKtxqpYfdSePTeAdeSW41 - serialization.py: QmPQuxNbWMemgAJ9Lwzzi8K95aTES2DVSVzhJzsN1Njo1E + __init__.py: QmYQASXkhmnZbo5Vxh6mVbcaCkBP46UCZzQHQFtWYvBELL + dialogues.py: QmU4GFxG3DL5F515aF3z2GamzieU6XXWqzRy7V5oA7HnjV + message.py: QmWC9RCfMMNQUA6YLnSyHi6uScuVQGW5XSzibHKtvU4cDY + serialization.py: QmQrpKMDr9E4gtoknmjVenVnhGL8jbkgEmWzsSB8YYx4oh state_update.proto: QmVZjMwsoSC5UYWDVCv2Sp8ELS9TG6sCR5gHqBWAKayRY7 state_update_pb2.py: QmdnTuC7FLetYMPWBk1Nv3TccghovZYddW6BGXcHUQ9nSh fingerprint_ignore_patterns: [] diff --git a/packages/fetchai/protocols/state_update/serialization.py b/packages/fetchai/protocols/state_update/serialization.py index c46759acce..6773b71c48 100644 --- a/packages/fetchai/protocols/state_update/serialization.py +++ b/packages/fetchai/protocols/state_update/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/tac/__init__.py b/packages/fetchai/protocols/tac/__init__.py index 0dd910252c..d7682129cc 100644 --- a/packages/fetchai/protocols/tac/__init__.py +++ b/packages/fetchai/protocols/tac/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/tac/dialogues.py b/packages/fetchai/protocols/tac/dialogues.py index 9d26068c3d..a8dd60a151 100644 --- a/packages/fetchai/protocols/tac/dialogues.py +++ b/packages/fetchai/protocols/tac/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/tac/message.py b/packages/fetchai/protocols/tac/message.py index 2a462d2996..055498f12e 100644 --- a/packages/fetchai/protocols/tac/message.py +++ b/packages/fetchai/protocols/tac/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/packages/fetchai/protocols/tac/protocol.yaml b/packages/fetchai/protocols/tac/protocol.yaml index e300b3f221..4f64b7b1ee 100644 --- a/packages/fetchai/protocols/tac/protocol.yaml +++ b/packages/fetchai/protocols/tac/protocol.yaml @@ -8,11 +8,11 @@ license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: README.md: QmbuKk4QvBmziXByv3nq46o3pmapyf7gMPRQDLRBB6Sric - __init__.py: QmSAC7PGra9fig8RhhF1j3XEVpgie9UZNNYPc2AB9Kx9xJ + __init__.py: QmRuxxNxJgmwtf4eMqVWpFKB2hrpT48GFLUmqRpi1Xicaj custom_types.py: QmXQATfnvuCpt4FicF4QcqCcLj9PQNsSHjCBvVQknWpyaN - dialogues.py: QmSYjYBpE3YZxPuXE42DK7fo47kgF8dqWT9ySYb9qcB51T - message.py: QmcGUgrk2aJffnyyyotv3WpyPph25MarjXjmNhyny32L6V - serialization.py: Qmani4DjtVVNYF1DDUSWAnQu7pC2v1DPvEW6ZQwmp2nMSA + dialogues.py: QmRjWdedEm43231zPpSad7R97BZ567qNSjjjtPnabrziRd + message.py: QmejFGHZAmBW7NTJrSsYA3ZJynGLiBn3DCgUkEjcE7MWpe + serialization.py: QmVHNaPqLYsNe5PTA8qFSWaHonmfkcnxcMbBdjLexSmCH7 tac.proto: QmUXk2kwqp1vo22oZdvLbWKirojeqkXdGSmiz6r14bMqSE tac_pb2.py: QmVHkh5GctFUU36wiVZZfZYYoQxT4uZP8eUzAtgKgUBxn6 fingerprint_ignore_patterns: [] diff --git a/packages/fetchai/protocols/tac/serialization.py b/packages/fetchai/protocols/tac/serialization.py index 559b5a6626..29fcb7a366 100644 --- a/packages/fetchai/protocols/tac/serialization.py +++ b/packages/fetchai/protocols/tac/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol/__init__.py b/tests/data/generator/t_protocol/__init__.py index 56de0d0a42..6e3181c92b 100644 --- a/tests/data/generator/t_protocol/__init__.py +++ b/tests/data/generator/t_protocol/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol/dialogues.py b/tests/data/generator/t_protocol/dialogues.py index 2aeb20997a..596bf59c2b 100644 --- a/tests/data/generator/t_protocol/dialogues.py +++ b/tests/data/generator/t_protocol/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol/message.py b/tests/data/generator/t_protocol/message.py index 4d9eeeab32..96f72f05a1 100644 --- a/tests/data/generator/t_protocol/message.py +++ b/tests/data/generator/t_protocol/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol/protocol.yaml b/tests/data/generator/t_protocol/protocol.yaml index 692604f6e6..d727cad719 100644 --- a/tests/data/generator/t_protocol/protocol.yaml +++ b/tests/data/generator/t_protocol/protocol.yaml @@ -6,11 +6,11 @@ description: A protocol for testing purposes. license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: - __init__.py: QmQy21g5sVYfmy4vSYuEFyPnobM4SA1dEouz5deXNssPWx + __init__.py: QmVEtFTkpwEHfaijkMso9P6JMJCSepSvUKeqcRJLVJMyji custom_types.py: QmWg8HFav8w9tfZfMrTG5Uo7QpexvYKKkhpGPD18233pLw - dialogues.py: Qmaq3wGigpyL2xZwCgqv1CN6tNy1uayzym9ybwjvZRYm5C - message.py: QmYGdkkon76MfdhgBvHrpuog3yCpgZhgDyKXBh4ZRUN7GS - serialization.py: QmYJLfQ49vsqRSpC7GV1ukc3PnkVBuRMM3GBtGoCgDEv5z + dialogues.py: QmWAdikDRJWTG7HUXsCsZRg4Wxnf8cMr5KujpyC4M75gnB + message.py: Qmf4TsCLKT2PudSxm8eUCQVKPfvwUmpSt3bDssEaqPZp1J + serialization.py: QmPBxFuGJ9EZdwT1mP1wZqffYCsQYo7t2kxvSSWnG6msFp t_protocol.proto: QmWdNaAJ9Mkf2SHF1RSZrsk2a5jZyXZtCD7XU5PHLCph5z t_protocol_pb2.py: QmamuU3UsGEFKWPSbt5Qeo31kXSCLpTD81TkuX44njkYgh fingerprint_ignore_patterns: [] diff --git a/tests/data/generator/t_protocol/serialization.py b/tests/data/generator/t_protocol/serialization.py index 2c6b422f86..f0f61e1f1c 100644 --- a/tests/data/generator/t_protocol/serialization.py +++ b/tests/data/generator/t_protocol/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol_no_ct/__init__.py b/tests/data/generator/t_protocol_no_ct/__init__.py index 2518f2ff75..c79ce27d80 100644 --- a/tests/data/generator/t_protocol_no_ct/__init__.py +++ b/tests/data/generator/t_protocol_no_ct/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol_no_ct/dialogues.py b/tests/data/generator/t_protocol_no_ct/dialogues.py index 67d0f9b1cd..bd800706e8 100644 --- a/tests/data/generator/t_protocol_no_ct/dialogues.py +++ b/tests/data/generator/t_protocol_no_ct/dialogues.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol_no_ct/message.py b/tests/data/generator/t_protocol_no_ct/message.py index 8f07d60d6c..c36dd9ccd9 100644 --- a/tests/data/generator/t_protocol_no_ct/message.py +++ b/tests/data/generator/t_protocol_no_ct/message.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/data/generator/t_protocol_no_ct/protocol.yaml b/tests/data/generator/t_protocol_no_ct/protocol.yaml index 515aa32085..03541a26cf 100644 --- a/tests/data/generator/t_protocol_no_ct/protocol.yaml +++ b/tests/data/generator/t_protocol_no_ct/protocol.yaml @@ -6,10 +6,10 @@ description: A protocol for testing purposes. license: Apache-2.0 aea_version: '>=0.8.0, <0.9.0' fingerprint: - __init__.py: QmaaZ7Je2PRTkcnqy8oLR58yBDVpcRQ4BcaRe3sd3fug3Z - dialogues.py: QmXCaVN6cAjHk7cMjRbD2GHxEosKvCp7nFGKMqXf8FThQQ - message.py: QmXB1CxBaQtNqtYRcJFRruFuvytxA4KQtBXpo3jTsdwPrA - serialization.py: QmPX4KzaEfK9JwbbHAmmB1rZp1guJTW5PgPan7ZGAp36DH + __init__.py: QmWsBeSVmG4FZWq4mGEzvERUDmDh4jC59UhYQWFvdYvcPo + dialogues.py: Qmeq7m8vf1LW5WeehNG8qnoGoRstQrABw2vdQh5tmB3KxX + message.py: QmWFjERqT1xxKNEMwwnJx5JXNVtYzxJL1nLLu65nkfWdk5 + serialization.py: QmcxwDYsX1r8Yyy13Ci8vpZFtrpD7RaNXd2MJHCPuXCvBG t_protocol_no_ct.proto: Qmc8KkKnWZ9utBxrbEyWhVDRdut87DkFvmHP3SYUg4J3EU t_protocol_no_ct_pb2.py: QmdCYBHYVZ49fdZ5CHkL72Zt5xQ9H1b3KFyTx9MvnPDnFw fingerprint_ignore_patterns: [] diff --git a/tests/data/generator/t_protocol_no_ct/serialization.py b/tests/data/generator/t_protocol_no_ct/serialization.py index 3e2752017a..b94350afa9 100644 --- a/tests/data/generator/t_protocol_no_ct/serialization.py +++ b/tests/data/generator/t_protocol_no_ct/serialization.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------ # -# Copyright 2020 fetchai +# Copyright 2021 fetchai # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 81a3d45ba4714d14b1665faa42efcf56d0ac4eff Mon Sep 17 00:00:00 2001 From: Yuri Turchenkov Date: Fri, 1 Jan 2021 14:53:46 +0300 Subject: [PATCH 124/204] Update aea/cli/local_registry_sync.py Co-authored-by: David Minarsch --- aea/cli/local_registry_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aea/cli/local_registry_sync.py b/aea/cli/local_registry_sync.py index 0dc92dfc5e..db48d845c8 100644 --- a/aea/cli/local_registry_sync.py +++ b/aea/cli/local_registry_sync.py @@ -60,7 +60,7 @@ def do_local_registry_update(base_dir: Union[str, Path]) -> None: ): click.echo(f"{package_id} is the latest version.") continue - click.echo(f"Updating to {latest_public_id}.") + click.echo(f"Updating {current_public_id} to {latest_public_id}.") replace_package(str(package_id.package_type), latest_public_id, package_dir) From 80e7a26f6dc5b5a6360e32edbbbaf5f820763462 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Fri, 1 Jan 2021 12:15:46 +0000 Subject: [PATCH 125/204] fix hashes --- packages/hashes.csv | 26 +++++++++++++------------- tests/data/hashes.csv | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/hashes.csv b/packages/hashes.csv index 29d0c2287c..6673e87f4c 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -49,20 +49,20 @@ fetchai/contracts/oracle,QmbmDutosj3Zn5qLxvuCK81D1obww1xAMb1iTpchuL9Y3s fetchai/contracts/oracle_client,Qmcv9ctDViLGVxaPBNtaB6M2o6bRQDekJM1ue9XZxA7naB fetchai/contracts/scaffold,QmR8eqeu9i2auL7XjkvbyV4DMf3cG5hEy3eDfVEeeMMrD5 fetchai/contracts/staking_erc20,QmQfqoSKY1CeyWvrvZdEB2ftLXKboXvku7be2t3cNH3t2H -fetchai/protocols/contract_api,QmTRqbP6JrXC76VAx9WsUwgDHYKGbfiib1WGDzXmC8dbF1 -fetchai/protocols/default,QmXmrDnJzARnRyhCMhPXy12Bu8Zq85ZrjbxUUbcNJvYVjm -fetchai/protocols/fipa,QmdZevSTKavH7wCVXeZyiQ5y3sDAkhEQ3SiwhaiXWYgnDJ -fetchai/protocols/gym,QmW8fqeoXvsgdjZzZt6Wfj6vfCJBEweAUFdUk1dqNyBa1m -fetchai/protocols/http,QmYF5MEFZbJUo2xTLJwCt6cTkADbytUqfeWvsGzvuy6hJF -fetchai/protocols/ledger_api,QmWjvwNBpomTDCPEfA86dhHihr2HkCpnveCVZDWU7x5m42 -fetchai/protocols/ml_trade,QmezUacB3b2N31wcs6zxCgurFWubEFJXrfPdaT368LVhTX -fetchai/protocols/oef_search,QmQXGHb6GRgeeeYGy5KVPz6bkaWxo3S6tm3fHqoNJ3Zoij -fetchai/protocols/prometheus,QmR9KCLCWMMwJRkGV313WGxYwk49bLtg6SxoAavYb3C97h -fetchai/protocols/register,QmNwCo2HYKjZzHbjPfmAZnYUBFVURbbFR2JZAye9Wg2WmY +fetchai/protocols/contract_api,QmNQ18rgSAuUWH1KMCMmudi3EXKZmZGEp9GBqZXxavKQMu +fetchai/protocols/default,QmQebB6xck4FfPRpUCsDsQV4sYJoiL3jNeCXwEHkY4XyNK +fetchai/protocols/fipa,QmeduYjwZKP1PQNqQi7T8UP2NLLdPduL854xtjEpR6DDoo +fetchai/protocols/gym,QmSPb74yYvvwACaHQQZ6FUiu55kEeq7XwnDLcrcmasp6mD +fetchai/protocols/http,QmRMF4HUX58VLQusBTDtRpB4Vn2SKoFse3XkZ2CM22Kd58 +fetchai/protocols/ledger_api,QmYhCs8F79TjGJtVC67WZ2GA1a63yChsosXfdkXP3rtFKt +fetchai/protocols/ml_trade,QmRxSdyvHggeCdNBELhUohmvFnTZ6uUBaJ1SLzoLJ4yJ9t +fetchai/protocols/oef_search,QmU5hhGcrtna6EjBJyAatMbM9sAnLuNG39GfnMELXngGQY +fetchai/protocols/prometheus,QmTC8MW6s1xXKnz9TwcxA2Ffw7aPqgZoG4McvxzkeQuRd9 +fetchai/protocols/register,QmWm6SeMSTqRr1wtY1RY2RfoiHBjkJpTnnRTimvtcHS3H3 fetchai/protocols/scaffold,QmUEd8eCvcdPXvV37tZr3pQE7oTFtBZbLkf6GPTMwF8Aiu -fetchai/protocols/signing,QmcTcVkMJ7uCnPF4CnchTdp7k1iz4ADt5xXV3emvzF9iQb -fetchai/protocols/state_update,Qma95xUrwSX7XGPRyfWYPziJLx8kEpYL6j5gCxFYZyLARQ -fetchai/protocols/tac,QmTWucqbXue3H332ivNHMG5eZkSSPxz1aYNtPs3q8YEff5 +fetchai/protocols/signing,QmTVRQBUtxJKhFBQL92qDRF81tCrZxJccLkK69diFiTeD5 +fetchai/protocols/state_update,QmRn5XnwEUU77tEzTTNNS933Fdj5orxZVJgn3pGef822uH +fetchai/protocols/tac,QmXeSbSFunSy7VeNCSq5C7ciwDY5B3ufKd1afKB31zz8AL fetchai/skills/aries_alice,QmdQEHTqFhEkFB9ZALnD19DwskyfLD39du3ukAPYpXYMhv fetchai/skills/aries_faber,QmeveDnkEFYnUmfJN8cdXJGokrtQD47qxyYvtjEt5Qx9oL fetchai/skills/carpark_client,QmWabiH48yVv4hNaufU3jtCqe3GSEsmhMe5DEPeciTYTP3 diff --git a/tests/data/hashes.csv b/tests/data/hashes.csv index dbfeb31e3c..91a088f68e 100644 --- a/tests/data/hashes.csv +++ b/tests/data/hashes.csv @@ -2,7 +2,7 @@ dummy_author/agents/dummy_aea,QmfM8jdCnfg8duVMoVnM5qtMH9w3GRLyLkuCPZHbJMdtx8 dummy_author/skills/dummy_skill,QmSmzcjGNhiz2KpSE6wSi19AeesCVjzupJm1NpXYJVnJuZ fetchai/connections/dummy_connection,Qmep3ive7fVWJi965SSdJv6fdfzAKD1wmP9A1kYRoQjRNn fetchai/contracts/dummy_contract,QmXhMKRF5upco1efaC9oNcesStEMkPWKhBUv6CK9Vngk2S -fetchai/protocols/t_protocol,QmNhzDCznorQpySrvKt4F4sQFXjzF1ZecEd5inixiNavDH -fetchai/protocols/t_protocol_no_ct,QmWwMwHxvuPZJaMh44Bd1d5aBzYiogep8TJswTKkRwLfBu +fetchai/protocols/t_protocol,QmWjV39cYymZmHhNj55zfoGMY9Sa768UykXLhuYiJbD56v +fetchai/protocols/t_protocol_no_ct,QmcKpRB2A89wiMvA9f6wfRZH5K3EmTHP3qrkQazmub1zsu fetchai/skills/dependencies_skill,QmSZtiXSrzj8mnYztRHKM6A8fYwUqhrTLL317r2JqXu8Ga fetchai/skills/exception_skill,QmboGWn3yLp3kLdoCYHYEQ92W4u6LGrzdvaiZ4WaigcCqB From 3e5400a931f3bd2c70f2c2690d11bb37181d21f7 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Sat, 2 Jan 2021 14:04:51 +0000 Subject: [PATCH 126/204] address pr comments --- aea/cli/local_registry_sync.py | 27 ++++++++------ aea/configurations/base.py | 2 +- aea/configurations/data_types.py | 4 +-- aea/configurations/loader.py | 37 +++++++++++++++----- tests/test_cli/test_local_registry_update.py | 8 +++-- tests/test_configurations/test_base.py | 1 + tests/test_exceptions.py | 26 +++++++++++++- 7 files changed, 79 insertions(+), 26 deletions(-) diff --git a/aea/cli/local_registry_sync.py b/aea/cli/local_registry_sync.py index db48d845c8..57f320e798 100644 --- a/aea/cli/local_registry_sync.py +++ b/aea/cli/local_registry_sync.py @@ -30,8 +30,9 @@ from aea.cli.registry.utils import get_package_meta from aea.cli.utils.context import Context from aea.cli.utils.decorators import pass_ctx +from aea.cli.utils.loggers import logger from aea.configurations.data_types import PackageId, PackageType, PublicId -from aea.configurations.loader import load_component_configuration +from aea.configurations.loader import load_package_configuration PACKAGES_DIRS = [i.to_plural() for i in PackageType] @@ -41,10 +42,13 @@ @pass_ctx def local_registry_sync(ctx: Context): """Upgrade the local package registry.""" - do_local_registry_update(ctx.cwd) + skip_consistency_check = ctx.config["skip_consistency_check"] + do_local_registry_update(ctx.cwd, skip_consistency_check) -def do_local_registry_update(base_dir: Union[str, Path]) -> None: +def do_local_registry_update( + base_dir: Union[str, Path], skip_consistency_check: bool = True +) -> None: """ Perform local registry update. @@ -52,7 +56,7 @@ def do_local_registry_update(base_dir: Union[str, Path]) -> None: :return: None """ - for package_id, package_dir in enlist_packages(base_dir): + for package_id, package_dir in enlist_packages(base_dir, skip_consistency_check): current_public_id = package_id.public_id latest_public_id = get_package_latest_public_id(package_id) if not ( # pylint: disable=superfluous-parens @@ -101,7 +105,7 @@ def get_package_latest_public_id(package_id: PackageId) -> PublicId: def enlist_packages( - base_dir: Union[Path, str] + base_dir: Union[Path, str], skip_consistency_check: bool = True ) -> Generator[Tuple[PackageId, Union[Path, str]], None, None]: """ Generate list of the packages in local repo directory. @@ -131,10 +135,13 @@ def enlist_packages( if package_type.to_plural() != package_type_plural: # incorrect package placing continue # pragma: nocover - config = load_component_configuration( - package_type, Path(package_dir), skip_consistency_check=True + config = load_package_configuration( + package_type, + Path(package_dir), + skip_consistency_check=skip_consistency_check, ) yield (config.package_id, package_dir) - except ValueError: # pragma: nocover - # can not determine package type, not a package? just skip - pass # pragma: nocover + except ValueError as e: # pragma: nocover + logger.error( # pragma: nocover + f"Error with package_dir={package_dir}: {e}" + ) diff --git a/aea/configurations/base.py b/aea/configurations/base.py index 051a8586be..edb6629e23 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -492,7 +492,7 @@ def check_fingerprint(self, directory: Path) -> None: if not directory.exists() or not directory.is_dir(): raise ValueError("Directory {} is not valid.".format(directory)) _compare_fingerprints( - self, directory, False, self.component_type.to_configuration_type() + self, directory, False, self.component_type.to_package_type() ) def check_aea_version(self): diff --git a/aea/configurations/data_types.py b/aea/configurations/data_types.py index 7ac75957f3..f6638a908d 100644 --- a/aea/configurations/data_types.py +++ b/aea/configurations/data_types.py @@ -163,7 +163,7 @@ class ComponentType(Enum): SKILL = SKILL CONTRACT = CONTRACT - def to_configuration_type(self) -> PackageType: + def to_package_type(self) -> PackageType: """Get package type for component type.""" return PackageType(self.value) @@ -565,7 +565,7 @@ def __init__(self, component_type: Union[ComponentType, str], public_id: PublicI :param public_id: the public id. """ component_type = ComponentType(component_type) - super().__init__(component_type.to_configuration_type(), public_id) + super().__init__(component_type.to_package_type(), public_id) @property def component_type(self) -> ComponentType: diff --git a/aea/configurations/loader.py b/aea/configurations/loader.py index 521160facf..e7468368b5 100644 --- a/aea/configurations/loader.py +++ b/aea/configurations/loader.py @@ -308,22 +308,41 @@ def from_package_type( def load_component_configuration( - component_type: Union[ComponentType, PackageType], + component_type: ComponentType, directory: Path, skip_consistency_check: bool = False, -) -> "ComponentConfiguration": +) -> ComponentConfiguration: """ Load configuration and check that it is consistent against the directory. - :param component_type: the component or package type. + :param component_type: the component type. :param directory: the root of the package :param skip_consistency_check: if True, the consistency check are skipped. :return: the configuration object. """ - if isinstance(component_type, ComponentType): - component_type = component_type.to_configuration_type() - configuration_object = _load_configuration_object(component_type, directory) - if not skip_consistency_check: + package_type = component_type.to_package_type() + configuration_object = load_package_configuration( + package_type, directory, skip_consistency_check + ) + configuration_object = cast(ComponentConfiguration, configuration_object) + return configuration_object + + +def load_package_configuration( + package_type: PackageType, directory: Path, skip_consistency_check: bool = False, +) -> PackageConfiguration: + """ + Load configuration and check that it is consistent against the directory. + + :param package_type: the package type. + :param directory: the root of the package + :param skip_consistency_check: if True, the consistency check are skipped. + :return: the configuration object. + """ + configuration_object = _load_configuration_object(package_type, directory) + if not skip_consistency_check and isinstance( + configuration_object, ComponentConfiguration + ): configuration_object._check_configuration_consistency( # pylint: disable=protected-access directory ) @@ -332,11 +351,11 @@ def load_component_configuration( def _load_configuration_object( package_type: PackageType, directory: Path -) -> ComponentConfiguration: +) -> PackageConfiguration: """ Load the configuration object, without consistency checks. - :param component_type: the component type. + :param package_type: the package type. :param directory: the directory of the configuration. :return: the configuration object. :raises FileNotFoundError: if the configuration file is not found. diff --git a/tests/test_cli/test_local_registry_update.py b/tests/test_cli/test_local_registry_update.py index 3b5a41b6ed..eb9896275f 100644 --- a/tests/test_cli/test_local_registry_update.py +++ b/tests/test_cli/test_local_registry_update.py @@ -60,13 +60,15 @@ def test_local_registry_update(): "aea.cli.local_registry_sync.replace_package" ) as replace_package_mock: result = runner.invoke( - cli, "local-registry-sync", catch_exceptions=False + cli, ["-s", "local-registry-sync"], catch_exceptions=False ) assert result.exit_code == 0, result.stdout assert replace_package_mock.call_count == len(PACKAGES) # do actual upgrade - result = runner.invoke(cli, "local-registry-sync", catch_exceptions=False) + result = runner.invoke( + cli, ["-s", "local-registry-sync"], catch_exceptions=False + ) assert result.exit_code == 0, result.stdout # check next update will do nothing @@ -74,7 +76,7 @@ def test_local_registry_update(): "aea.cli.local_registry_sync.replace_package" ) as replace_package_mock: result = runner.invoke( - cli, "local-registry-sync", catch_exceptions=False + cli, ["-s", "local-registry-sync"], catch_exceptions=False ) assert result.exit_code == 0, result.stdout assert replace_package_mock.call_count == 0 diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index 67cac4557e..0c53823c87 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -844,6 +844,7 @@ def test_agent_config_to_json_with_optional_configurations(): agent_config.default_connection = "author/name:0.1.0" agent_config.default_ledger = DEFAULT_LEDGER agent_config.json + assert agent_config.package_id == PackageId.from_uri_path("agent/author/name/0.1.0") def test_protocol_specification_attributes(): diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 92fa0dbe0c..bc7e7b6ec6 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -42,7 +42,7 @@ def test_stop_runtime(): assert e.reraise == test -def test_parse_exception(): +def test_parse_exception_i(): """Test parse exception.""" def exception_raise(): @@ -61,3 +61,27 @@ def exception_raise(): 'raise ValueError("expected")\n\nValueError: expected\n', ] assert all([string in out for string in expected]) + + +def test_parse_exception_ii(): + """Test parse exception.""" + + def exception_raise(): + """A function that raises an exception.""" + raise AEAEnforceError("expected") + + try: + exception_raise() + except Exception as e: + out = parse_exception(e) + + expected = [ + "Traceback (most recent call last):\n\n", + 'test_exceptions.py", line ', + "in test_parse_exception_ii\n", + "exception_raise()\n\n", + ", line", + "in exception_raise\n", + 'raise AEAEnforceError("expected")\n\naea.exceptions.AEAEnforceError: expected\n', + ] + assert all([string in out for string in expected]) From d45755600463db706d67c5e204ba6d1c9d85be18 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Sat, 2 Jan 2021 15:51:49 +0000 Subject: [PATCH 127/204] add ethereum support for public key recovery --- aea/crypto/base.py | 2 +- aea/crypto/cosmos.py | 4 +-- aea/crypto/ethereum.py | 18 +++++++++-- aea/helpers/acn/agent_record.py | 2 +- tests/conftest.py | 2 +- tests/test_crypto/test_cosmos.py | 15 +++++++++ tests/test_crypto/test_ethereum.py | 15 +++++++++ tests/test_crypto/test_helpers.py | 13 ++++++++ tests/test_helpers/test_acn/__init__.py | 20 ++++++++++++ .../test_agent_record.py} | 0 tests/test_helpers/test_acn/test_uri.py | 31 +++++++++++++++++++ 11 files changed, 114 insertions(+), 8 deletions(-) create mode 100644 tests/test_helpers/test_acn/__init__.py rename tests/test_helpers/{test_acn.py => test_acn/test_agent_record.py} (100%) create mode 100644 tests/test_helpers/test_acn/test_uri.py diff --git a/aea/crypto/base.py b/aea/crypto/base.py index 31935093e6..cd469499ac 100644 --- a/aea/crypto/base.py +++ b/aea/crypto/base.py @@ -205,7 +205,7 @@ def recover_message( @classmethod @abstractmethod - def recover_verifying_keys_from_message( + def recover_public_keys_from_message( cls, message: bytes, signature: str, is_deprecated_mode: bool = False ) -> Tuple[str, ...]: """ diff --git a/aea/crypto/cosmos.py b/aea/crypto/cosmos.py index e2926c7463..be94c33be1 100644 --- a/aea/crypto/cosmos.py +++ b/aea/crypto/cosmos.py @@ -150,14 +150,14 @@ def recover_message( :param is_deprecated_mode: if the deprecated signing was used :return: the recovered addresses """ - public_keys = cls.recover_verifying_keys_from_message(message, signature) + public_keys = cls.recover_public_keys_from_message(message, signature) addresses = [ cls.get_address_from_public_key(public_key) for public_key in public_keys ] return tuple(addresses) @classmethod - def recover_verifying_keys_from_message( + def recover_public_keys_from_message( cls, message: bytes, signature: str, is_deprecated_mode: bool = False ) -> Tuple[str, ...]: """ diff --git a/aea/crypto/ethereum.py b/aea/crypto/ethereum.py index f83344d587..fc20db3090 100644 --- a/aea/crypto/ethereum.py +++ b/aea/crypto/ethereum.py @@ -28,8 +28,9 @@ import requests from eth_account import Account +from eth_account._utils.signing import to_standard_signature_bytes from eth_account.datastructures import HexBytes, SignedTransaction -from eth_account.messages import encode_defunct +from eth_account.messages import _hash_eip191_message, encode_defunct from eth_keys import keys from web3 import HTTPProvider, Web3 from web3.datastructures import AttributeDict @@ -366,7 +367,7 @@ def recover_message( return (address,) @classmethod - def recover_verifying_keys_from_message( + def recover_public_keys_from_message( cls, message: bytes, signature: str, is_deprecated_mode: bool = False ) -> Tuple[str, ...]: """ @@ -377,7 +378,18 @@ def recover_verifying_keys_from_message( :param is_deprecated_mode: if the deprecated signing was used :return: the recovered public keys """ - raise NotImplementedError # pragma: no cover + if not is_deprecated_mode: + signable_message = encode_defunct(primitive=message) + message = _hash_eip191_message(signable_message) + hash_bytes = HexBytes(message) + # code taken from https://github.com/ethereum/eth-account/blob/master/eth_account/account.py#L428 + if len(hash_bytes) != 32: # pragma: nocover + raise ValueError("The message hash must be exactly 32-bytes") + signature_bytes = HexBytes(signature) + signature_bytes_standard = to_standard_signature_bytes(signature_bytes) + signature_obj = keys.Signature(signature_bytes=signature_bytes_standard) + pubkey = signature_obj.recover_public_key_from_msg_hash(hash_bytes) + return (str(pubkey),) @staticmethod def get_hash(message: bytes) -> str: diff --git a/aea/helpers/acn/agent_record.py b/aea/helpers/acn/agent_record.py index eaf7e74157..8dec989319 100644 --- a/aea/helpers/acn/agent_record.py +++ b/aea/helpers/acn/agent_record.py @@ -43,7 +43,7 @@ def signature_from_cert_request( signature = bytes.fromhex(Path(cert.save_path).read_bytes().decode("ascii")).decode( "ascii" ) - public_keys = FetchAIHelper.recover_verifying_keys_from_message( + public_keys = FetchAIHelper.recover_public_keys_from_message( cert.get_message(message), signature ) if len(public_keys) == 0: # pragma: no cover diff --git a/tests/conftest.py b/tests/conftest.py index e6b8df3642..cce9b4af67 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -633,7 +633,7 @@ def update_default_ethereum_ledger_api(ethereum_testnet_config): @pytest.mark.integration @pytest.mark.ledger @pytest.fixture(scope="session") -# @action_for_platform("Linux", skip=False) +@action_for_platform("Linux", skip=False) def ganache( ganache_configuration, ganache_addr, diff --git a/tests/test_crypto/test_cosmos.py b/tests/test_crypto/test_cosmos.py index 6b66e9f18d..a1f0d3ecbb 100644 --- a/tests/test_crypto/test_cosmos.py +++ b/tests/test_crypto/test_cosmos.py @@ -61,6 +61,21 @@ def test_sign_and_recover_message(): ), "Failed to recover the correct address." +def test_sign_and_recover_message_public_key(): + """Test the signing and the recovery function for the eth_crypto.""" + account = CosmosCrypto(COSMOS_PRIVATE_KEY_PATH) + sign_bytes = account.sign_message(message=b"hello") + assert len(sign_bytes) > 0, "The len(signature) must not be 0" + recovered_public_keys = CosmosApi.recover_public_keys_from_message( + message=b"hello", signature=sign_bytes + ) + assert len(recovered_public_keys) == 2, "Wrong number of public keys recovered." + assert ( + CosmosApi.get_address_from_public_key(recovered_public_keys[0]) + == account.address + ), "Failed to recover the correct address." + + def test_get_hash(): """Test the get hash functionality.""" expected_hash = "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" diff --git a/tests/test_crypto/test_ethereum.py b/tests/test_crypto/test_ethereum.py index b301710ac1..204de633cf 100644 --- a/tests/test_crypto/test_ethereum.py +++ b/tests/test_crypto/test_ethereum.py @@ -113,6 +113,21 @@ def test_sign_and_recover_message_deprecated(): ), "Failed to recover the correct address." +def test_sign_and_recover_message_public_key(): + """Test the signing and the recovery function for the eth_crypto.""" + account = EthereumCrypto(ETHEREUM_PRIVATE_KEY_PATH) + sign_bytes = account.sign_message(message=b"hello") + assert len(sign_bytes) > 0, "The len(signature) must not be 0" + recovered_public_keys = EthereumApi.recover_public_keys_from_message( + message=b"hello", signature=sign_bytes + ) + assert len(recovered_public_keys) == 1, "Wrong number of public keys recovered." + assert ( + EthereumApi.get_address_from_public_key(recovered_public_keys[0]) + == account.address + ), "Failed to recover the correct address." + + def test_get_hash(): """Test the get hash functionality.""" expected_hash = "0x1c8aff950685c2ed4bc3174f3472287b56d9517b9c948127319a09a7a36deac8" diff --git a/tests/test_crypto/test_helpers.py b/tests/test_crypto/test_helpers.py index 70cb90f931..6bdd4642ef 100644 --- a/tests/test_crypto/test_helpers.py +++ b/tests/test_crypto/test_helpers.py @@ -20,6 +20,7 @@ import logging import os from pathlib import Path +from tempfile import TemporaryDirectory from unittest.mock import mock_open, patch import pytest @@ -30,6 +31,7 @@ from aea.crypto.fetchai import FetchAICrypto from aea.crypto.helpers import ( create_private_key, + make_certificate, private_key_verify_or_create, try_generate_testnet_wealth, try_validate_private_key_path, @@ -168,3 +170,14 @@ def test_private_key_verify_or_create(): private_key_verify_or_create(agent_conf, Path(".")) mock_validate.assert_not_called() mock_create.assert_not_called() + + +def test_make_certificate(): + """Test make_certificate.""" + with TemporaryDirectory() as tmp_dir: + make_certificate( + "fetchai", + os.path.join(CUR_PATH, "data", "fetchai_private_key.txt"), + b"message", + os.path.join(tmp_dir, "test.txt"), + ) diff --git a/tests/test_helpers/test_acn/__init__.py b/tests/test_helpers/test_acn/__init__.py new file mode 100644 index 0000000000..6a47cd282c --- /dev/null +++ b/tests/test_helpers/test_acn/__init__.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the tests for the helper.acn module.""" diff --git a/tests/test_helpers/test_acn.py b/tests/test_helpers/test_acn/test_agent_record.py similarity index 100% rename from tests/test_helpers/test_acn.py rename to tests/test_helpers/test_acn/test_agent_record.py diff --git a/tests/test_helpers/test_acn/test_uri.py b/tests/test_helpers/test_acn/test_uri.py new file mode 100644 index 0000000000..8a6e8a2e5d --- /dev/null +++ b/tests/test_helpers/test_acn/test_uri.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +"""This module contains the tests for acn helper module.""" + +from aea.helpers.acn.uri import Uri + + +def test_uri(): + """Test URI class""" + Uri(uri="localhost:9000") + uri = Uri(host="localhost", port=9000) + assert str(uri) == "localhost:9000" + assert uri.host == "localhost" + assert uri.port == 9000 + Uri() From 7349db3d392b6d58f3b61bd127907d7f574efd60 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sat, 2 Jan 2021 17:30:23 +0100 Subject: [PATCH 128/204] add tests for 'docs/ledger-integration.md' --- docs/ledger-integration.md | 2 +- tests/test_docs/helper.py | 6 +-- tests/test_docs/test_data_models.py | 4 +- tests/test_docs/test_ledger_integration.py | 63 ++++++++++++++++++++++ 4 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 tests/test_docs/test_ledger_integration.py diff --git a/docs/ledger-integration.md b/docs/ledger-integration.md index fe2fcd15ce..83577d3539 100644 --- a/docs/ledger-integration.md +++ b/docs/ledger-integration.md @@ -58,7 +58,7 @@ my_ledger_api = make_ledger_api("my_ledger_id") from aea.crypto.registries import faucet_apis_registry, make_faucet_api, register_faucet_api # by default we can use the native faucet apis -CONFIG = {"network": "testnet"} +CONFIG = dict(poll_interval=1.0) fetchai_faucet_api = make_faucet_api("fetchai", **CONFIG) # we can check what faucet apis are registered diff --git a/tests/test_docs/helper.py b/tests/test_docs/helper.py index 6ff0dd720b..cb7a276aa4 100644 --- a/tests/test_docs/helper.py +++ b/tests/test_docs/helper.py @@ -129,10 +129,10 @@ def _python_selector(cls, block: Dict) -> bool: return block["type"] == "block_code" and block["info"].strip() == "python" @abstractmethod - def _assert(self, **locals_): + def _assert(self, *mocks, **locals_): """Do assertions after Python code execution.""" - def test_python_blocks(self): + def test_python_blocks(self, *mocks): """Run Python code block in sequence.""" python_blocks = list(filter(self._python_selector, self.blocks)) @@ -140,4 +140,4 @@ def test_python_blocks(self): for python_block in python_blocks: python_code = python_block["text"] exec(python_code, globals_, locals_) # nosec - self._assert(**locals_) + self._assert(*mocks, **locals_) diff --git a/tests/test_docs/test_data_models.py b/tests/test_docs/test_data_models.py index ec22b684b3..d999c9881d 100644 --- a/tests/test_docs/test_data_models.py +++ b/tests/test_docs/test_data_models.py @@ -27,11 +27,11 @@ class TestDataModel(BasePythonMarkdownDocs): - """Test the integrity of the language agnostic definitions in docs.""" + """Test the data models code snippets.""" DOC_PATH = Path(ROOT_DIR, "docs", "defining-data-models.md") - def _assert(self, **locals_): + def _assert(self, *_mocks, **locals_): attribute = locals_["attr_title"] assert isinstance(attribute, Attribute) diff --git a/tests/test_docs/test_ledger_integration.py b/tests/test_docs/test_ledger_integration.py new file mode 100644 index 0000000000..285c25691b --- /dev/null +++ b/tests/test_docs/test_ledger_integration.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the tests for the content of ledger-integration.md file.""" +from importlib import import_module +from pathlib import Path +from unittest import mock +from unittest.mock import MagicMock + +from aea.crypto import FetchAICrypto +from aea.crypto.fetchai import FetchAIFaucetApi + +from tests.conftest import ROOT_DIR +from tests.test_docs.helper import BasePythonMarkdownDocs + + +def _import_module_mock(arg): + """ + Mock importlib.import_module only if argument is a dummy one: 'some.dotted.path'. + + This choice is tight to the code examples in 'ledger-integration.md'. + It helps to tests the cases in which the import path is not a fake one. + """ + if arg.startswith("some.dotted.path"): + return MagicMock() + return import_module(arg) + + +# we mock only if the import is from dumyy import path like "some.dotted.path". +@mock.patch("importlib.import_module", side_effect=_import_module_mock) +class TestLedgerIntegration(BasePythonMarkdownDocs): + """Test the ledger integration code snippets.""" + + DOC_PATH = Path(ROOT_DIR, "docs", "ledger-integration.md") + + def _assert_isinstance(self, locals_key, cls, **locals_): + """Assert that the member of 'locals' is an instance of a class.""" + assert locals_key in locals_ + obj = locals_[locals_key] + assert isinstance(obj, cls) + + def _assert(self, *mocks, **locals_): + """Assert code outputs.""" + self._assert_isinstance("fetchai_crypto", FetchAICrypto, **locals_) + self._assert_isinstance("fetchai_faucet_api", FetchAIFaucetApi, **locals_) + self._assert_isinstance("my_ledger_api", MagicMock, **locals_) + self._assert_isinstance("my_faucet_api", MagicMock, **locals_) From bee8b28f171b7971a100b0ad8810f1635f89d70d Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sat, 2 Jan 2021 17:56:58 +0100 Subject: [PATCH 129/204] add tests for 'docs/multi-agent-manager.md' --- aea/configurations/base.py | 1 + docs/multi-agent-manager.md | 20 +++++-- tests/conftest.py | 1 + tests/test_docs/helper.py | 1 - tests/test_docs/test_ledger_integration.py | 2 +- tests/test_docs/test_multiagent_manager.py | 69 ++++++++++++++++++++++ 6 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 tests/test_docs/test_multiagent_manager.py diff --git a/aea/configurations/base.py b/aea/configurations/base.py index eca468f924..bf7958d693 100644 --- a/aea/configurations/base.py +++ b/aea/configurations/base.py @@ -1069,6 +1069,7 @@ class AgentConfig(PackageConfiguration): ) CHECK_EXCLUDES = [ ("private_key_paths",), + ("connection_private_key_paths",), ("default_routing",), ("logging_config",), ] diff --git a/docs/multi-agent-manager.md b/docs/multi-agent-manager.md index 16a879c6af..22cf6c0895 100644 --- a/docs/multi-agent-manager.md +++ b/docs/multi-agent-manager.md @@ -6,6 +6,7 @@ The `MultiAgentManager` allows managing multiple ag We intantiate the manager by providing it with the working directory in which to operate and starting it: ``` python +from pathlib import Path from aea.manager import MultiAgentManager WORKING_DIR = "." @@ -31,14 +32,21 @@ manager.add_project(weather_station_id) Save the following private keys in the respective files. ``` python -# 72d3149f5689f0749eaec5ebf6dba5deeb1e89b93ae1c58c71fd43dfaa231e87 +FET_PRIVATE_KEY_1 = b"72d3149f5689f0749eaec5ebf6dba5deeb1e89b93ae1c58c71fd43dfaa231e87" FET_PRIVATE_KEY_PATH_1 = "fetchai_private_key_1.txt" -# bf529acb2546e13615ef6004c48e393f0638a5dc0c4979631a9a4bc554079f6f +Path(FET_PRIVATE_KEY_PATH_1).write_bytes(FET_PRIVATE_KEY_1) + +COSMOS_PRIVATE_KEY_1 = b"bf529acb2546e13615ef6004c48e393f0638a5dc0c4979631a9a4bc554079f6f" COSMOS_PRIVATE_KEY_PATH_1 = "cosmos_private_key_1.txt" -# 589839ae54b71b8754a7fe96b52045364077c28705a1806b74441debcae16e0a +Path(COSMOS_PRIVATE_KEY_PATH_1).write_bytes(COSMOS_PRIVATE_KEY_1) + +FET_PRIVATE_KEY_2 = b"589839ae54b71b8754a7fe96b52045364077c28705a1806b74441debcae16e0a" FET_PRIVATE_KEY_PATH_2 = "fetchai_private_key_2.txt" -# c9b38eff57f678f5ab5304447997351edb08eceb883267fa4ad849074bec07e4 +Path(FET_PRIVATE_KEY_PATH_2).write_bytes(FET_PRIVATE_KEY_2) + +COSMOS_PRIVATE_KEY_2 = b"c9b38eff57f678f5ab5304447997351edb08eceb883267fa4ad849074bec07e4" COSMOS_PRIVATE_KEY_PATH_2 = "cosmos_private_key_2.txt" +Path(COSMOS_PRIVATE_KEY_PATH_2).write_bytes(COSMOS_PRIVATE_KEY_2) ``` Add the agent instances @@ -52,7 +60,7 @@ manager.add_agent(weather_station_id, agent_overrides=agent_overrides) component_overrides = { "name": "p2p_libp2p", "author": "fetchai", - "version": "0.9.0", + "version": "0.13.0", "type": "connection", "config": { "delegate_uri": "127.0.0.1:11001", @@ -73,7 +81,7 @@ manager.add_agent(weather_client_id, component_overrides=[component_overrides], ``` python manager.start_agent(weather_station_id.name) # wait for ~10 seconds for peer node to go live -manager.start_agent(weather_station_id.name) +manager.start_agent(weather_client_id.name) ``` ## Stopping the agents: diff --git a/tests/conftest.py b/tests/conftest.py index 63b6afc1bd..985f732cdc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -227,6 +227,7 @@ MAX_FLAKY_RERUNS_ETH = 1 MAX_FLAKY_RERUNS_INTEGRATION = 1 +PACKAGES_DIR = os.path.join(ROOT_DIR, "packages") FETCHAI_PREF = os.path.join(ROOT_DIR, "packages", "fetchai") PROTOCOL_SPECS_PREF_1 = os.path.join(ROOT_DIR, "examples", "protocol_specification_ex") PROTOCOL_SPECS_PREF_2 = os.path.join(ROOT_DIR, "tests", "data") diff --git a/tests/test_docs/helper.py b/tests/test_docs/helper.py index cb7a276aa4..e0f890a680 100644 --- a/tests/test_docs/helper.py +++ b/tests/test_docs/helper.py @@ -128,7 +128,6 @@ class BasePythonMarkdownDocs(BaseTestMarkdownDocs): def _python_selector(cls, block: Dict) -> bool: return block["type"] == "block_code" and block["info"].strip() == "python" - @abstractmethod def _assert(self, *mocks, **locals_): """Do assertions after Python code execution.""" diff --git a/tests/test_docs/test_ledger_integration.py b/tests/test_docs/test_ledger_integration.py index 285c25691b..a236c36ec8 100644 --- a/tests/test_docs/test_ledger_integration.py +++ b/tests/test_docs/test_ledger_integration.py @@ -42,7 +42,7 @@ def _import_module_mock(arg): return import_module(arg) -# we mock only if the import is from dumyy import path like "some.dotted.path". +# we mock only if the import is from dummy import path like "some.dotted.path". @mock.patch("importlib.import_module", side_effect=_import_module_mock) class TestLedgerIntegration(BasePythonMarkdownDocs): """Test the ledger integration code snippets.""" diff --git a/tests/test_docs/test_multiagent_manager.py b/tests/test_docs/test_multiagent_manager.py new file mode 100644 index 0000000000..026cc916b6 --- /dev/null +++ b/tests/test_docs/test_multiagent_manager.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the tests for the content of multi-agent-manager.md file.""" +import os +import shutil +import tempfile +from importlib import import_module +from pathlib import Path +from unittest import mock +from unittest.mock import MagicMock + +from aea.crypto import FetchAICrypto +from aea.crypto.fetchai import FetchAIFaucetApi + +from tests.conftest import ROOT_DIR, PACKAGES_DIR +from tests.test_docs.helper import BasePythonMarkdownDocs + + +def _import_module_mock(arg): + """ + Mock importlib.import_module only if argument is a dummy one: 'some.dotted.path'. + + This choice is tight to the code examples in 'ledger-integration.md'. + It helps to tests the cases in which the import path is not a fake one. + """ + if arg.startswith("some.dotted.path"): + return MagicMock() + return import_module(arg) + + +class TestMultiAgentManager(BasePythonMarkdownDocs): + """Test the ledger integration code snippets.""" + + DOC_PATH = Path(ROOT_DIR, "docs", "multi-agent-manager.md") + + @classmethod + def setup_class(cls): + """Set up the class.""" + super().setup_class() + cls.old_cwd = os.getcwd() + cls.temp_dir = tempfile.mkdtemp() + shutil.copytree(PACKAGES_DIR, Path(cls.temp_dir, "packages")) + os.chdir(cls.temp_dir) + + def _assert(self, *mocks, **locals_): + """Assert code outputs.""" + + @classmethod + def teardown_class(cls): + """Teardown class.""" + os.chdir(cls.old_cwd) + shutil.rmtree(cls.temp_dir) From 06807eda101b38679b6763ec430d26ae165e3ce3 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sat, 2 Jan 2021 17:59:20 +0100 Subject: [PATCH 130/204] add tests for 'docs/oracle-demo.md' --- .../md_files/bash-oracle-demo.md | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/test_docs/test_bash_yaml/md_files/bash-oracle-demo.md diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-oracle-demo.md b/tests/test_docs/test_bash_yaml/md_files/bash-oracle-demo.md new file mode 100644 index 0000000000..e0722b8d4d --- /dev/null +++ b/tests/test_docs/test_bash_yaml/md_files/bash-oracle-demo.md @@ -0,0 +1,45 @@ +``` bash +aea fetch fetchai/coin_price_oracle:0.2.0 +cd coin_price_oracle +aea install +aea build +``` +``` bash +aea create coin_price_oracle +cd coin_price_oracle +aea add connection fetchai/http_client:0.15.0 +aea add connection fetchai/ledger:0.11.0 +aea add connection fetchai/p2p_libp2p:0.13.0 +aea add skill fetchai/coin_price:0.2.0 +aea add skill fetchai/simple_oracle:0.2.0 +aea install +aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +``` +``` bash +aea config set --type dict agent.default_routing \ +'{ +"fetchai/contract_api:0.9.0": "fetchai/ledger:0.11.0", +"fetchai/http:0.10.0": "fetchai/http_client:0.15.0", +"fetchai/ledger_api:0.8.0": "fetchai/ledger:0.11.0" +}' +``` +``` bash +aea config set agent.default_ledger ethereum +``` +``` bash +aea generate-key ethereum +aea add-key ethereum ethereum_private_key.txt +``` +``` bash +docker run -p 8545:8545 trufflesuite/ganache-cli:latest --verbose --gasPrice=0 --gasLimit=0x1fffffffffffff --account="$(cat coin_price_oracle/ethereum_private_key.txt),1000000000000000000000" +``` +``` bash +aea run +``` +``` bash +info: [coin_price_oracle] Oracle contract successfully deployed! +... +info: [coin_price_oracle] Oracle role successfully granted! +... +info: [coin_price_oracle] Oracle value successfully updated! +``` \ No newline at end of file From 6695c9128ae239eeca142267880d98f67f707769 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sat, 2 Jan 2021 18:12:50 +0100 Subject: [PATCH 131/204] add tests for 'docs/query-language.md' --- docs/query-language.md | 2 +- tests/test_docs/helper.py | 1 - tests/test_docs/test_multiagent_manager.py | 6 +---- tests/test_docs/test_query_language.py | 30 ++++++++++++++++++++++ 4 files changed, 32 insertions(+), 7 deletions(-) create mode 100644 tests/test_docs/test_query_language.py diff --git a/docs/query-language.md b/docs/query-language.md index 7b3fa0be23..fb6b199248 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -120,7 +120,7 @@ The _distance_ is a constraint type that allows you to put a limit on a Date: Sat, 2 Jan 2021 19:13:15 +0100 Subject: [PATCH 132/204] fix 'ledger-integration' and 'query-language' tests. --- tests/test_docs/helper.py | 13 ++++++++++++- tests/test_docs/test_ledger_integration.py | 16 +++++++++++++++- tests/test_docs/test_query_language.py | 9 +++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/tests/test_docs/helper.py b/tests/test_docs/helper.py index a4bdb49058..0f6d237dc0 100644 --- a/tests/test_docs/helper.py +++ b/tests/test_docs/helper.py @@ -123,6 +123,17 @@ def setup_class(cls): class BasePythonMarkdownDocs(BaseTestMarkdownDocs): """Test Markdown documentation by running Python snippets in sequence.""" + @classmethod + def setup_class(cls): + """ + Set up class. + + It sets the initial value of locals and globals. + """ + super().setup_class() + cls.locals = {} + cls.globals = {} + @classmethod def _python_selector(cls, block: Dict) -> bool: return block["type"] == "block_code" and block["info"].strip() == "python" @@ -134,7 +145,7 @@ def test_python_blocks(self, *mocks): """Run Python code block in sequence.""" python_blocks = list(filter(self._python_selector, self.blocks)) - globals_, locals_ = {}, {} + globals_, locals_ = self.globals, self.locals for python_block in python_blocks: python_code = python_block["text"] exec(python_code, globals_, locals_) # nosec diff --git a/tests/test_docs/test_ledger_integration.py b/tests/test_docs/test_ledger_integration.py index a236c36ec8..39811a3618 100644 --- a/tests/test_docs/test_ledger_integration.py +++ b/tests/test_docs/test_ledger_integration.py @@ -24,7 +24,12 @@ from unittest.mock import MagicMock from aea.crypto import FetchAICrypto -from aea.crypto.fetchai import FetchAIFaucetApi +from aea.crypto.fetchai import FetchAIApi, FetchAIFaucetApi +from aea.crypto.registries import ( + crypto_registry, + faucet_apis_registry, + ledger_apis_registry, +) from tests.conftest import ROOT_DIR from tests.test_docs.helper import BasePythonMarkdownDocs @@ -58,6 +63,15 @@ def _assert_isinstance(self, locals_key, cls, **locals_): def _assert(self, *mocks, **locals_): """Assert code outputs.""" self._assert_isinstance("fetchai_crypto", FetchAICrypto, **locals_) + self._assert_isinstance("fetchai_ledger_api", FetchAIApi, **locals_) self._assert_isinstance("fetchai_faucet_api", FetchAIFaucetApi, **locals_) + self._assert_isinstance("my_ledger_crypto", MagicMock, **locals_) self._assert_isinstance("my_ledger_api", MagicMock, **locals_) self._assert_isinstance("my_faucet_api", MagicMock, **locals_) + + @classmethod + def teardown_class(cls): + """Tear down the test.""" + crypto_registry.specs.pop("my_ledger_id") + ledger_apis_registry.specs.pop("my_ledger_id") + faucet_apis_registry.specs.pop("my_ledger_id") diff --git a/tests/test_docs/test_query_language.py b/tests/test_docs/test_query_language.py index 31a521a07b..1e70afdc5f 100644 --- a/tests/test_docs/test_query_language.py +++ b/tests/test_docs/test_query_language.py @@ -19,12 +19,21 @@ """This module contains the tests for the content of query-language.md file.""" from pathlib import Path +from unittest import mock +from unittest.mock import MagicMock from tests.conftest import ROOT_DIR from tests.test_docs.helper import BasePythonMarkdownDocs +@mock.patch("aea.helpers.search.models.Query.check_validity") class TestQueryLanguage(BasePythonMarkdownDocs): """Test the data models code snippets.""" DOC_PATH = Path(ROOT_DIR, "docs", "query-language.md") + + @classmethod + def setup_class(cls): + """Set up the test.""" + super().setup_class() + cls.locals["book_model"] = MagicMock() From 2841873c10401cc152e4abcc5ad7e2cc1c3db208 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Sat, 2 Jan 2021 19:13:45 +0000 Subject: [PATCH 133/204] refactor agent record --- aea/helpers/acn/agent_record.py | 169 +++++++++--------- aea/helpers/base.py | 12 ++ .../connections/p2p_libp2p/connection.py | 34 +--- .../connections/p2p_libp2p/connection.yaml | 2 +- .../p2p_libp2p_client/connection.py | 50 +----- .../p2p_libp2p_client/connection.yaml | 2 +- packages/hashes.csv | 4 +- scripts/missing: | 20 +++ scripts/whitelist.py | 1 + .../test_acn/test_agent_record.py | 115 +++++------- 10 files changed, 181 insertions(+), 228 deletions(-) create mode 100644 scripts/missing: diff --git a/aea/helpers/acn/agent_record.py b/aea/helpers/acn/agent_record.py index 8dec989319..74eaab45fa 100644 --- a/aea/helpers/acn/agent_record.py +++ b/aea/helpers/acn/agent_record.py @@ -19,71 +19,74 @@ """This module contains types and helpers for acn Proof-of-Representation.""" -from pathlib import Path -from typing import Tuple +from typing import Optional -from aea.crypto.fetchai import FetchAIHelper +from aea.crypto.registries import make_ledger_api from aea.helpers.base import CertRequest -def signature_from_cert_request( - cert: CertRequest, message: str, signer_address: str -) -> Tuple[str, str]: - """ - Get signature and its verifying key from a CertRequest and its message. - - Must match aea/cli/issue_certificates.py:_process_certificate - - :param cert: cert request containing the signature - :param message: the message used to generate signature - :param signer_address: the address of the signer - :return: the signature and the verifying public key - """ - - signature = bytes.fromhex(Path(cert.save_path).read_bytes().decode("ascii")).decode( - "ascii" - ) - public_keys = FetchAIHelper.recover_public_keys_from_message( - cert.get_message(message), signature - ) - if len(public_keys) == 0: # pragma: no cover - raise Exception("Malformed signature") - addresses = [ - FetchAIHelper.get_address_from_public_key(public_key) - for public_key in public_keys - ] - try: - verify_key = public_keys[addresses.index(signer_address)] - except ValueError: - raise Exception("Not signed by agent") - return signature, verify_key - - class AgentRecord: - """Agent Proof-of-Representation to peer""" + """Agent Proof-of-Representation to representative.""" + + VALID_LEDGER_IDS = ["fetchai", "cosmos", "ethereum"] def __init__( self, address: str, - public_key: str, - peer_public_key: str, + representative_public_key: str, + message: bytes, signature: str, - service_id: str, + ledger_id: str, ): """ Initialize the AgentRecord :param address: agent address - :param public key: agent public key (associated to the address) - :param peer_public_key: representative peer public key + :param representative_public_key: representative's public key + :param message: message to be signed as proof-of-represenation of this AgentRecord :param signature: proof-of-representation of this AgentRecord - :param service_id: type of service for which the record is used + :param ledger_id: ledger id """ - self._service_id = service_id self._address = address - self._public_key = public_key - self._peer_public_key = peer_public_key + self._representative_public_key = representative_public_key + self._message = message self._signature = signature + self._ledger_id = ledger_id + self._public_key: Optional[str] = None + self._check_validity() + + def _check_validity(self) -> None: + """ + Checks validity of record. + + Specificyally: + - if ledger_id is valid + - if agent signed the message + - if message is correctly formatted + """ + if self.ledger_id not in self.VALID_LEDGER_IDS: + raise ValueError( + f"Not a valid ledger_id. Found={self.ledger_id}, valid ids={self.VALID_LEDGER_IDS}." + ) + if self.message != self._get_message(self.representative_public_key): + raise ValueError("Invalid message.") # pragma: no cover + ledger_api = make_ledger_api(self.ledger_id) + public_keys = ledger_api.recover_public_keys_from_message( + self.message, self.signature + ) + if len(public_keys) == 0: + raise ValueError("Malformed signature!") # pragma: no cover + public_key: Optional[str] = None + for public_key_ in public_keys: + address = ledger_api.get_address_from_public_key(public_key_) + if address == self.address: + public_key = public_key_ + break + if public_key is None: + raise ValueError( + "Invalid signature for provided representative_public_key and agent address!" + ) + self._public_key = public_key @property def address(self) -> str: @@ -93,12 +96,14 @@ def address(self) -> str: @property def public_key(self) -> str: """Get agent public key""" + if self._public_key is None: + raise ValueError("Inconsistent record!") #  pragma: nocover return self._public_key @property - def peer_public_key(self) -> str: - """Get agent's representative peer public key""" - return self._peer_public_key + def representative_public_key(self) -> str: + """Get agent representative's public key""" + return self._representative_public_key @property def signature(self) -> str: @@ -106,38 +111,40 @@ def signature(self) -> str: return self._signature @property - def service_id(self) -> str: - """Get record service id""" - return self._service_id + def message(self) -> bytes: + """Get the message.""" + return self._message + + @property + def ledger_id(self) -> str: + """Get ledger id.""" + return self._ledger_id + + def _get_message(self, public_key: str) -> bytes: # pylint: disable=no-self-use + """Get the message.""" + # Refactor, needs to match CertRequest! + message = public_key.encode("ascii") + # + self.identifier.encode("ascii") # noqa: E800 + # + self.not_before_string.encode("ascii") # noqa: E800 + # + self.not_after_string.encode("ascii") # noqa: E800 + return message def __str__(self): # pragma: no cover """Get string representation.""" - return f"(address={self.address}, public_key={self.public_key}, peer_public_key={self.peer_public_key}, signature={self.signature})" - - def check_validity(self, address: str, peer_public_key: str) -> None: - """ - Check if the agent record is valid for `address` and `peer_public_key`. - - Raises an Exception if invalid. - - :param address: the expected agent address concerned by the record - :param peer_public_key: the expected representative peer public key - """ - - if self._address != address: - raise Exception( - "Proof-of-representation is not generated for the intended agent" - ) - if self._peer_public_key != peer_public_key: - raise Exception( - "Proof-of-representation is not generated for intended peer" - ) - recovered_address = FetchAIHelper.get_address_from_public_key(self._public_key) - if self._address != recovered_address: - raise Exception( - f"Agent address {self._address} and public key doesn't match" - ) - if self._address not in FetchAIHelper.recover_message( - self._peer_public_key.encode("utf-8"), self._signature - ): - raise Exception("Invalid signature") + return f"(address={self.address}, public_key={self.public_key}, peer_public_key={self.peer_public_key}, signature={self.signature}, ledger_id={self.ledger_id})" + + @classmethod + def from_cert_request( + cls, cert_request: CertRequest, address: str, representative_public_key: str, + ) -> "AgentRecord": + """Get agent record from cert request.""" + message = cert_request.get_message(representative_public_key) + signature = cert_request.get_signature() + record = cls( + address, + representative_public_key, + message, + signature, + cert_request.ledger_id, + ) + return record diff --git a/aea/helpers/base.py b/aea/helpers/base.py index 965c89dfe8..a1677b1ff4 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -754,6 +754,18 @@ def get_message(self, public_key: str) -> bytes: # pylint: disable=no-self-use # + self.not_after_string.encode("ascii") # noqa: E800 return message + def get_signature(self) -> str: + """Get signature from save_path.""" + if not Path(self.save_path).is_file(): + raise Exception( # pragma: no cover + f"cert_request 'save_path' field {self.save_path} is not a file. " + "Please ensure that 'issue-certificates' command is called beforehand." + ) + signature = bytes.fromhex( + Path(self.save_path).read_bytes().decode("ascii") + ).decode("ascii") + return signature + @property def json(self) -> Dict: """Compute the JSON representation.""" diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index de4dc8b7b9..3aff86126a 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -35,7 +35,7 @@ from aea.connections.base import Connection, ConnectionStates from aea.crypto.base import Crypto from aea.exceptions import enforce -from aea.helpers.acn.agent_record import AgentRecord, signature_from_cert_request +from aea.helpers.acn.agent_record import AgentRecord from aea.helpers.acn.uri import Uri from aea.helpers.multiaddr.base import MultiAddr from aea.helpers.pipe import IPCChannel, make_ipc_channel @@ -264,10 +264,10 @@ async def start(self) -> None: self._config += "AEA_P2P_POR_ADDRESS={}\n".format(self.record.address) self._config += "AEA_P2P_POR_PUBKEY={}\n".format(self.record.public_key) self._config += "AEA_P2P_POR_PEER_PUBKEY={}\n".format( - self.record.peer_public_key + self.record.representative_public_key ) self._config += "AEA_P2P_POR_SIGNATURE={}\n".format(self.record.signature) - self._config += "AEA_P2P_POR_SERVICE_ID={}\n".format(self.record.service_id) + self._config += "AEA_P2P_POR_SERVICE_ID={}\n".format(POR_DEFAULT_SERVICE_ID) self._config += "AEA_P2P_CFG_REGISTRATION_DELAY={}\n".format( str(self.peer_registration_delay) if self.peer_registration_delay is not None @@ -537,31 +537,11 @@ def __init__(self, **kwargs): raise ValueError( # pragma: no cover "cert_requests field must be set and contain exactly one entry!" ) - if not Path(cert_requests[0].save_path).is_file(): - raise Exception( # pragma: no cover - f"cert_request 'save_path' field {cert_requests[0].save_path} is not a file. " - "Please ensure that 'issue-certificates' command is called beforehand" - ) - try: - signature, agent_public_key = signature_from_cert_request( - cert_requests[0], key.public_key, self.address - ) - except Exception as e: # pragma: no cover - raise ValueError( - f"Incorrect certificate from file {cert_requests[0].save_path} : {str(e)}" - ) + cert_request = cert_requests[0] - record = AgentRecord( - self.address, - agent_public_key, - key.public_key, - signature, - POR_DEFAULT_SERVICE_ID, + agent_record = AgentRecord.from_cert_request( + cert_request, self.address, key.public_key ) - try: - record.check_validity(self.address, key.public_key) - except Exception as e: # pragma: no cover - raise ValueError("Invalid Proof-of-Representation: {}".format(str(e))) # libp2p local node self.logger.debug("Public key used by libp2p node: {}".format(key.public_key)) @@ -570,7 +550,7 @@ def __init__(self, **kwargs): self._check_node_built() self.node = Libp2pNode( - record, + agent_record, key, self.configuration.build_directory, LIBP2P_NODE_CLARGS, diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index 092327a3db..39f253baa0 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -16,7 +16,7 @@ fingerprint: aea/pipe_unix.go: QmSzbAexrSUSA9ZE6VT6MmcWF5MhEcmceQjAbnfRoK7Ruv aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK check_dependencies.py: QmcZATMqEThMQBdsud1AwJEYtKq86VfPSXhnfdE7TSWcsc - connection.py: QmToAmEktB6M4iU8jHB648vtdNru9gGkMFTNH2Fy9rLFJo + connection.py: QmdpGLSTnxytiQ7P3Aiiipp4zsVkrTeL9B2GdQUsBh6XQy dht/dhtclient/dhtclient.go: QmZCgMowWw4ueCSsyYY12om3RMQA3Ydp98ti65Eqa5ccuz dht/dhtclient/dhtclient_test.go: QmbTvSmNU1uqjLQo9LdD7zWmDcLuzEacDeg7AG4VXhmriV dht/dhtclient/options.go: QmW3UjASUP6zGwK8b61hHQhVoLqBxQ13vZ1gcHuTmdrsDf diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.py b/packages/fetchai/connections/p2p_libp2p_client/connection.py index 476c863a53..709b531d36 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.py +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.py @@ -32,7 +32,7 @@ from aea.connections.base import Connection, ConnectionStates from aea.crypto.registries import make_crypto from aea.exceptions import enforce -from aea.helpers.acn.agent_record import AgentRecord, signature_from_cert_request +from aea.helpers.acn.agent_record import AgentRecord from aea.helpers.acn.uri import Uri from aea.mail.base import Envelope @@ -127,48 +127,12 @@ def __init__(self, **kwargs): self.delegate_uris = [Uri(node_uri) for node_uri in nodes_uris] # delegates PoRs - self.delegate_pors = [] - agent_public_key = "" - signatures: List[str] = [] - for i, cert in enumerate(cert_requests): - try: - signature, agent_public_key = signature_from_cert_request( - cert, nodes_public_keys[i], self.address - ) - signatures.append(signature) - except Exception as e: # pragma: nocover - raise ValueError( - f"Incorrect certificate from file {cert.save_path} " - f"for node {nodes_uris[i]} : {str(e)}" - ) - - records: List[AgentRecord] = [] - for i, signature in enumerate(signatures): - records.append( - AgentRecord( - self.address, - agent_public_key, - nodes_public_keys[i], - signature, - POR_DEFAULT_SERVICE_ID, - ) + self.delegate_pors: List[AgentRecord] = [] + for i, cert_request in enumerate(cert_requests): + agent_record = AgentRecord.from_cert_request( + cert_request, self.address, nodes_public_keys[i] ) - - for i, public_key in enumerate(nodes_public_keys): - uri = self.delegate_uris[i] - record = next( - record for record in records if record.peer_public_key == public_key - ) - try: - record.check_validity(self.address, public_key) - except Exception as e: # pragma: nocover - raise ValueError( - "Invalid Proof-of-Representation for node {}: {}".format( - str(uri), str(e) - ) - ) - - self.delegate_pors.append(record) + self.delegate_pors.append(agent_record) # select a delegate index = random.randint(0, len(self.delegate_uris) - 1) # nosec @@ -227,7 +191,7 @@ async def _setup_connection(self): record.public_key = self.node_por.public_key record.peer_public_key = self.node_por.peer_public_key record.signature = self.node_por.signature - record.service_id = self.node_por.service_id # pylint: disable=no-member + record.service_id = POR_DEFAULT_SERVICE_ID registration = Register() registration.record.CopyFrom(record) # pylint: disable=no-member diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml index 4856ba79a7..c5e0475135 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml @@ -12,7 +12,7 @@ fingerprint: __init__.py: QmT1FEHkPGMHV5oiVEfQHHr25N2qdZxydSNRJabJvYiTgf acn_message.proto: QmSwpCP72J94kt2MaLSPQdt8puqtLMVPp3Migwp4xRwadZ acn_message_pb2.py: QmbVqUUCHyQpfx7dHvW5e5j3RGcjNYqsDjuPV8eNcca2Nf - connection.py: QmPsDJ1jJ1qamuCzopygzunvszYPHtgbgTp4HBet4FN1Ya + connection.py: QmahcgM3MHb2tLcphtXzoMFUCZjN2yamWagVyVSKDAvpbs fingerprint_ignore_patterns: [] connections: [] protocols: [] diff --git a/packages/hashes.csv b/packages/hashes.csv index 6673e87f4c..0557c411b9 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -34,8 +34,8 @@ fetchai/connections/http_server,QmQ8hNfEts6th9zfnocYHzyYw4dXkDNDj8zoKtov1wB49n fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 -fetchai/connections/p2p_libp2p,QmZF328Yjdr6drv44C6KzrWes66DHfBWMPuCNPoA7dioA8 -fetchai/connections/p2p_libp2p_client,QmREbD3SiKrMdmzYbSwFX52M8wRmpPWcRs9XFJu3Cdsz5L +fetchai/connections/p2p_libp2p,QmddLzTtGexWtuvZPLfe58UvChfXpNwXFyCJdE4goaZ6Zp +fetchai/connections/p2p_libp2p_client,QmeLwBkdC54VN9YAEG3U8VcZxLondzhLjC6Tb5XKhZhQMR fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T fetchai/connections/scaffold,QmaCupER4vedakPPLzn2Rw9YHPmqSPaZh5xnqi6xVxQHvn diff --git a/scripts/missing: b/scripts/missing: new file mode 100644 index 0000000000..7cffdd801f --- /dev/null +++ b/scripts/missing: @@ -0,0 +1,20 @@ +missing: +- docs on PoR +- docs on interaction protocols + +Mo: +Documentation week & launch AW-4 + +- Lokman focus on Ethereum support, deploy new public DHT nodes +- Marco > doc tests; https://github.com/fetchai/agents-aea/issues/2078 +- Yuri > https://github.com/fetchai/agents-aea/issues/1923 +- Ali > TAC: feedback from BAI, IJCAI + + +Big themes Jan: +- messaging internal & external (protocols) +- Yuri to finish config analysis +- Yuri to review and fix: https://docs.google.com/document/d/1hY5N_-TtvSD3Z961xPjsbtw2oOKlIBJucr-jTzbl_1E/edit +- Marco: crypto support in configs + +- updated benchmark for papers \ No newline at end of file diff --git a/scripts/whitelist.py b/scripts/whitelist.py index 6e2242e902..0adc620823 100644 --- a/scripts/whitelist.py +++ b/scripts/whitelist.py @@ -246,3 +246,4 @@ Uri # unused class (aea/helpers/acn/uri.py:26) not_before_string # unused property (aea/helpers/base.py:724) not_after_string # unused property (aea/helpers/base.py:729) +from_cert_request # unused method (aea/helpers/acn/agent_record.py:136) diff --git a/tests/test_helpers/test_acn/test_agent_record.py b/tests/test_helpers/test_acn/test_agent_record.py index 2deb0db026..2d13832832 100644 --- a/tests/test_helpers/test_acn/test_agent_record.py +++ b/tests/test_helpers/test_acn/test_agent_record.py @@ -22,52 +22,21 @@ from aea.configurations.constants import DEFAULT_LEDGER from aea.crypto.registries import make_crypto -from aea.helpers.acn.agent_record import AgentRecord, signature_from_cert_request +from aea.helpers.acn.agent_record import AgentRecord from aea.helpers.base import CertRequest from tests.conftest import _process_cert -def test_signature_from_cert_request_errors(): +def test_agent_record(): """Test signature and public key proper retrieval from a CertRequest""" agent_key_1 = make_crypto(DEFAULT_LEDGER) agent_key_2 = make_crypto(DEFAULT_LEDGER) - peer_public_key = make_crypto(DEFAULT_LEDGER).public_key - - cert_path = "test_acn_cert.txt" - - cert = CertRequest( - peer_public_key, - "test_service", - DEFAULT_LEDGER, - "2021-01-01", - "2022-01-01", - cert_path, - ) - _process_cert(agent_key_1, cert) - - # success - _, signer_public_key = signature_from_cert_request( - cert, peer_public_key, agent_key_1.address - ) - assert signer_public_key == agent_key_1.public_key - - # error: wrong signer - with pytest.raises(Exception): - signature_from_cert_request(cert, peer_public_key, agent_key_2.address) - - -def test_agent_record_errors(): - """Test AgentRecord check_validity""" - agent_key_1 = make_crypto(DEFAULT_LEDGER) - agent_key_2 = make_crypto(DEFAULT_LEDGER) - peer_public_key_1 = make_crypto(DEFAULT_LEDGER).public_key peer_public_key_2 = make_crypto(DEFAULT_LEDGER).public_key cert_path = "test_acn_cert.txt" - service_id = "test_acn_service" cert = CertRequest( peer_public_key_1, @@ -78,57 +47,57 @@ def test_agent_record_errors(): cert_path, ) _process_cert(agent_key_1, cert) - signature, _ = signature_from_cert_request( - cert, peer_public_key_1, agent_key_1.address + + # success + agent_record = AgentRecord.from_cert_request( + cert, agent_key_1.address, peer_public_key_1 + ) + assert ( + agent_record.address == agent_key_1.address + and agent_record.public_key == agent_key_1.public_key + and agent_record.representative_public_key == peer_public_key_1 + and agent_record.signature == cert.get_signature() + and agent_record.message == cert.get_message(peer_public_key_1) ) # success agent_record = AgentRecord( agent_key_1.address, - agent_key_1.public_key, peer_public_key_1, - signature, - service_id, + cert.get_message(peer_public_key_1), + cert.get_signature(), + DEFAULT_LEDGER, ) - agent_record.check_validity(agent_key_1.address, peer_public_key_1) assert ( agent_record.address == agent_key_1.address and agent_record.public_key == agent_key_1.public_key - and agent_record.peer_public_key == peer_public_key_1 - and agent_record.signature == signature - and agent_record.service_id == service_id + and agent_record.representative_public_key == peer_public_key_1 + and agent_record.signature == cert.get_signature() + and agent_record.message == cert.get_message(peer_public_key_1) ) - # error: wrong agent address - with pytest.raises(Exception): - agent_record.check_validity(agent_key_2.address, peer_public_key_1) + # error: wrong ledger + with pytest.raises( + ValueError, match="Not a valid ledger_id. Found=wrong_ledger, valid ids=" + ): + agent_record = AgentRecord( + agent_key_1.address, + peer_public_key_1, + cert.get_message(peer_public_key_1), + cert.get_signature(), + "wrong_ledger", + ) - # error: wrong peer - with pytest.raises(Exception): - agent_record.check_validity(agent_key_1.address, peer_public_key_2) - - # error: agent address and public key don't match - agent_record = AgentRecord( - agent_key_2.address, - agent_key_1.public_key, - peer_public_key_1, - signature, - service_id, - ) - with pytest.raises(Exception): - agent_record.check_validity(agent_key_2.address, peer_public_key_1) + # error: wrong signer + with pytest.raises( + ValueError, + match="Invalid signature for provided representative_public_key and agent address!", + ): + AgentRecord.from_cert_request(cert, agent_key_2.address, peer_public_key_1) - # error: invalid signature - _process_cert(agent_key_2, cert) - signature, _ = signature_from_cert_request( - cert, peer_public_key_1, agent_key_2.address - ) - agent_record = AgentRecord( - agent_key_1.address, - agent_key_1.public_key, - peer_public_key_1, - signature, - service_id, - ) - with pytest.raises(Exception): - agent_record.check_validity(agent_key_1.address, peer_public_key_1) + # error: wrong signer + with pytest.raises( + ValueError, + match="Invalid signature for provided representative_public_key and agent address!", + ): + AgentRecord.from_cert_request(cert, agent_key_1.address, peer_public_key_2) From d7f359133378cd16eac1c1574b47d69721f039e0 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Sat, 2 Jan 2021 19:27:21 +0000 Subject: [PATCH 134/204] fix broken pylint --- aea/helpers/acn/agent_record.py | 8 +------- scripts/missing: | 20 ------------------- .../test_acn/test_agent_record.py | 12 ----------- 3 files changed, 1 insertion(+), 39 deletions(-) delete mode 100644 scripts/missing: diff --git a/aea/helpers/acn/agent_record.py b/aea/helpers/acn/agent_record.py index 74eaab45fa..296d7c759d 100644 --- a/aea/helpers/acn/agent_record.py +++ b/aea/helpers/acn/agent_record.py @@ -28,8 +28,6 @@ class AgentRecord: """Agent Proof-of-Representation to representative.""" - VALID_LEDGER_IDS = ["fetchai", "cosmos", "ethereum"] - def __init__( self, address: str, @@ -64,10 +62,6 @@ def _check_validity(self) -> None: - if agent signed the message - if message is correctly formatted """ - if self.ledger_id not in self.VALID_LEDGER_IDS: - raise ValueError( - f"Not a valid ledger_id. Found={self.ledger_id}, valid ids={self.VALID_LEDGER_IDS}." - ) if self.message != self._get_message(self.representative_public_key): raise ValueError("Invalid message.") # pragma: no cover ledger_api = make_ledger_api(self.ledger_id) @@ -131,7 +125,7 @@ def _get_message(self, public_key: str) -> bytes: # pylint: disable=no-self-use def __str__(self): # pragma: no cover """Get string representation.""" - return f"(address={self.address}, public_key={self.public_key}, peer_public_key={self.peer_public_key}, signature={self.signature}, ledger_id={self.ledger_id})" + return f"(address={self.address}, public_key={self.public_key}, representative_public_key={self.representative_public_key}, signature={self.signature}, ledger_id={self.ledger_id})" @classmethod def from_cert_request( diff --git a/scripts/missing: b/scripts/missing: deleted file mode 100644 index 7cffdd801f..0000000000 --- a/scripts/missing: +++ /dev/null @@ -1,20 +0,0 @@ -missing: -- docs on PoR -- docs on interaction protocols - -Mo: -Documentation week & launch AW-4 - -- Lokman focus on Ethereum support, deploy new public DHT nodes -- Marco > doc tests; https://github.com/fetchai/agents-aea/issues/2078 -- Yuri > https://github.com/fetchai/agents-aea/issues/1923 -- Ali > TAC: feedback from BAI, IJCAI - - -Big themes Jan: -- messaging internal & external (protocols) -- Yuri to finish config analysis -- Yuri to review and fix: https://docs.google.com/document/d/1hY5N_-TtvSD3Z961xPjsbtw2oOKlIBJucr-jTzbl_1E/edit -- Marco: crypto support in configs - -- updated benchmark for papers \ No newline at end of file diff --git a/tests/test_helpers/test_acn/test_agent_record.py b/tests/test_helpers/test_acn/test_agent_record.py index 2d13832832..b3cf452561 100644 --- a/tests/test_helpers/test_acn/test_agent_record.py +++ b/tests/test_helpers/test_acn/test_agent_record.py @@ -76,18 +76,6 @@ def test_agent_record(): and agent_record.message == cert.get_message(peer_public_key_1) ) - # error: wrong ledger - with pytest.raises( - ValueError, match="Not a valid ledger_id. Found=wrong_ledger, valid ids=" - ): - agent_record = AgentRecord( - agent_key_1.address, - peer_public_key_1, - cert.get_message(peer_public_key_1), - cert.get_signature(), - "wrong_ledger", - ) - # error: wrong signer with pytest.raises( ValueError, From 95a3c93b471392943e8c0566067761eee60cc28f Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Sat, 2 Jan 2021 20:28:51 +0000 Subject: [PATCH 135/204] fix client test --- packages/fetchai/connections/p2p_libp2p_client/connection.py | 2 +- packages/fetchai/connections/p2p_libp2p_client/connection.yaml | 2 +- packages/hashes.csv | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.py b/packages/fetchai/connections/p2p_libp2p_client/connection.py index 709b531d36..08d662a0aa 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.py +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.py @@ -189,7 +189,7 @@ async def _setup_connection(self): record = AgentRecordPb() record.address = self.node_por.address record.public_key = self.node_por.public_key - record.peer_public_key = self.node_por.peer_public_key + record.peer_public_key = self.node_por.representative_public_key record.signature = self.node_por.signature record.service_id = POR_DEFAULT_SERVICE_ID diff --git a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml index c5e0475135..e82924dd31 100644 --- a/packages/fetchai/connections/p2p_libp2p_client/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p_client/connection.yaml @@ -12,7 +12,7 @@ fingerprint: __init__.py: QmT1FEHkPGMHV5oiVEfQHHr25N2qdZxydSNRJabJvYiTgf acn_message.proto: QmSwpCP72J94kt2MaLSPQdt8puqtLMVPp3Migwp4xRwadZ acn_message_pb2.py: QmbVqUUCHyQpfx7dHvW5e5j3RGcjNYqsDjuPV8eNcca2Nf - connection.py: QmahcgM3MHb2tLcphtXzoMFUCZjN2yamWagVyVSKDAvpbs + connection.py: QmP81pnqEQ1xCrPdSUauDEe1NAh58mdQqR5ddxZCKy47iT fingerprint_ignore_patterns: [] connections: [] protocols: [] diff --git a/packages/hashes.csv b/packages/hashes.csv index 0557c411b9..ce3c6d1f63 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -35,7 +35,7 @@ fetchai/connections/ledger,QmdnQCqqfcp2riC6wChYx2syHnvi2ckzfVAoBUkfzoUBSY fetchai/connections/local,QmPVKd6twXeWqZUVkhFHmYEpgJiCsss5fSqgvSqPRtcvkQ fetchai/connections/oef,QmcTkUM6Yw6UW71S9Yg5ifZ9mhceTjJy3dUwRS3uTVjQR6 fetchai/connections/p2p_libp2p,QmddLzTtGexWtuvZPLfe58UvChfXpNwXFyCJdE4goaZ6Zp -fetchai/connections/p2p_libp2p_client,QmeLwBkdC54VN9YAEG3U8VcZxLondzhLjC6Tb5XKhZhQMR +fetchai/connections/p2p_libp2p_client,QmeHHAsmCPaygs9A7CrSz4Y4xLVrNQefBiQJ3K2Ex99eXi fetchai/connections/p2p_stub,QmauyRLKUc9KFg6UbAsWyTNPXg1S77tHPWF7GW13RxcvZf fetchai/connections/prometheus,QmVbQAy8CuSwvRRABNSZRiyaFRLYmiiLPR4hzkLAxJzi1T fetchai/connections/scaffold,QmaCupER4vedakPPLzn2Rw9YHPmqSPaZh5xnqi6xVxQHvn From 583d0370a09ca8e1d52eeee144ae08f366c489ad Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sun, 3 Jan 2021 11:32:00 +0100 Subject: [PATCH 136/204] add tests for 'docs/simple-oef-usage.md' --- tests/test_docs/helper.py | 4 +- .../test_bash_yaml/test_demo_docs.py | 2 +- tests/test_docs/test_simple_oef_usage.py | 37 +++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 tests/test_docs/test_simple_oef_usage.py diff --git a/tests/test_docs/helper.py b/tests/test_docs/helper.py index 0f6d237dc0..72db902688 100644 --- a/tests/test_docs/helper.py +++ b/tests/test_docs/helper.py @@ -30,7 +30,7 @@ MISTUNE_BLOCK_CODE_ID = "block_code" -def extract_code_blocks(filepath, filter=None): +def extract_code_blocks(filepath, filter_=None): """Extract code blocks from .md files.""" code_blocks = [] with open(filepath, "r", encoding="utf-8") as f: @@ -42,7 +42,7 @@ def extract_code_blocks(filepath, filter=None): out = re.match("[^`]*```(.*)$", line) if out: - if filter and filter.strip() != out.group(1).strip(): + if filter_ and filter_.strip() != out.group(1).strip(): continue code_block = [f.readline()] while re.search("```", code_block[-1]) is None: diff --git a/tests/test_docs/test_bash_yaml/test_demo_docs.py b/tests/test_docs/test_bash_yaml/test_demo_docs.py index dcf0cc39af..1a607a41f7 100644 --- a/tests/test_docs/test_bash_yaml/test_demo_docs.py +++ b/tests/test_docs/test_bash_yaml/test_demo_docs.py @@ -40,7 +40,7 @@ def _test_blocks(self, filename: str, filter_: str): bash_file = Path(self.BASH_DIR_PATH, filename).read_text() md_path = os.path.join(ROOT_DIR, "docs", filename.replace("bash-", "")) - bash_code_blocks = extract_code_blocks(filepath=md_path, filter=filter_) + bash_code_blocks = extract_code_blocks(filepath=md_path, filter_=filter_) for blocks in bash_code_blocks: assert blocks in bash_file, "[{}]: FAILED. Code must be identical".format( filename diff --git a/tests/test_docs/test_simple_oef_usage.py b/tests/test_docs/test_simple_oef_usage.py new file mode 100644 index 0000000000..563ef645a9 --- /dev/null +++ b/tests/test_docs/test_simple_oef_usage.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the tests for the content of simple-oef-usage.md file.""" +from pathlib import Path + +from aea.helpers.search.models import Description +from packages.fetchai.protocols.oef_search import OefSearchMessage +from tests.conftest import ROOT_DIR +from tests.test_docs.helper import BasePythonMarkdownDocs + + +class TestSimpleOefUsage(BasePythonMarkdownDocs): + """Test the data models code snippets.""" + + DOC_PATH = Path(ROOT_DIR, "docs", "simple-oef-usage.md") + + def _assert(self, *mocks, **locals_): + """Assert executed code.""" + assert isinstance(locals_["message"], OefSearchMessage) + assert isinstance(locals_["service_description"], Description) From c21dea2254ac538d5dbf8e3c6a1c6f432d2caa69 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sun, 3 Jan 2021 12:06:49 +0100 Subject: [PATCH 137/204] add tests for 'docs/skill-testing.md' --- docs/skill-testing.md | 18 +++++++++ tests/test_docs/helper.py | 4 +- tests/test_docs/test_data_models.py | 2 +- tests/test_docs/test_ledger_integration.py | 16 ++++---- tests/test_docs/test_multiagent_manager.py | 3 -- tests/test_docs/test_simple_oef_usage.py | 6 ++- tests/test_docs/test_skill_testing.py | 43 ++++++++++++++++++++++ 7 files changed, 76 insertions(+), 16 deletions(-) create mode 100644 tests/test_docs/test_skill_testing.py diff --git a/docs/skill-testing.md b/docs/skill-testing.md index dc9b76d9a3..052ade7374 100644 --- a/docs/skill-testing.md +++ b/docs/skill-testing.md @@ -9,6 +9,19 @@ Let us assume you want to test the `my_behaviour` behaviour of a `CustomSkill` s You can create a `TestMyBehaviour` class which inherits `BaseSkillTestCase` as below: ``` python +import asyncio + +from asyncio import Queue +from pathlib import Path +from types import SimpleNamespace +from typing import cast + +from aea.configurations.constants import DEFAULT_LEDGER +from aea.context.base import AgentContext +from aea.crypto.ledger_apis import DEFAULT_CURRENCY_DENOMINATIONS +from aea.identity.base import Identity +from aea.multiplexer import AsyncMultiplexer, OutBox, Multiplexer +from aea.skills.tasks import TaskManager from aea.test_tools.test_skill import BaseSkillTestCase class TestMyBehaviour(BaseSkillTestCase): @@ -58,6 +71,8 @@ agent_context = AgentContext( decision_maker_message_queue=Queue(), decision_maker_handler_context=SimpleNamespace(), task_manager=TaskManager(), + default_ledger_id=DEFAULT_LEDGER, + currency_denominations={}, default_connection=None, default_routing={}, search_service_address="dummy_search_service_address", @@ -88,6 +103,9 @@ There are a number of methods that `BaseSkillTestCase` offers to make testing sk You can check the output of your skill's `logger` by mocking it using `unittest.mock` before executing a part of your skill as such: ``` python +import logging +from unittest import mock + with mock.patch.object(self.my_behaviour.context.logger, "log") as mock_logger: self.my_behaviour.act() diff --git a/tests/test_docs/helper.py b/tests/test_docs/helper.py index 72db902688..bcbf7db71d 100644 --- a/tests/test_docs/helper.py +++ b/tests/test_docs/helper.py @@ -138,7 +138,7 @@ def setup_class(cls): def _python_selector(cls, block: Dict) -> bool: return block["type"] == "block_code" and block["info"].strip() == "python" - def _assert(self, *mocks, **locals_): + def _assert(self, locals_, *mocks): """Do assertions after Python code execution.""" def test_python_blocks(self, *mocks): @@ -149,4 +149,4 @@ def test_python_blocks(self, *mocks): for python_block in python_blocks: python_code = python_block["text"] exec(python_code, globals_, locals_) # nosec - self._assert(*mocks, **locals_) + self._assert(locals_, *mocks) diff --git a/tests/test_docs/test_data_models.py b/tests/test_docs/test_data_models.py index d999c9881d..77edb4a84f 100644 --- a/tests/test_docs/test_data_models.py +++ b/tests/test_docs/test_data_models.py @@ -31,7 +31,7 @@ class TestDataModel(BasePythonMarkdownDocs): DOC_PATH = Path(ROOT_DIR, "docs", "defining-data-models.md") - def _assert(self, *_mocks, **locals_): + def _assert(self, locals_, *_mocks): attribute = locals_["attr_title"] assert isinstance(attribute, Attribute) diff --git a/tests/test_docs/test_ledger_integration.py b/tests/test_docs/test_ledger_integration.py index 39811a3618..b044bfc3a4 100644 --- a/tests/test_docs/test_ledger_integration.py +++ b/tests/test_docs/test_ledger_integration.py @@ -54,20 +54,20 @@ class TestLedgerIntegration(BasePythonMarkdownDocs): DOC_PATH = Path(ROOT_DIR, "docs", "ledger-integration.md") - def _assert_isinstance(self, locals_key, cls, **locals_): + def _assert_isinstance(self, locals_key, cls, locals_): """Assert that the member of 'locals' is an instance of a class.""" assert locals_key in locals_ obj = locals_[locals_key] assert isinstance(obj, cls) - def _assert(self, *mocks, **locals_): + def _assert(self, locals_, *mocks): """Assert code outputs.""" - self._assert_isinstance("fetchai_crypto", FetchAICrypto, **locals_) - self._assert_isinstance("fetchai_ledger_api", FetchAIApi, **locals_) - self._assert_isinstance("fetchai_faucet_api", FetchAIFaucetApi, **locals_) - self._assert_isinstance("my_ledger_crypto", MagicMock, **locals_) - self._assert_isinstance("my_ledger_api", MagicMock, **locals_) - self._assert_isinstance("my_faucet_api", MagicMock, **locals_) + self._assert_isinstance("fetchai_crypto", FetchAICrypto, locals_) + self._assert_isinstance("fetchai_ledger_api", FetchAIApi, locals_) + self._assert_isinstance("fetchai_faucet_api", FetchAIFaucetApi, locals_) + self._assert_isinstance("my_ledger_crypto", MagicMock, locals_) + self._assert_isinstance("my_ledger_api", MagicMock, locals_) + self._assert_isinstance("my_faucet_api", MagicMock, locals_) @classmethod def teardown_class(cls): diff --git a/tests/test_docs/test_multiagent_manager.py b/tests/test_docs/test_multiagent_manager.py index 7cf5956cab..4925cb380e 100644 --- a/tests/test_docs/test_multiagent_manager.py +++ b/tests/test_docs/test_multiagent_manager.py @@ -55,9 +55,6 @@ def setup_class(cls): shutil.copytree(PACKAGES_DIR, Path(cls.temp_dir, "packages")) os.chdir(cls.temp_dir) - def _assert(self, *mocks, **locals_): - """Assert code outputs.""" - @classmethod def teardown_class(cls): """Teardown class.""" diff --git a/tests/test_docs/test_simple_oef_usage.py b/tests/test_docs/test_simple_oef_usage.py index 563ef645a9..b2a28eb8bb 100644 --- a/tests/test_docs/test_simple_oef_usage.py +++ b/tests/test_docs/test_simple_oef_usage.py @@ -21,17 +21,19 @@ from pathlib import Path from aea.helpers.search.models import Description + from packages.fetchai.protocols.oef_search import OefSearchMessage + from tests.conftest import ROOT_DIR from tests.test_docs.helper import BasePythonMarkdownDocs class TestSimpleOefUsage(BasePythonMarkdownDocs): - """Test the data models code snippets.""" + """Test the simple OEF usage code snippets.""" DOC_PATH = Path(ROOT_DIR, "docs", "simple-oef-usage.md") - def _assert(self, *mocks, **locals_): + def _assert(self, locals_, *mocks): """Assert executed code.""" assert isinstance(locals_["message"], OefSearchMessage) assert isinstance(locals_["service_description"], Description) diff --git a/tests/test_docs/test_skill_testing.py b/tests/test_docs/test_skill_testing.py new file mode 100644 index 0000000000..4bb2a0ace1 --- /dev/null +++ b/tests/test_docs/test_skill_testing.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the tests for the content of skill-testing.md file.""" +from pathlib import Path +from unittest import mock +from unittest.mock import MagicMock + +from tests.conftest import ROOT_DIR +from tests.test_docs.helper import BasePythonMarkdownDocs + + +@mock.patch("unittest.mock.MagicMock.assert_any_call") +class TestSkillTesting(BasePythonMarkdownDocs): + """Test the skill testing code snippets.""" + + DOC_PATH = Path(ROOT_DIR, "docs", "skill-testing.md") + + @classmethod + def setup_class(cls): + """Set up the test.""" + super().setup_class() + # without this, it fails at the first block + # with: "NameError: name 'Path' is not defined" + cls.globals.update(globals()) + + cls.locals.update(dict(cls=MagicMock(), self=MagicMock())) From 151a9f01794c8e57a5ec10e4fbbd7826f77c5674 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sun, 3 Jan 2021 14:51:17 +0100 Subject: [PATCH 138/204] add tests for 'docs/quickstart.md', plus refactoring --- docs/config.md | 4 +- docs/quickstart.md | 4 +- tests/test_docs/helper.py | 60 +++++++++++-------- .../test_agent_vs_aea/test_agent_vs_aea.py | 2 +- .../test_bash_yaml/md_files/bash-config.md | 21 +++++++ .../md_files/bash-quickstart.md | 21 +++++-- .../test_bash_yaml/test_demo_docs.py | 4 +- .../test_programmatic_aea.py | 2 +- .../test_cli_vs_programmatic_aea.py | 2 +- .../test_decision_maker_transaction.py | 2 +- .../test_generic_step_by_step_guide.py | 2 +- .../test_multiplexer_standalone.py | 2 +- tests/test_docs/test_quickstart.py | 30 ++++++++++ .../test_skill_guide/test_skill_guide.py | 4 +- .../test_standalone_transaction.py | 2 +- 15 files changed, 116 insertions(+), 46 deletions(-) create mode 100644 tests/test_docs/test_quickstart.py diff --git a/docs/config.md b/docs/config.md index af3094736c..e3ca95e066 100644 --- a/docs/config.md +++ b/docs/config.md @@ -57,14 +57,14 @@ decision_maker_handler: None # The decision maker handler to The `aea-config.yaml` can further be extended with component configuration overrides. For custom connection configurations: -``` +``` yaml public_id: some_author/some_package:0.1.0 # The public id of the connection (must satisfy PUBLIC_ID_REGEX). type: connection # for connections, this must be "connection". config: ... # a dictionary to overwrite the `config` field (see below) ``` For custom skill configurations: -``` +``` yaml public_id: some_author/some_package:0.1.0 # The public id of the connection (must satisfy PUBLIC_ID_REGEX). type: skill # for skills, this must be "skill". behaviours: # override configurations for behaviours diff --git a/docs/quickstart.md b/docs/quickstart.md index ecd75ac296..f340b5b76c 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -23,7 +23,7 @@ The following hints can help: - Python is already included by default on many Linux distributions (e.g. Ubuntu), as well as MacOS. To check you have the right version, open a terminal and run: -``` +``` bash python3 --version ``` @@ -391,7 +391,7 @@ Place the above code into a file `test.py` in your AEA project directory (the sa To run, execute the following: -``` python +``` bash pytest test.py ``` diff --git a/tests/test_docs/helper.py b/tests/test_docs/helper.py index bcbf7db71d..a95ff7ef36 100644 --- a/tests/test_docs/helper.py +++ b/tests/test_docs/helper.py @@ -18,10 +18,10 @@ # ------------------------------------------------------------------------------ """This module contains helper function to extract code from the .md files.""" -import re import traceback +from functools import partial from pathlib import Path -from typing import Dict +from typing import Dict, Optional import mistune import pytest @@ -30,25 +30,36 @@ MISTUNE_BLOCK_CODE_ID = "block_code" +def block_code_filter(b: Dict) -> bool: + """Check Mistune block is a code block.""" + return b["type"] == MISTUNE_BLOCK_CODE_ID + + +def type_filter(type_: Optional[str], b: Dict) -> bool: + """ + Check Mistune code block is of a certain type. + + If the field "info" is None, return False. + If type_ is None, this function always return true. + + :param type_: the expected type of block (optional) + :param b: the block dicionary. + :return: True if the block should be accepted, false otherwise. + """ + if type_ is None: + return True + return b["info"].strip() == type_ if b["info"] is not None else False + + def extract_code_blocks(filepath, filter_=None): """Extract code blocks from .md files.""" - code_blocks = [] - with open(filepath, "r", encoding="utf-8") as f: - while True: - line = f.readline() - if not line: - # EOF - break - - out = re.match("[^`]*```(.*)$", line) - if out: - if filter_ and filter_.strip() != out.group(1).strip(): - continue - code_block = [f.readline()] - while re.search("```", code_block[-1]) is None: - code_block.append(f.readline()) - code_blocks.append("".join(code_block[:-1])) - return code_blocks + content = Path(filepath).read_text(encoding="utf-8") + markdown_parser = mistune.create_markdown(renderer=mistune.AstRenderer()) + blocks = markdown_parser(content) + actual_type_filter = partial(type_filter, filter_) + code_blocks = list(filter(block_code_filter, blocks)) + bash_code_blocks = filter(actual_type_filter, code_blocks) + return list(b["text"] for b in bash_code_blocks) def extract_python_code(filepath): @@ -105,11 +116,6 @@ class BaseTestMarkdownDocs: DOC_PATH: Path - @classmethod - def _block_code_filter(cls, b: Dict) -> bool: - """Check Mistune block is a code block.""" - return b["type"] == MISTUNE_BLOCK_CODE_ID - @classmethod def setup_class(cls): """Set up the test.""" @@ -117,7 +123,7 @@ def setup_class(cls): cls.doc_path = cls.DOC_PATH cls.doc_content = cls.doc_path.read_text() cls.blocks = markdown_parser(cls.doc_content) - cls.code_blocks = list(filter(cls._block_code_filter, cls.blocks)) + cls.code_blocks = list(filter(block_code_filter, cls.blocks)) class BasePythonMarkdownDocs(BaseTestMarkdownDocs): @@ -136,7 +142,9 @@ def setup_class(cls): @classmethod def _python_selector(cls, block: Dict) -> bool: - return block["type"] == "block_code" and block["info"].strip() == "python" + return block["type"] == MISTUNE_BLOCK_CODE_ID and ( + block["info"].strip() == "python" if block["info"] else False + ) def _assert(self, locals_, *mocks): """Do assertions after Python code execution.""" diff --git a/tests/test_docs/test_agent_vs_aea/test_agent_vs_aea.py b/tests/test_docs/test_agent_vs_aea/test_agent_vs_aea.py index 7711033113..838772f0f9 100644 --- a/tests/test_docs/test_agent_vs_aea/test_agent_vs_aea.py +++ b/tests/test_docs/test_agent_vs_aea/test_agent_vs_aea.py @@ -42,7 +42,7 @@ def setup_class(cls): """Setup the test class.""" BaseAEATestCase.setup_class() doc_path = os.path.join(ROOT_DIR, MD_FILE) - cls.code_blocks = extract_code_blocks(filepath=doc_path, filter="python") + cls.code_blocks = extract_code_blocks(filepath=doc_path, filter_="python") test_code_path = os.path.join(CUR_PATH, PY_FILE) cls.python_file = extract_python_code(test_code_path) diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-config.md b/tests/test_docs/test_bash_yaml/md_files/bash-config.md index 295aa3ddd0..e01650b5ba 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-config.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-config.md @@ -45,6 +45,27 @@ error_handler: None # The error handler to be used. decision_maker_handler: None # The decision maker handler to be used. ``` ``` yaml +public_id: some_author/some_package:0.1.0 # The public id of the connection (must satisfy PUBLIC_ID_REGEX). +type: connection # for connections, this must be "connection". +config: ... # a dictionary to overwrite the `config` field (see below) +``` +``` yaml +public_id: some_author/some_package:0.1.0 # The public id of the connection (must satisfy PUBLIC_ID_REGEX). +type: skill # for skills, this must be "skill". +behaviours: # override configurations for behaviours + behaviour_1: # override configurations for "behaviour_1" + args: # arguments for a specific behaviour (see below) + foo: bar +handlers: # override configurations for handlers + handler_1: # override configurations for "handler_1" + args: # arguments for a specific handler (see below) + foo: bar +models: # override configurations for models + model_1: # override configurations for "model_1" + args: # arguments for a specific model (see below) + foo: bar +``` +``` yaml name: scaffold # Name of the package (must satisfy PACKAGE_REGEX) author: fetchai # Author handle of the package's author (must satisfy AUTHOR_REGEX) version: 0.1.0 # Version of the package (a semantic version number, see https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string") diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-quickstart.md b/tests/test_docs/test_bash_yaml/md_files/bash-quickstart.md index 1d3c5ee3ca..5d0339c0ee 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-quickstart.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-quickstart.md @@ -1,12 +1,18 @@ +``` bash +python3 --version +``` +``` bash +sudo apt-get install python3.7-dev +``` ```bash docker pull fetchai/aea-user:latest ``` - ```bash - docker run -it -v $(pwd):/agents --workdir=/agents fetchai/aea-user:latest - ``` - ```bash - docker run -it -v %cd%:/agents --workdir=/agents fetchai/aea-user:latest - ``` +```bash +docker run -it -v $(pwd):/agents --workdir=/agents fetchai/aea-user:latest +``` +```bash +docker run -it -v %cd%:/agents --workdir=/agents fetchai/aea-user:latest +``` ``` bash mkdir my_aea_projects/ cd my_aea_projects/ @@ -121,5 +127,8 @@ info: Echo Behaviour: teardown method called. aea interact ``` ``` bash +pytest test.py +``` +``` bash aea delete my_first_aea ``` diff --git a/tests/test_docs/test_bash_yaml/test_demo_docs.py b/tests/test_docs/test_bash_yaml/test_demo_docs.py index 1a607a41f7..a09103e4b1 100644 --- a/tests/test_docs/test_bash_yaml/test_demo_docs.py +++ b/tests/test_docs/test_bash_yaml/test_demo_docs.py @@ -45,7 +45,9 @@ def _test_blocks(self, filename: str, filter_: str): assert blocks in bash_file, "[{}]: FAILED. Code must be identical".format( filename ) - logger.info("[{}]: PASSED".format(filename)) + logger.info( + f"[{filename}]: PASSED. Tested {len(bash_code_blocks)} '{filter_}' blocks." + ) def test_code_blocks_exist(self): """Test that all the code-blocks exist in the python file.""" diff --git a/tests/test_docs/test_build_aea_programmatically/test_programmatic_aea.py b/tests/test_docs/test_build_aea_programmatically/test_programmatic_aea.py index 04a75baf07..1e01dee735 100644 --- a/tests/test_docs/test_build_aea_programmatically/test_programmatic_aea.py +++ b/tests/test_docs/test_build_aea_programmatically/test_programmatic_aea.py @@ -42,7 +42,7 @@ def setup_class(cls): """Setup the test class.""" BaseAEATestCase.setup_class() doc_path = os.path.join(ROOT_DIR, MD_FILE) - cls.code_blocks = extract_code_blocks(filepath=doc_path, filter="python") + cls.code_blocks = extract_code_blocks(filepath=doc_path, filter_="python") test_code_path = os.path.join(CUR_PATH, PY_FILE) cls.python_file = extract_python_code(test_code_path) diff --git a/tests/test_docs/test_cli_vs_programmatic_aeas/test_cli_vs_programmatic_aea.py b/tests/test_docs/test_cli_vs_programmatic_aeas/test_cli_vs_programmatic_aea.py index 08d167d8a5..aca4386a37 100644 --- a/tests/test_docs/test_cli_vs_programmatic_aeas/test_cli_vs_programmatic_aea.py +++ b/tests/test_docs/test_cli_vs_programmatic_aeas/test_cli_vs_programmatic_aea.py @@ -54,7 +54,7 @@ class TestCliVsProgrammaticAEA(AEATestCaseMany): def test_read_md_file(self): """Compare the extracted code with the python file.""" doc_path = os.path.join(ROOT_DIR, MD_FILE) - code_blocks = extract_code_blocks(filepath=doc_path, filter="python") + code_blocks = extract_code_blocks(filepath=doc_path, filter_="python") test_code_path = os.path.join(CUR_PATH, PY_FILE) python_file = extract_python_code(test_code_path) assert code_blocks[-1] == python_file, "Files must be exactly the same." diff --git a/tests/test_docs/test_decision_maker_transaction/test_decision_maker_transaction.py b/tests/test_docs/test_decision_maker_transaction/test_decision_maker_transaction.py index 2c1dde598e..024ee8f3bf 100644 --- a/tests/test_docs/test_decision_maker_transaction/test_decision_maker_transaction.py +++ b/tests/test_docs/test_decision_maker_transaction/test_decision_maker_transaction.py @@ -54,7 +54,7 @@ def setup_class(cls): BaseAEATestCase.setup_class() cls._patch_logger() doc_path = os.path.join(ROOT_DIR, MD_FILE) - cls.code_blocks = extract_code_blocks(filepath=doc_path, filter="python") + cls.code_blocks = extract_code_blocks(filepath=doc_path, filter_="python") test_code_path = os.path.join(CUR_PATH, PY_FILE) cls.python_file = extract_python_code(test_code_path) diff --git a/tests/test_docs/test_generic_step_by_step_guide/test_generic_step_by_step_guide.py b/tests/test_docs/test_generic_step_by_step_guide/test_generic_step_by_step_guide.py index 4c2a4e7080..accf4596c4 100644 --- a/tests/test_docs/test_generic_step_by_step_guide/test_generic_step_by_step_guide.py +++ b/tests/test_docs/test_generic_step_by_step_guide/test_generic_step_by_step_guide.py @@ -37,7 +37,7 @@ class TestDemoDocs: def setup_class(cls): """Setup the test class.""" md_path = os.path.join(ROOT_DIR, "docs", "generic-skills-step-by-step.md") - code_blocks = extract_code_blocks(filepath=md_path, filter="python") + code_blocks = extract_code_blocks(filepath=md_path, filter_="python") cls.generic_seller = code_blocks[0:11] cls.generic_buyer = code_blocks[11 : len(code_blocks)] diff --git a/tests/test_docs/test_multiplexer_standalone/test_multiplexer_standalone.py b/tests/test_docs/test_multiplexer_standalone/test_multiplexer_standalone.py index a83578d408..8b0cf19b0f 100644 --- a/tests/test_docs/test_multiplexer_standalone/test_multiplexer_standalone.py +++ b/tests/test_docs/test_multiplexer_standalone/test_multiplexer_standalone.py @@ -41,7 +41,7 @@ def setup_class(cls): """Setup the test class.""" BaseAEATestCase.setup_class() doc_path = os.path.join(ROOT_DIR, MD_FILE) - cls.code_blocks = extract_code_blocks(filepath=doc_path, filter="python") + cls.code_blocks = extract_code_blocks(filepath=doc_path, filter_="python") test_code_path = os.path.join(CUR_PATH, PY_FILE) cls.python_file = extract_python_code(test_code_path) diff --git a/tests/test_docs/test_quickstart.py b/tests/test_docs/test_quickstart.py new file mode 100644 index 0000000000..4dd904c29e --- /dev/null +++ b/tests/test_docs/test_quickstart.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the tests for the content of quickstart.md file.""" +from pathlib import Path + +from tests.conftest import ROOT_DIR +from tests.test_docs.helper import BasePythonMarkdownDocs + + +class TestQuickstartTest(BasePythonMarkdownDocs): + """Test the quickstart test.""" + + DOC_PATH = Path(ROOT_DIR, "docs", "quickstart.md") diff --git a/tests/test_docs/test_skill_guide/test_skill_guide.py b/tests/test_docs/test_skill_guide/test_skill_guide.py index 381d7231a2..35fed8f301 100644 --- a/tests/test_docs/test_skill_guide/test_skill_guide.py +++ b/tests/test_docs/test_skill_guide/test_skill_guide.py @@ -59,7 +59,7 @@ def setup_class(cls): """Setup the test class.""" AEATestCaseMany.setup_class() cls.doc_path = os.path.join(ROOT_DIR, MD_FILE) - cls.code_blocks = extract_code_blocks(filepath=cls.doc_path, filter="python") + cls.code_blocks = extract_code_blocks(filepath=cls.doc_path, filter_="python") def test_read_md_file(self): """Teat that the md file is not empty.""" @@ -138,7 +138,7 @@ def test_update_skill_and_run(self): os.rename(path, path_new) path = Path(self.t, search_aea, "skills", skill_name, "skill.yaml") - yaml_code_block = extract_code_blocks(self.doc_path, filter="yaml") + yaml_code_block = extract_code_blocks(self.doc_path, filter_="yaml") with open(path, "w") as file: file.write(yaml_code_block[0]) # block one is yaml diff --git a/tests/test_docs/test_standalone_transaction/test_standalone_transaction.py b/tests/test_docs/test_standalone_transaction/test_standalone_transaction.py index 47e99223a9..d42e2ceccb 100644 --- a/tests/test_docs/test_standalone_transaction/test_standalone_transaction.py +++ b/tests/test_docs/test_standalone_transaction/test_standalone_transaction.py @@ -60,7 +60,7 @@ def setup_class(cls): BaseAEATestCase.setup_class() cls._patch_logger() doc_path = os.path.join(ROOT_DIR, MD_FILE) - cls.code_blocks = extract_code_blocks(filepath=doc_path, filter="python") + cls.code_blocks = extract_code_blocks(filepath=doc_path, filter_="python") test_code_path = os.path.join(CUR_PATH, PY_FILE) cls.python_file = extract_python_code(test_code_path) From 97b34a642f149979d00f6a6cebef99d3834634bc Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sun, 3 Jan 2021 15:16:19 +0100 Subject: [PATCH 139/204] fix bash-oracle-demo.md --- .../test_docs/test_bash_yaml/md_files/bash-oracle-demo.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-oracle-demo.md b/tests/test_docs/test_bash_yaml/md_files/bash-oracle-demo.md index e0722b8d4d..b1ed200143 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-oracle-demo.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-oracle-demo.md @@ -31,6 +31,13 @@ aea generate-key ethereum aea add-key ethereum ethereum_private_key.txt ``` ``` bash +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +``` +``` bash +aea issue-certificates +``` +``` bash docker run -p 8545:8545 trufflesuite/ganache-cli:latest --verbose --gasPrice=0 --gasLimit=0x1fffffffffffff --account="$(cat coin_price_oracle/ethereum_private_key.txt),1000000000000000000000" ``` ``` bash From 253dd59d142ae2c9bca90c8b6ce4f1bd62eb1cd4 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sun, 3 Jan 2021 15:31:41 +0100 Subject: [PATCH 140/204] temporarily skip docs/multi-agent-manager.md test, as it depends on other work. --- tests/test_docs/test_multiagent_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_docs/test_multiagent_manager.py b/tests/test_docs/test_multiagent_manager.py index 4925cb380e..74a8413a78 100644 --- a/tests/test_docs/test_multiagent_manager.py +++ b/tests/test_docs/test_multiagent_manager.py @@ -25,6 +25,8 @@ from pathlib import Path from unittest.mock import MagicMock +import pytest + from tests.conftest import PACKAGES_DIR, ROOT_DIR from tests.test_docs.helper import BasePythonMarkdownDocs @@ -41,6 +43,7 @@ def _import_module_mock(arg): return import_module(arg) +@pytest.mark.skip class TestMultiAgentManager(BasePythonMarkdownDocs): """Test the ledger integration code snippets.""" From 38daa59de196bf3f5d0a723246e6130cee214061 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Sun, 3 Jan 2021 18:36:23 +0100 Subject: [PATCH 141/204] open expected bash snippets with utf-8 encoding --- tests/test_docs/test_bash_yaml/test_demo_docs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_docs/test_bash_yaml/test_demo_docs.py b/tests/test_docs/test_bash_yaml/test_demo_docs.py index a09103e4b1..f466665a6c 100644 --- a/tests/test_docs/test_bash_yaml/test_demo_docs.py +++ b/tests/test_docs/test_bash_yaml/test_demo_docs.py @@ -37,7 +37,7 @@ class TestDemoDocs: def _test_blocks(self, filename: str, filter_: str): """Test code blocks of a certain type determined by 'filter_' param.""" - bash_file = Path(self.BASH_DIR_PATH, filename).read_text() + bash_file = Path(self.BASH_DIR_PATH, filename).read_text(encoding="utf-8") md_path = os.path.join(ROOT_DIR, "docs", filename.replace("bash-", "")) bash_code_blocks = extract_code_blocks(filepath=md_path, filter_=filter_) From 7a059e1e2707e22bd87ae31f09ccf2e7ef3f11c1 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Sun, 3 Jan 2021 18:33:49 +0000 Subject: [PATCH 142/204] add more contribution guides, add PoR first draft --- README.md | 29 ++++++++++++++++++++++++++++- docs/por.md | 4 ++++ mkdocs.yml | 1 + 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 docs/por.md diff --git a/README.md b/README.md index 029f7e1453..2e6bfe26b2 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,34 @@ The following dependency is **only relevant if you intend to contribute** to the The following steps are **only relevant if you intend to contribute** to the repository. They are **not required** for agent development. -- To install development dependencies (here optionally skipping `Pipfile.lock` creation): +### Recommended + +- The simplest way to get setup for development on the framework is to run the following: + + make new_env + pipenv shell + make install_env + +- For linting and static analysis use: + + make lint + make static + make pylint + make security + +- For testing aea.SUBMODULE with tests/TESTMODULE use: + + make dir=SUBMODULE tdir=TESTMODULE test-sub + + e.g. + + make dir=cli tdir=cli test-sub + +- To run demos against local packages use flag `--local` in `aea` CLI commands. + +### Alternative + +- To install development dependencies manually (here optionally skipping `Pipfile.lock` creation): pipenv install --dev --skip-lock diff --git a/docs/por.md b/docs/por.md new file mode 100644 index 0000000000..4824f021f1 --- /dev/null +++ b/docs/por.md @@ -0,0 +1,4 @@ + + +An AEA can use several key pairs. In particular, it can use different keys for securing its communication and for engaging in exchange. In the ACN we make use of this fact. To be able to signal to other agents that one key pair is allowed to represent another, the key pair which is being represented must sign a message to prove that the other key pair is allowed to represent it. + diff --git a/mkdocs.yml b/mkdocs.yml index 2a9b3c6433..43dca9d169 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -69,6 +69,7 @@ nav: - Multi agent manager: 'multi-agent-manager.md' - Debugging: 'debug.md' - Profiling: 'runtime-cost.md' + - Proof of Representation: 'por.md' - Use case components: - Generic skills: 'generic-skills.md' - Front-end intergration: 'connect-a-frontend.md' From 9a69c059aff0930af0dd0d65ede8405350410755 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Mon, 4 Jan 2021 09:43:29 +0000 Subject: [PATCH 143/204] update simple oef docs --- README.md | 4 ++-- docs/simple-oef.md | 27 +++++++++++++++++---------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 2e6bfe26b2..9db4945912 100644 --- a/README.md +++ b/README.md @@ -142,9 +142,9 @@ The following steps are **only relevant if you intend to contribute** to the rep make pylint make security -- For testing aea.SUBMODULE with tests/TESTMODULE use: +- For testing aea.{SUBMODULE} with tests/test_{TESTMODULE} use: - make dir=SUBMODULE tdir=TESTMODULE test-sub + make dir={SUBMODULE} tdir={TESTMODULE} test-sub e.g. diff --git a/docs/simple-oef.md b/docs/simple-oef.md index 8bc1b15cb3..81960dc231 100644 --- a/docs/simple-oef.md +++ b/docs/simple-oef.md @@ -1,6 +1,6 @@ -# Simple-OEF +# Simple-OEF: Agent Search and Discovery -This documentation has been produced for the Simple-OEF version `0.2.7`. +This documentation has been produced for the Simple-OEF version `0.3.4`. ## Concepts @@ -32,7 +32,7 @@ We cover all of these in this next section. It's important to understand the dif ### Personality Pieces -Agents can have a number of personality peices. These describe how an agent appears, where it is, and other properties such as heading, supported protocols and types of transactions. All personality pieces are optional. +Agents can have a number of personality peices. These describe how an agent appears, where it is, and other properties such as heading, supported protocols and types of transactions. All personality pieces are optional, but **the more you set, the easier it is for searchers to narrow you down accurately**. | Piece | Description | | ------------------- | ------------------------------------------------------------ | @@ -41,6 +41,7 @@ Agents can have a number of personality peices. These describe how an agent appe | `architecture` | Agent's architecture. See the architecture table below. Introduced in version `0.1.20`. The vast majority of agents should set this to `agentframework`. | | `dynamics.moving` | Boolean, indicates if the agent is moving or not. | | `dynamics.heading` | Indicates the heading of the agent, in radians, with 0.0 pointing due north. | +| `dynamics.altitude` | Altitude of the agent in metres from MSL | | `dynamics.position` | Indicates the GPS co-ordinates of the agent as latitude and longitude. | | `action.buyer` | Boolean, indicates whether the agent wishes to buy information, i.e., is an agent that requires value from another agent. | | `action.seller` | Boolean, indicates whether the agent sells information, i.e., provides value. Value provided can be zero-cost. | @@ -51,7 +52,7 @@ A genus is a coarse agent class. It is the roughest description of what an agent | Name | Description | | ----------- | ------------------------------------------------------------ | -| `test` | Agent is a test agent, and should be generally ignored | +| `test` | Agent is a test agent, and should be generally ignored. | | `vehicle` | Moving objects such as trains, planes and automobiles | | `avatar` | An agent that _represents_ a human being | | `service` | An agent that provides a service | @@ -61,8 +62,9 @@ A genus is a coarse agent class. It is the roughest description of what an agent | `building` | Large fixed location item such as house, railway station, school | | `buyer` | Indicates the agent is a buyer _only_ and does not have value to deliver | | `viewer` |The agent is a view in the world, acting as a "camera" to view content | +| `financial` |Financial agent: service, exchange, autonomous market maker, etc. | -The best way to use genus is to pick the *best fit* choice. If there isn't one for you, then do not specify it. If you feel that a high-level genus is missing, please make the suggestion in our Developer Discord (see here). +The best way to use genus is to pick the **best fit** choice. If there isn't one for you, then do not specify it. If you feel that a high-level genus is missing, please make the suggestion in our Developer Discord (see here). #### Architectures @@ -94,6 +96,8 @@ Agents can have a number of service keys. Service keys are simple key/value pair buying_genus buying_architecture buying_classifications +data_type +si_unit ``` This allows searches to look for potential buyers of classifications, genus or with a compatible architecture. @@ -105,6 +109,7 @@ The soef is designed for **geographic searches** where agents are able to find o Geographic searches are performed using the `find_around_me` operation. This allows searches that: * Are within a certain range in KM +* Optionally must be positioned within an angle of a heading * That have a specified set of personality pieces (with wildcards where applicable) * That have a specified set of service keys (with wildcards) * Where chain identifiers match @@ -157,7 +162,7 @@ The soef returns XML that includes information about all found agents. An exampl For the majority of use cases, the soef will be used from the Agent Framework. As a result, talking to it directly will not be needed. There are some occasions where interacting with the soef directly may be required, and this section documents the API functionality. -Until version 1.0 (expected in Q3/Q4 2020), some of the security and paid-for-services are not implemented and where they are, generally not enforced. Digital signatures for the sign-on process and unique identity recovery will be implemented, as will encryption on sensitive data transport, for example. Thus the API is likely to change substantially in the coming months, particularly the initial registration process. It is not recommended that you invest in substantial code that talks to the soef directly until after 1.0, and it is always preferred to go through the Agent Framework. +Until version 1.0 and main-net version 2 (expected in early 2021), some of the security and paid-for-services are not implemented and where they are, they generally not enforced. Digital signatures for the sign-on process and unique identity recovery will be implemented, as will encryption on sensitive data transport, for example. Thus the API is likely to change substantially in the coming months, particularly the initial registration process. It is not recommended that you invest in substantial code that talks to the soef directly until after 1.0, and it is always preferred to go through the Agent Framework. ### Registration @@ -232,10 +237,13 @@ The soef has a number of commands that can be used to set or update personality #### Find commands in detail -`find_around_me` and `find_on_this_node` are the big commands. Ultimately, they will cost a small amount of tokens to use, depending on the size of the request, as it involves the most computing time. This provides an incentive for soef operators to maintain soef nodes that correspond to subject areas, geographic areas or both. The command has a number of parameters specifying the filtering required. For `find_around_me`, the `range_in_km` is *required*. This cannot exceed a certain range, typically between 50 and 75km. This, and other configuration items, are available on the soef's configuration page. There are other parameters that are optional, although for `find_on_this_node` at least one `ppfilter` or `skfilter` must be specified. The parameters are: +`find_around_me` and `find_on_this_node` are the big commands. Ultimately, they will cost a small amount of tokens to use, depending on the size of the request, as it involves the most computing time. This provides an incentive for soef operators to maintain soef nodes that correspond to subject areas, geographic areas or both. The command has a number of parameters specifying the filtering required. For `find_around_me`, the `range_in_km` is *required*, whereas narrowing down agents to be within a certain angle of a direction is optional. This cannot exceed a certain range, typically between 50 and 75km. This, and other configuration items, are available on the soef's configuration page. There are other parameters that are optional, although for `find_on_this_node` at least one `ppfilter` or `skfilter` must be specified. The parameters are: | Parameter | Use | | ------------------- | ------------------------------------------------------------ | +| `range_in_km` | Range in kilometers to include agents in results. | +| `of_heading` | Optional: if a pizza-slice type search is required, this is the direction, in degrees, with 0.0 being north. | +| `within` | Optional: if a pizza-slice search, this is the angle in degrees from the `of_heading` that is allowed. If either `of_heading` or `within` are specified, **both** must be specified. Example: `of_heading` set to 90.0 and `within` set to 30 would exclude any agents that are not within 30 degrees of direct east of the me agent. | | `chains_must_match` | Boolean. Must be `true` or `false`. Default is `false`. If specified, this ensures that any agents returned in the search will have the same chain identifier as you. | | `ppfilter` | Specify a personality piece filter. Multiple `ppfilter`s can be specified. Example use is: `ppfilter=dynamics.moving,true`. Wildcards can be used where relevant, e.g.: `ppfilter=classification,mobility*` will match all classifications that *start* with `mobility`, whereas `ppfilter=classification,*mobility*` will match all classifications with `mobility` anywhere in it. | | `skfilter` | Specify a service key filter. Multiple `skfilter`s can be specified. Example use is: `skfilter=fruit,peach` which will require any returned results to have a service key of `fruit` and a value of `peach`. Wildcards can be specified, so `skfilter=fruit,pea*` will match any agent with a service key of `fruit` that starts `pea`, so `pear` and `peach` would match. | @@ -261,10 +269,9 @@ In this example, the key `type` must be present, and it must match to `fruit`. I ## Further information -You can find further information, or talk to us, in the #agents channel on our official developer Discord server, which you can access -here. +You can find further information, or talk to us, in the #agents channel on our official developer Discord server, which you can access here. We welcome your feedback and strive to deliver the best decentralised search and discovery service for agents that is possible. There are many upcoming features, including the operation incentive mechanisms, additional security and encryption, active searches (where results happen without `find_around_me` being issued), non-geographic searches across one and many soef nodes and dimensional-reduction based approximate searches. -[Docs: issue 13, 0.2.7, 05-Nov-2020, TWS] +[Docs: issue 15, 0.3.4, 28-Dec-2020, TWS] From 70ea64be31097afc672426c635b4fb31394202c3 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Mon, 4 Jan 2021 11:56:20 +0000 Subject: [PATCH 144/204] add interaction protocol explanation, fix tests for some docs, add por details --- docs/interaction-protocol.md | 23 +++++++++++++ docs/p2p-connection.md | 34 +++++++++++++++++-- docs/por.md | 18 +++++++++- .../md_files/bash-p2p-connection.md | 20 ++++++++++- .../test_bash_yaml/md_files/bash-por.md | 9 +++++ 5 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 tests/test_docs/test_bash_yaml/md_files/bash-por.md diff --git a/docs/interaction-protocol.md b/docs/interaction-protocol.md index e69de29bb2..a872dbf1ea 100644 --- a/docs/interaction-protocol.md +++ b/docs/interaction-protocol.md @@ -0,0 +1,23 @@ + +Interaction protocols are possible communication scenarios between agents or agent components (specifically, skills and connections). + +There are multiple types of interactions an AEA can have: + +- AEA to AEA interactions, as for instance demonstrated in the demo sections. + +- AEA internal interactions, between components of the framework. + + +Usually, an interaction involves three types of framework packages: skills, protocols and connections. + +## Example 1: negotiation + +In the generic buyer/seller skills the protocol `fetchai/fipa` is used for maintaining the negotiation dialogue between two AEAs. The skills `fetchai/generic_buyer` and `fetchai/generic_seller` are used to implement the handling and generating of individual messages and associated logic. The connextion `fetchai/p2p_libp2p` is used for connecting to the agent communication network. + +## Example 2: AEA <> web client + +In the http connection and skill guide we demo how a skill (`fetchai/http_echo`) can be used to process http requests received by a http server connection (`fetchai/http_server`). The `fetchai/http` protocol is used for communication between the connection and skill. + +## Example 3 : AEA <> 3rd party server + +The `fetchai/http_client` connection can be used to make requests to third party servers. Alternatively, a third party SDK can be wrapped in a connection and shared with other developers as a package. Often, the developer will also create a custom protocol to enforce the type of interactions permitted with the SDK wrapped in such a connection. diff --git a/docs/p2p-connection.md b/docs/p2p-connection.md index 84ff5ef6c6..17563e6292 100644 --- a/docs/p2p-connection.md +++ b/docs/p2p-connection.md @@ -11,9 +11,28 @@ aea create my_genesis_aea cd my_genesis_aea aea add connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea build +``` + +Establish the proof of representation: + +``` bash +aea generate-key fetchai +aea add-key fetchai fetchai_private_key.txt +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +aea issue-certificates +``` + +Run the AEA: + +``` bash aea run --connections fetchai/p2p_libp2p:0.13.0 ``` +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.13.0 -u public_uri` to retrieve the address.) +This is the entry peer address for the local agent communication network created by the genesis AEA. + ### Create and run another AEA Create a second AEA: @@ -23,6 +42,17 @@ aea create my_other_aea cd my_other_aea aea add connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea build +``` + +Establish the proof of representation: + +``` bash +aea generate-key fetchai +aea add-key fetchai fetchai_private_key.txt +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +aea issue-certificates ``` Provide the AEA with the information it needs to find the genesis: @@ -31,13 +61,13 @@ Provide the AEA with the information it needs to find the genesis: aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ '{ "delegate_uri": "127.0.0.1:11001", - "entry_peers": MULTI_ADDRESSES, + "entry_peers": ["SOME_ADDRESS"], "local_uri": "127.0.0.1:9001", "log_file": "libp2p_node.log", "public_uri": "127.0.0.1:9001" }' ``` -Here `MULTI_ADDRESSES` needs to be replaced with the list of multi addresses displayed in the log output of the genesis AEA. +Here `SOME_ADDRESS` needs to be replaced with the list of multi addresses displayed in the log output of the genesis AEA. Run the AEA: diff --git a/docs/por.md b/docs/por.md index 4824f021f1..1c54936596 100644 --- a/docs/por.md +++ b/docs/por.md @@ -1,4 +1,20 @@ +An AEA can use several key pairs. In particular, it can use different keys for securing its communication and for engaging in exchange. In the ACN we make use of this fact. To be able to signal to other agents that the address derived from one key pair is allowed to represent the agent controlling the other key pair, the key pair which is being represented must sign a message to prove that the other key pair is allowed to represent it. The `aea issue-certificates` command allows to create this association. -An AEA can use several key pairs. In particular, it can use different keys for securing its communication and for engaging in exchange. In the ACN we make use of this fact. To be able to signal to other agents that one key pair is allowed to represent another, the key pair which is being represented must sign a message to prove that the other key pair is allowed to represent it. +The proof of representation feature is used in the context of the `fetchai/p2p_libp2p` and `fetchai/p2p_libp2p_client` connection. +In the former connection, the config yaml specifies a `cert_requests` field: + +``` yaml +cert_requests: +- identifier: acn + ledger_id: fetchai + not_after: '2022-01-01' + not_before: '2021-01-01' + public_key: fetchai + save_path: .certs/conn_cert.txt +``` + +The `identifier` refers to the environment for which the signature is generated, here `acn`. The `ledger_id` refers to the key pair to be used from the `private_key_paths` specified in `aea-config.yaml` for signing. The `not_after` and `not_before` fields specify constraints on the validity of the signature. The `public_key` can specify either the identifier of the key-pair in `connection_private_key_paths` of which the public key is signed or it can contain the to be signed public key in plain text. The `save_path` specifies the path where the certificate is to be saved at. + +In the above example, the connection requests a certificate which is a signature of the `fetchai` public key in `connection_private_key_paths` with the `fetchai` key pai in `private_key_paths`. The validity of the signature will be constrained to the year `2021` for the environment `acn`. diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-p2p-connection.md b/tests/test_docs/test_bash_yaml/md_files/bash-p2p-connection.md index 06990f1474..abdb7d8f06 100644 --- a/tests/test_docs/test_bash_yaml/md_files/bash-p2p-connection.md +++ b/tests/test_docs/test_bash_yaml/md_files/bash-p2p-connection.md @@ -3,6 +3,16 @@ aea create my_genesis_aea cd my_genesis_aea aea add connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea build +``` +``` bash +aea generate-key fetchai +aea add-key fetchai fetchai_private_key.txt +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +aea issue-certificates +``` +``` bash aea run --connections fetchai/p2p_libp2p:0.13.0 ``` ``` bash @@ -10,12 +20,20 @@ aea create my_other_aea cd my_other_aea aea add connection fetchai/p2p_libp2p:0.13.0 aea config set agent.default_connection fetchai/p2p_libp2p:0.13.0 +aea build +``` +``` bash +aea generate-key fetchai +aea add-key fetchai fetchai_private_key.txt +aea generate-key fetchai fetchai_connection_private_key.txt +aea add-key fetchai fetchai_connection_private_key.txt --connection +aea issue-certificates ``` ``` bash aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config \ '{ "delegate_uri": "127.0.0.1:11001", - "entry_peers": MULTI_ADDRESSES, + "entry_peers": ["SOME_ADDRESS"], "local_uri": "127.0.0.1:9001", "log_file": "libp2p_node.log", "public_uri": "127.0.0.1:9001" diff --git a/tests/test_docs/test_bash_yaml/md_files/bash-por.md b/tests/test_docs/test_bash_yaml/md_files/bash-por.md new file mode 100644 index 0000000000..29cfef1f91 --- /dev/null +++ b/tests/test_docs/test_bash_yaml/md_files/bash-por.md @@ -0,0 +1,9 @@ +``` yaml +cert_requests: +- identifier: acn + ledger_id: fetchai + not_after: '2022-01-01' + not_before: '2021-01-01' + public_key: fetchai + save_path: .certs/conn_cert.txt +``` \ No newline at end of file From d19ad8711ca4fb83da93d8bcb115b68956afe216 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Mon, 4 Jan 2021 15:50:29 +0300 Subject: [PATCH 145/204] support vendorized packages for local push --- aea/cli/push.py | 22 ++++++++++++++++------ tests/test_cli/test_push.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/aea/cli/push.py b/aea/cli/push.py index a6d6bfc662..7b8c2d19ac 100644 --- a/aea/cli/push.py +++ b/aea/cli/push.py @@ -16,13 +16,13 @@ # limitations under the License. # # ------------------------------------------------------------------------------ - """Implementation of the 'aea push' subcommand.""" - +import os from shutil import copytree from typing import cast import click +from click.exceptions import ClickException from aea.cli.registry.push import check_package_public_id, push_item from aea.cli.utils.click_utils import PublicIdParameter @@ -99,10 +99,20 @@ def _save_item_locally(ctx: Context, item_type: str, item_id: PublicId) -> None: :return: None """ item_type_plural = item_type + "s" - - source_path = try_get_item_source_path( - ctx.cwd, None, item_type_plural, item_id.name - ) + try: + # try non vendor first + source_path = try_get_item_source_path( + ctx.cwd, None, item_type_plural, item_id.name + ) + except ClickException: + # failed on user's packages + # try vendors + source_path = try_get_item_source_path( + os.path.join(ctx.cwd, "vendor"), + item_id.author, + item_type_plural, + item_id.name, + ) check_package_public_id(source_path, item_type, item_id) diff --git a/tests/test_cli/test_push.py b/tests/test_cli/test_push.py index f34462e789..ae46a29c52 100644 --- a/tests/test_cli/test_push.py +++ b/tests/test_cli/test_push.py @@ -30,6 +30,8 @@ from aea.test_tools.constants import DEFAULT_AUTHOR from aea.test_tools.test_cases import AEATestCaseEmpty +from packages.fetchai.skills.echo import PUBLIC_ID + from tests.conftest import AUTHOR, CLI_LOG_OPTION, CliRunner from tests.test_cli.tools_for_testing import ContextMock, PublicIdMock @@ -68,6 +70,36 @@ def test_save_item_locally_positive( copy_tree_mock.assert_called_once_with("source", "target") +@mock.patch("aea.cli.push.copytree") +class TestPushLocally(AEATestCaseEmpty): + """Test case for clu push --local.""" + + ITEM_PUBLIC_ID = PUBLIC_ID + ITEM_TYPE = "skill" + + @classmethod + def setup_class(cls): + """Set up test case.""" + super(TestPushLocally, cls).setup_class() + cls.add_item(cls.ITEM_TYPE, str(cls.ITEM_PUBLIC_ID), local=True) + + def test_vendor_ok( + self, copy_tree_mock, + ): + """Test ok for vendor's item.""" + self.invoke("push", "--local", "skill", "fetchai/echo") + copy_tree_mock.assert_called_once() + + def test_fail_no_item( + self, *mocks, + ): + """Test fail, item_noit_exists .""" + with pytest.raises( + ClickException, match=f'Item "not_exists" not found in source folder.' + ): + self.invoke("push", "--local", "skill", "fetchai/not_exists") + + @mock.patch( "aea.cli.registry.push.load_yaml", return_value={"author": AUTHOR, "name": "name", "version": "0.1.0"}, From cb5016a3df606a31ef0180a09880cf0babe649a3 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 4 Jan 2021 13:58:55 +0100 Subject: [PATCH 146/204] add test for 'cli-commands.md' --- tests/test_docs/test_cli_commands.py | 71 ++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 tests/test_docs/test_cli_commands.py diff --git a/tests/test_docs/test_cli_commands.py b/tests/test_docs/test_cli_commands.py new file mode 100644 index 0000000000..f38c9bd631 --- /dev/null +++ b/tests/test_docs/test_cli_commands.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2018-2019 Fetch.AI Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ + +"""This module contains the tests for the content of cli-commands.md file.""" +import pprint +import re +from pathlib import Path + +from aea.cli import cli + +from tests.conftest import ROOT_DIR +from tests.test_docs.helper import BaseTestMarkdownDocs + + +IGNORE_MATCHES = ["`-v DEBUG run`", "`config set [path] [--type TYPE]`"] + + +class TestCliCommands(BaseTestMarkdownDocs): + """Test cli-commands.md documentation.""" + + DOC_PATH = Path(ROOT_DIR, "docs", "cli-commands.md") + + def test_cli_commands(self): + """Test CLI commands.""" + commands_raw = re.compile(r"\\| `.*` +\\|").findall(self.doc_content) + commands_raw = [ + re.compile("`([A-Za-z0-9\\-]+) ?.*`").search(s) for s in commands_raw + ] + commands_raw = list( + filter(lambda x: x.group(0) not in IGNORE_MATCHES, commands_raw) + ) + actual_commands = list(map(lambda match: match.group(1), commands_raw)) + + actual_commands_set = set(actual_commands) + expected_commands = set(cli.commands.keys()) + + # test no duplicates + assert len(actual_commands) == len( + actual_commands_set + ), "Found duplicate commands in the documentation." + + # test that there is no missing command + missing = expected_commands.difference(actual_commands) + assert ( + len(missing) == 0 + ), f"Missing the following commands: {pprint.pformat(missing)}" + + # test that there are no more commands + more = actual_commands_set.difference(expected_commands) + assert len(more) == 0, f"There are unknown commands: {pprint.pformat(missing)}" + + # test that they are in the same order. + actual = actual_commands + expected = sorted(expected_commands) + assert actual == expected, "Commands are not in alphabetical order." From feb56720a99bbc7c6fcdf16a0bf2b966441da0ef Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 4 Jan 2021 14:12:01 +0100 Subject: [PATCH 147/204] fix makefile 'new_env' command. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 49feadb09c..539999a0bc 100644 --- a/Makefile +++ b/Makefile @@ -118,11 +118,11 @@ release: echo "Please change to master branch for release.";\ fi -v := $(shell pip -V | grep -r virtualenvs) +v := $(shell pip -V | grep virtualenvs) .PHONY: new_env new_env: clean - if [ "$v" == "" ];\ + if [ -z $v ];\ then\ pipenv --rm;\ pipenv --python 3.7;\ From 5f7862dd5981616aed256b7db9222eb7d9970927 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 4 Jan 2021 14:16:16 +0100 Subject: [PATCH 148/204] another minor fix --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 539999a0bc..9b677fc147 100644 --- a/Makefile +++ b/Makefile @@ -122,7 +122,7 @@ v := $(shell pip -V | grep virtualenvs) .PHONY: new_env new_env: clean - if [ -z $v ];\ + if [ -z "$v" ];\ then\ pipenv --rm;\ pipenv --python 3.7;\ From d4d2986e6badf7d88cf93c182369622cfc1dbf29 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Mon, 4 Jan 2021 13:58:14 +0000 Subject: [PATCH 149/204] respond to PR comments, add modes description --- Makefile | 7 ++++--- README.md | 22 +++++++++++++--------- docs/modes.md | 4 +++- mkdocs.yml | 2 +- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 9b677fc147..7f303f26db 100644 --- a/Makefile +++ b/Makefile @@ -59,15 +59,16 @@ security: static: mypy aea benchmark examples packages scripts tests -.PHONY: misc_checks -misc_checks: +.PHONY: package_checks +package_checks: # pip install '.[all]' # python scripts/freeze_dependencies.py -o requirements.txt # liccheck -s strategy.ini -r requirements.txt -l PARANOID # rm -fr requirements.txt - python scripts/check_copyright_notice.py + # python scripts/check_copyright_notice.py python scripts/generate_ipfs_hashes.py --check python scripts/check_package_versions_in_docs.py + python scripts/check_package_dependencies.py .PHONY: docs docs: diff --git a/README.md b/README.md index 9db4945912..3a519c1eb0 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,10 @@ This repository contains submodules. Clone with recursive strategy: git clone https://github.com/fetchai/agents-aea.git --recursive && cd agents-aea +- To fetch/update submodules (for existing local repo): + + git submodule sync --recursive && git submodule update --init --recursive + ### Dependencies All python specific framework dependencies are specified in `setup.py` and installed with the framework. All development dependencies are specified in `Pipfile` (and installed via the commands specified in [Preliminaries](#preliminaries)). @@ -127,7 +131,7 @@ The following dependency is **only relevant if you intend to contribute** to the The following steps are **only relevant if you intend to contribute** to the repository. They are **not required** for agent development. -### Recommended +### Recommended commands - The simplest way to get setup for development on the framework is to run the following: @@ -142,6 +146,10 @@ The following steps are **only relevant if you intend to contribute** to the rep make pylint make security +- For checking packages integrity: + + make package_checks + - For testing aea.{SUBMODULE} with tests/test_{TESTMODULE} use: make dir={SUBMODULE} tdir={TESTMODULE} test-sub @@ -150,9 +158,7 @@ The following steps are **only relevant if you intend to contribute** to the rep make dir=cli tdir=cli test-sub -- To run demos against local packages use flag `--local` in `aea` CLI commands. - -### Alternative +### Alternative commands - To install development dependencies manually (here optionally skipping `Pipfile.lock` creation): @@ -176,13 +182,11 @@ The following steps are **only relevant if you intend to contribute** to the rep - To run security checks: `tox -e bandit` and `tox -e safety` or `make security` -- To start a live-reloading docs server on localhost: `mkdocs serve` - -- To amend the docs, create a new documentation file in `docs/` and add a reference to it in `mkdocs.yml`. +### Documentation -- To fetch/update submodules: +- To start a live-reloading docs server on localhost: `mkdocs serve`. To amend the docs, create a new documentation file in `docs/` and add a reference to it in `mkdocs.yml`. - git submodule sync --recursive && git submodule update --init --recursive +- To run demos against local packages use flag `--local` in `aea` CLI commands. ## Cite diff --git a/docs/modes.md b/docs/modes.md index 2ac0f92bdd..2632eb521e 100644 --- a/docs/modes.md +++ b/docs/modes.md @@ -1,2 +1,4 @@ -We can run an AEA in multiple modes +We can run an AEA in multiple modes thanks to the configurable design of the framework. + +The AEA contains two runnable parts, the AgentLoop, which operates the skills, and the Multiplexer, which operates the connections. The AgentLoop can be configured to run in `async` or `sync` mode. The Multiplexer by default runs in `async` mode. The AEA itself, can be configured to run in `async` mode, if both the Multiplexer and AgentLoop have the same mode, or in `threaded` mode. The latter ensures that AgentLoop and Multiplexer are run in separate threads. diff --git a/mkdocs.yml b/mkdocs.yml index 43dca9d169..827ced1b04 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -45,10 +45,10 @@ nav: - Core components - Part 2: 'core-components-2.md' - Interaction protocols: 'interaction-protocol.md' - Trade between two AEAs: 'generic-skills-step-by-step.md' + - Development setup: 'development-setup.md' - Development - Advanced: - Topic guides: - Ways to build an AEA: 'step-one.md' - - Development setup: 'development-setup.md' - Build an AEA with the CLI: 'build-aea-step-by-step.md' - Scaffolding packages: 'scaffolding.md' - Generating protocols: 'protocol-generator.md' From 90d6071250c166b15f00fe9c0be4665184375125 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Mon, 4 Jan 2021 14:05:24 +0000 Subject: [PATCH 150/204] remove comments from Makefile --- Makefile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Makefile b/Makefile index 7f303f26db..182661ce42 100644 --- a/Makefile +++ b/Makefile @@ -61,11 +61,6 @@ static: .PHONY: package_checks package_checks: - # pip install '.[all]' - # python scripts/freeze_dependencies.py -o requirements.txt - # liccheck -s strategy.ini -r requirements.txt -l PARANOID - # rm -fr requirements.txt - # python scripts/check_copyright_notice.py python scripts/generate_ipfs_hashes.py --check python scripts/check_package_versions_in_docs.py python scripts/check_package_dependencies.py From 517a2ee14a2839da088b59bba992cd04bb40e386 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Mon, 4 Jan 2021 14:23:31 +0000 Subject: [PATCH 151/204] address PR review to simplify make commands --- Makefile | 11 ++++------- README.md | 1 - 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 182661ce42..f6a86d79d9 100644 --- a/Makefile +++ b/Makefile @@ -122,13 +122,10 @@ new_env: clean then\ pipenv --rm;\ pipenv --python 3.7;\ - echo "Enter clean virtual environment now: 'pipenv shell'.";\ + pipenv install --dev --skip-lock;\ + pipenv run pip uninstall typing -y;\ + pipenv run pip install -e .[all];\ + echo "Enter virtual environment with all development dependencies now: 'pipenv shell'.";\ else\ echo "In a virtual environment! Exit first: 'exit'.";\ fi - -.PHONY: install_env -install_env: - pipenv install --dev --skip-lock - pip uninstall typing -y - pip install -e .[all] diff --git a/README.md b/README.md index 3a519c1eb0..51b2b4f1c0 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,6 @@ The following steps are **only relevant if you intend to contribute** to the rep make new_env pipenv shell - make install_env - For linting and static analysis use: From 69b900563cbe121815efdc2318d1d3ab7b1c663e Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 4 Jan 2021 15:40:19 +0100 Subject: [PATCH 152/204] fix test 'cli-commands.md' --- docs/cli-commands.md | 15 ++++++++++----- tests/test_docs/test_cli_commands.py | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/cli-commands.md b/docs/cli-commands.md index 64dac6a086..4e326f3ab6 100644 --- a/docs/cli-commands.md +++ b/docs/cli-commands.md @@ -5,36 +5,41 @@ | `add [package_type] [public_id]` | Add a `package_type` connection, contract, protocol, or skill, with `[public_id]`, to the AEA. `add --local` to add from local `packages` directory. | | `add-key [ledger_id] file [--connection]` | Add a private key from a file for `ledger_id`. | | `build` | Build the agent and its components. | -| `create [name]` | Create a new aea project called `name`. | | `config get [path]` | Reads the config specified in `path` and prints its target. | | `config set [path] [--type TYPE]` | Sets a new value for the target of the `path`. Optionally cast to type. | +| `create [name]` | Create a new aea project called `name`. | | `delete [name]` | Delete an aea project. See below for disabling a resource. | | `eject [package_type] [public_id]` | Move a package of `package_type` and `package_id` from vendor to project working directory. | | `fetch [public_id]` | Fetch an aea project with `public_id`. `fetch --local` to fetch from local `packages` directory. | | `fingerprint [package_type] [public_id]` | Fingerprint connection, contract, protocol, or skill, with `public_id`. | | `freeze` | Get all the dependencies needed for the aea project and its components. | -| `gui` | Run the GUI. | | `generate protocol [protocol_spec_path]` | Generate a protocol from the specification. | | `generate-key [ledger_id]` | Generate private keys. The AEA uses a private key to derive the associated public key and address. | | `generate-wealth [ledger_id]` | Generate wealth for address on test network. | | `get-address [ledger_id]` | Get the address associated with the private key. | -| `get-multiaddress [ledger_id]`... | Get the multiaddress associated with a private key or connection. | +| `get-multiaddress [ledger_id]...` | Get the multiaddress associated with a private key or connection. | | `get-wealth [ledger_id]` | Get the wealth associated with the private key. | -| `install [-r ]` | Install the dependencies. (With `--install-deps` to install dependencies.) | +| `gui` | Run the GUI. | | `init` | Initialize your AEA configurations. (With `--author` to define author.) | +| `install [-r ]` | Install the dependencies. (With `--install-deps` to install dependencies.) | | `interact` | Interact with a running AEA via the stub connection. | | `issue-certificates` | Issue the connection certificates. | | `launch [path_to_agent_project]...` | Launch many agents at the same time. | | `list [package_type]` | List the installed resources. | +| `local-registry-sync` | Upgrade the local package registry. | | `login USERNAME [--password password]` | Login to a registry account with credentials. | | `logout` | Logout from registry account. | | `publish` | Publish the AEA to registry. Needs to be executed from an AEA project.`publish --local` to publish to local `packages` directory. | | `push [package_type] [public_id]` | Push connection, protocol, or skill with `public_id` to registry. `push --local` to push to local `packages` directory. | +| `register ...` | Create a new registry account. | `remove [package_type] [name]` | Remove connection, protocol, or skill, called `name`, from AEA. | | `remove-key [ledger_id] [name]` | Remove a private key registered with id `ledger_id`. | +| `reset_password EMAIL` | Reset the password of the registry account. | | `run {using [connections, ...]}` | Run the AEA on the Fetch.ai network with default or specified connections. | -| `search [package_type]` | Search for components in the registry. `search --local [package_type] [--query searching_query]` to search in local `packages` directory. | | `scaffold [package_type] [name]` | Scaffold a new connection, protocol, or skill called `name`. | +| `search [package_type]` | Search for components in the registry. `search --local [package_type] [--query searching_query]` to search in local `packages` directory. | +| `transfer ...` | Transfer wealth associated with a private key of the agent to another account. | +| `upgrade ...` | Upgrade the packages of the agent. | | `-v DEBUG run` | Run with debugging. |