remove warning when tss instance had already released

Author: roxie-23

Makefile

@@ -113,10 +113,17 @@ mod:
 linter:
 	golangci-lint run
 
-## make cluster: Run cluster including 4 nodes
+## make cluster: Run cluster including 4 nodes(including compile and clean storage)
 cluster:install${TAGS}
 	@cd scripts && bash cluster.sh TAGS=${TAGS}
 
+## make cluster-restart: Restart cluster including 4 nodes without compile and clean storage
+cluster-restart:
+	@cd scripts && bash cluster_restart.sh TAGS=${TAGS}
+
+## clean nodes storage
+clean-storage:
+	@cd scripts && bash clean_storage.sh
 ## make solo: Run one node in solo mode
 solo:install${TAGS}
 	@cd scripts && bash solo.sh TAGS=${TAGS}

api/grpc/sign.go

@@ -102,11 +102,25 @@ func (cbs *ChainBrokerService) GetTssSigns(ctx context.Context, req *pb.GetSigns
 		go func() {
 			defer wg.Done()
 			result = cbs.getRemoteTssSign(tssReq, poolPk)
-			cbs.logger.WithFields(logrus.Fields{"result": result}).Info("get remote tss sign")
+			cbs.logger.WithFields(logrus.Fields{"content": tssReq.Content}).Info("get remote tss sign success")
 		}()
 
-		// 7. get sign by ourself
-		culprits, sign, keysignErr := cbs.getLocalTssSign(tssFlag, tssReq, tssSigners)
+		// 7. get sign by ourselves, if this node is contained in tssSigners
+		var (
+			culprits []string
+			sign     []byte
+			signErr  error
+		)
+		if utils.IsContained(fmt.Sprintf("%d", cbs.api.Network().LocalPeerID()), tssSigners) {
+			culprits, sign, signErr = cbs.getLocalTssSign(tssFlag, tssReq, tssSigners)
+		} else {
+			// if not contained in tssSigners, ignore tss msg
+			cbs.logger.Debugf("notifyNotParties in local %d", cbs.api.Network().LocalPeerID())
+			err = cbs.notifyTssNotParties(tssFlag, tssReq, tssSigners)
+			if err != nil {
+				cbs.logger.Errorf("notifyNotParties in local %d err: %s", cbs.api.Network().LocalPeerID(), err)
+			}
+		}
 		wg.Wait()
 
 		// 8. get a verified signature from others, return
@@ -118,7 +132,7 @@ func (cbs *ChainBrokerService) GetTssSigns(ctx context.Context, req *pb.GetSigns
 			}, nil
 		}
 		// 9. get a signature by myself, return
-		if tssFlag && keysignErr == nil {
+		if tssFlag && signErr == nil {
 			// 是tss节点
 			return &pb.SignResponse{
 				Sign: map[string][]byte{
@@ -246,8 +260,10 @@ func (cbs *ChainBrokerService) randomSignerRequest(signersALL []string) ([]strin
 	}
 	cbs.logger.Infof("====================== tss all signers: %s, signers: %s", strings.Join(signersALL, ","),
 		strings.Join(tssSigners, ","))
+	notPartiesIDs := utils.FindUnmatched(tssSigners, signersALL)
+
 	randomN := rand.New(rand.NewSource(time.Now().UnixNano())).Int()
-	extra := []byte(fmt.Sprintf("%s-%s", strings.Join(tssSigners, ","), strconv.Itoa(randomN)))
+	extra := []byte(fmt.Sprintf("%s-%s-%s", strings.Join(tssSigners, ","), strconv.Itoa(randomN), strings.Join(notPartiesIDs, ",")))
 	return tssSigners, extra
 }
 
@@ -340,3 +356,11 @@ func getConsensusTssInfoParties(infos []*pb.TssInfo, quorum uint64) ([]string, [
 
 	return idsAddr, []byte(pk), nil
 }
+
+func (cbs *ChainBrokerService) notifyTssNotParties(tssFlag bool, tssReq *pb.GetSignsRequest, singers []string) error {
+	if tssFlag {
+		err := cbs.api.Broker().SetTssNotParties(tssReq, singers)
+		return err
+	}
+	return nil
+}

go.mod

@@ -33,9 +33,9 @@ require (
 	github.com/libp2p/go-libp2p-swarm v0.2.4
 	github.com/looplab/fsm v0.2.0
 	github.com/magiconair/properties v1.8.5
-	github.com/meshplus/bitxhub-core v1.28.0
+	github.com/meshplus/bitxhub-core v1.28.1-0.20230411032641-11245b4adfc5
 	github.com/meshplus/bitxhub-kit v1.28.0
-	github.com/meshplus/bitxhub-model v1.28.0
+	github.com/meshplus/bitxhub-model v1.28.1-0.20230411032618-24ca54eec606
 	github.com/meshplus/consensus v0.0.0-20211228075008-5f469b198531
 	github.com/meshplus/eth-kit v1.28.0
 	github.com/meshplus/go-libp2p-cert v1.28.0

go.sum

@@ -544,6 +544,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
 github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
+github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw=
+github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
 github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4=
 github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU=
 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
@@ -803,17 +805,18 @@ github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO
 github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/meshplus/bitxhub-core v1.28.0 h1:QIp5aM7EYvxS+ORE2TLTeBRv7uUoXvyNupULA2N8A6s=
-github.com/meshplus/bitxhub-core v1.28.0/go.mod h1:uqQTX6LOoVVyTmJmW5fVJ64qcse6IyXrtmn2jHGDx4w=
+github.com/meshplus/bitxhub-core v1.28.1-0.20230411032641-11245b4adfc5 h1:J3xQCuBO4q1Nnax6a/hDRm10mcx2zp9scffH6tG5a5Q=
+github.com/meshplus/bitxhub-core v1.28.1-0.20230411032641-11245b4adfc5/go.mod h1:4/95iA54o3TGOH872VGxEEbs8eO195kjakTrlpPcJkY=
 github.com/meshplus/bitxhub-kit v1.0.0-rc1/go.mod h1:ra/AhOkPvpElI+wXrB9G6DjdcrdxFU3vMwA5MYKr9D0=
 github.com/meshplus/bitxhub-kit v1.0.0/go.mod h1:7cWyhXWZfrQ3+EaxkRoXfuiG3Y5R9DXYJomeZKkETW8=
 github.com/meshplus/bitxhub-kit v1.1.1/go.mod h1:r4l4iqn0RPJreb/OmoYKfjCjQJrXpZX++6Qc31VG/1k=
 github.com/meshplus/bitxhub-kit v1.2.1-0.20210902085548-07f4fa85bfc9/go.mod h1:wrEdhHp1tktzdwcWb4bOxYsVc+KkcrYL18IYWYeumPQ=
 github.com/meshplus/bitxhub-kit v1.28.0 h1:HNYlx6FzELVWqr0GjtWaNo9iMwQTb94bdsqLJo0U2LE=
 github.com/meshplus/bitxhub-kit v1.28.0/go.mod h1:S8qKlxfXRFS9HQgXyD4r+o1/0u6EJOQoMZyXtlEM8vM=
 github.com/meshplus/bitxhub-model v1.0.0-rc3/go.mod h1:ZCctQIYTlE3vJ8Lhkrgs9bWwNA+Dw4JzojOSIzLVU6E=
-github.com/meshplus/bitxhub-model v1.28.0 h1:UpZC7IHaZeZHG4IUyKwqOvntx12QmceoyApRIexo5ow=
 github.com/meshplus/bitxhub-model v1.28.0/go.mod h1:EehJ/neXJcXy0o1jglfdD1/vsU7tsXveNr9pGZTDtrY=
+github.com/meshplus/bitxhub-model v1.28.1-0.20230411032618-24ca54eec606 h1:HI9WHfsIaeR4HkoqsQ3ZX2uoIqtP9tSJpF96Y0eU/HU=
+github.com/meshplus/bitxhub-model v1.28.1-0.20230411032618-24ca54eec606/go.mod h1:EehJ/neXJcXy0o1jglfdD1/vsU7tsXveNr9pGZTDtrY=
 github.com/meshplus/consensus v0.0.0-20211228075008-5f469b198531 h1:lU2XI7vWdapTLJfObY3A+e+a3rvHbnT0pHN9LEghwRk=
 github.com/meshplus/consensus v0.0.0-20211228075008-5f469b198531/go.mod h1:2Sv1u1sOFta9dAO6OuHyxLaN1Z6/AdrIol99qi5ZZ0k=
 github.com/meshplus/eth-kit v1.28.0 h1:WDvIhbE5YbNoAgoQ4Etr/eE7DfsocKodFzyaqrbWkzk=

internal/app/feedhub.go

@@ -4,8 +4,11 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
+	"strings"
 	"time"
 
+	"github.com/meshplus/bitxhub/pkg/tssmgr"
+
 	"github.com/Rican7/retry"
 	"github.com/Rican7/retry/strategy"
 	"github.com/meshplus/bitxhub-core/governance"
@@ -94,17 +97,33 @@ func (bxh *BitXHub) listenEvent() {
 					bxh.TssMgr.SetOrderReadyPeers(id)
 					return
 				default:
-					if bxh.TssMgr.GetTssStatus() {
-						bxh.logger.Debugf("keygen task is done")
-						return
-					}
-					bxh.logger.Debugf("get tss msg to put")
+					bxh.logger.Debug("get tss msg to put")
 					wireMsg := &message.WireMessage{}
 					if err := json.Unmarshal(msg.Data, wireMsg); err != nil {
 						bxh.logger.Errorf(fmt.Sprintf("unmarshal wire msg error: %v", err))
-					} else {
-						bxh.TssMgr.PutTssMsg(msg, wireMsg.MsgID)
+						return
+					}
+					// check if msgID tss round done
+					done, err := bxh.TssMgr.IsTssRoundDone(wireMsg.MsgID)
+					if err != nil {
+						if strings.Contains(err.Error(), tssmgr.GetInfoErr) {
+							bxh.logger.WithFields(logrus.Fields{"done": done, "err": err}).Warning("check tss round done")
+							return
+						}
+					}
+					if done {
+						bxh.logger.Debugf("tss round done msgID:%s", wireMsg.MsgID)
+						return
+					}
+
+					if wireMsg.MsgType == message.TSSTaskDone {
+						//bxh.logger.Errorf("==========start set tss round done")
+						err := bxh.TssMgr.SetTssRoundDone(wireMsg.MsgID, false)
+						if err != nil {
+							bxh.logger.Errorf("set tss round done err:%s", err)
+						}
 					}
+					bxh.TssMgr.PutTssMsg(msg, wireMsg.MsgID)
 				}
 			}()
 		case ev := <-blockCh:

internal/coreapi/api/api.go

@@ -51,6 +51,8 @@ type BrokerAPI interface {
 
 	FetchSignsFromOtherPeers(req *pb.GetSignsRequest) map[string][]byte
 	FetchTssInfoFromOtherPeers() []*pb.TssInfo
+	SetTssNotParties(tssReq *pb.GetSignsRequest, singers []string) error
+
 	GetSign(req *pb.GetSignsRequest, signers []string) (string, []byte, []string, error)
 	GetBlockHeaders(start uint64, end uint64) ([]*pb.BlockHeader, error)
 	GetQuorum() uint64
@@ -63,6 +65,7 @@ type NetworkAPI interface {
 	PeerInfo() ([]byte, error)
 	PierManager() peermgr.PierManager
 	OtherPeers() map[uint64]*peer.AddrInfo
+	LocalPeerID() uint64
 }
 
 type ChainAPI interface {

internal/coreapi/api/mock_api/mock_api.go

@@ -170,6 +170,17 @@ func (b *BrokerAPI) OrderReady() error {
 	return b.bxh.Order.Ready()
 }
 
+func (b *BrokerAPI) convertTssID(str string) []uint64 {
+	peers := make([]uint64, 0)
+	for _, id := range strings.Split(str, ",") {
+		idInt, _ := strconv.ParseUint(id, 10, 64)
+		if _, ok := b.bxh.PeerMgr.OtherPeers()[idInt]; ok {
+			peers = append(peers, idInt)
+		}
+	}
+	return peers
+}
+
 func (b *BrokerAPI) FetchSignsFromOtherPeers(req *pb.GetSignsRequest) map[string][]byte {
 	// if type is solo, needn't fetch sign from others
 	if b.bxh.GetSoloType() {
@@ -180,17 +191,28 @@ func (b *BrokerAPI) FetchSignsFromOtherPeers(req *pb.GetSignsRequest) map[string
 		wg     = sync.WaitGroup{}
 		lock   = sync.Mutex{}
 	)
-
-	signerStr := strings.Split(string(req.Extra), "-")[0]
-	signers := []uint64{}
-	for _, id := range strings.Split(signerStr, ",") {
-		idInt, _ := strconv.ParseUint(id, 10, 64)
-		if _, ok := b.bxh.PeerMgr.OtherPeers()[idInt]; ok {
-			signers = append(signers, idInt)
+	if req.Type == pb.GetSignsRequest_TSS_IBTP_REQUEST || req.Type == pb.GetSignsRequest_TSS_IBTP_RESPONSE {
+		// req.Extra: <tssSigners>-<random>-<notParties>
+		notParties := b.convertTssID(strings.Split(string(req.Extra), "-")[2])
+		wg.Add(len(notParties))
+		for _, pid := range notParties {
+			go func(pid uint64, wg *sync.WaitGroup) {
+				defer wg.Done()
+				err := b.notifyPeerNotParties(pid, req)
+				if err != nil {
+					b.logger.WithFields(logrus.Fields{
+						"pid": pid,
+						"err": err.Error(),
+					}).Warnf("notify other peers not belong to parties with error")
+				}
+			}(pid, &wg)
 		}
+		wg.Wait()
 	}
 
-	// TODO: calculate threshold
+	// tss req.Extra: <tssSigners>-<random>-<notParties>
+	// multi req.Extra: <multiSigners>
+	signers := b.convertTssID(strings.Split(string(req.Extra), "-")[0])
 	wg.Add(len(signers))
 	for _, pid := range signers {
 		go func(pid uint64, result map[string][]byte, wg *sync.WaitGroup, lock *sync.Mutex) {
@@ -259,6 +281,28 @@ func (b *BrokerAPI) requestIBTPSignPeer(pid uint64, id string, typ pb.GetSignsRe
 	return data.Address, data.Signature, nil
 }
 
+func (b *BrokerAPI) notifyPeerNotParties(pid uint64, req *pb.GetSignsRequest) error {
+	keysignReqData, err := req.Marshal()
+	if err != nil {
+		return fmt.Errorf("GetSignsRequest marshal error: %w", err)
+	}
+
+	msg := pb.Message{
+		Type: pb.Message_Tss_KEYSIGN_NOT_PARTIES,
+		Data: keysignReqData,
+	}
+
+	resp, err := b.bxh.PeerMgr.Send(pid, &msg)
+	if err != nil {
+		return fmt.Errorf("send message to %d failed: %w", pid, err)
+	}
+
+	if string(resp.Data) != "ok" {
+		return fmt.Errorf("remote node: %d to handle tss not partied failed", pid)
+	}
+	return nil
+}
+
 func (b *BrokerAPI) requestIBTPTssSignPeer(pid uint64, req *pb.GetSignsRequest) error {
 	keysignReqData, err := req.Marshal()
 	if err != nil {
@@ -339,15 +383,17 @@ func (b *BrokerAPI) GetSign(req *pb.GetSignsRequest, signers []string) (string,
 	case pb.GetSignsRequest_MULTI_IBTP_RESPONSE:
 		addr, signData, err = utils.GetIBTPSign(b.bxh.Ledger, req.Content, false, b.bxh.GetPrivKey().PrivKey)
 	case pb.GetSignsRequest_TSS_IBTP_REQUEST:
+		// req.Extra: <tssSigners>-<random>-<notParties>
 		signInfo := strings.Split(string(req.Extra), "-")
-		if len(signInfo) != 2 {
+		if len(signInfo) != 3 {
 			err = fmt.Errorf("wrong tss req extra: %s", string(req.Extra))
 		} else {
 			signData, culpritIDs, err = utils.GetIBTPTssSign(b.bxh.TssMgr, b.bxh.Ledger, req.Content, true, signers, signInfo[1])
 		}
 	case pb.GetSignsRequest_TSS_IBTP_RESPONSE:
+		// req.Extra: <tssSigners>-<random>-<notParties>
 		signInfo := strings.Split(string(req.Extra), "-")
-		if len(signInfo) != 2 {
+		if len(signInfo) != 3 {
 			err = fmt.Errorf("wrong tss req extra: %s", string(req.Extra))
 		} else {
 			signData, culpritIDs, err = utils.GetIBTPTssSign(b.bxh.TssMgr, b.bxh.Ledger, req.Content, false, signers, signInfo[1])
@@ -359,7 +405,7 @@ func (b *BrokerAPI) GetSign(req *pb.GetSignsRequest, signers []string) (string,
 		}
 		signData, err = b.bxh.Ledger.GetBlockSign(height)
 		if err != nil {
-			return "", nil, nil, fmt.Errorf("get block header sign: %w", err)
+			return "", nil, nil, fmt.Errorf("get block sign: %w", err)
 		}
 	case pb.GetSignsRequest_MULTI_BURN:
 		addr, signData, err = b.handleMultiSignsBurnReq(req.Content)
@@ -445,7 +491,7 @@ func (b *BrokerAPI) GetPendingNonceByAccount(account string) uint64 {
 	return b.bxh.Order.GetPendingNonceByAccount(account)
 }
 
-func (b *BrokerAPI) GetPendingTransactions(_ int) []pb.Transaction {
+func (b *BrokerAPI) GetPendingTransactions(max int) []pb.Transaction {
 	// TODO
 	return nil
 }
@@ -517,3 +563,13 @@ func (b *BrokerAPI) requestTssInfo(pid uint64) (*pb.TssInfo, error) {
 		return info, nil
 	}
 }
+
+func (b *BrokerAPI) SetTssNotParties(tssReq *pb.GetSignsRequest, singers []string) error {
+	var isReq bool
+	if tssReq.Type == pb.GetSignsRequest_TSS_IBTP_REQUEST {
+		isReq = true
+	}
+	signInfo := strings.Split(string(tssReq.Extra), "-")
+	// req.Extra: <tssSigners>-<random>-<notParties>
+	return utils.NotifyNotTssParties(b.bxh.TssMgr, b.bxh.Ledger, tssReq.Content, isReq, singers, signInfo[1])
+}

internal/coreapi/broker.go

@@ -39,3 +39,7 @@ func (network *NetworkAPI) PierManager() peermgr.PierManager {
 func (network *NetworkAPI) OtherPeers() map[uint64]*peer.AddrInfo {
 	return network.bxh.PeerMgr.OtherPeers()
 }
+
+func (network *NetworkAPI) LocalPeerID() uint64 {
+	return network.bxh.PeerMgr.GetLocalID()
+}