Store Configuration paths/values in separate Atomix map (#1606)

* Modify Configuration store to store committed/applied paths/values in a separate Atomix map

* Update Atomix Go SDK to v0.13.0

* Use Atomix Map transactions to update configuration paths/values

* Prune Configuration values in Configuration store to remove redundant paths from store map

* Update path/value when index is changed in Configuration store

* Decrease log level of regular messages in controllers

* Optimize Watch methods in stores to reduce primitive writes

* Fix unit test failures

* Remove unnecessary 'replace' statement from go.mod

go.mod

@@ -3,7 +3,7 @@ module github.com/onosproject/onos-config
 go 1.19
 
 require (
-	github.com/atomix/go-sdk v0.12.7
+	github.com/atomix/go-sdk v0.13.2
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/mock v1.6.0
 	github.com/golang/protobuf v1.5.2
@@ -26,9 +26,9 @@ require (
 
 require (
 	github.com/Shopify/sarama v1.31.1 // indirect
-	github.com/atomix/atomix/api v0.9.2 // indirect
-	github.com/atomix/atomix/protocols/rsm v0.5.6 // indirect
-	github.com/atomix/atomix/runtime v0.9.0 // indirect
+	github.com/atomix/atomix/api v1.1.0 // indirect
+	github.com/atomix/atomix/protocols/rsm v1.1.0 // indirect
+	github.com/atomix/atomix/runtime v1.1.0 // indirect
 	github.com/atomix/atomix/sidecar v0.4.4 // indirect
 	github.com/bits-and-blooms/bitset v1.3.1 // indirect
 	github.com/bits-and-blooms/bloom/v3 v3.3.1 // indirect
@@ -87,5 +87,3 @@ require (
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
-
-replace github.com/docker/docker => github.com/docker/engine v1.4.2-0.20200229013735-71373c6105e3

go.sum

@@ -47,16 +47,16 @@ github.com/Shopify/sarama v1.31.1/go.mod h1:99E1xQ1Ql2bYcuJfwdXY3cE17W8+549Ty8PG
 github.com/Shopify/toxiproxy/v2 v2.3.0 h1:62YkpiP4bzdhKMH+6uC5E95y608k3zDwdzuBMsnn3uQ=
 github.com/Shopify/toxiproxy/v2 v2.3.0/go.mod h1:KvQTtB6RjCJY4zqNJn7C7JDFgsG5uoHYDirfUfpIm0c=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/atomix/atomix/api v0.9.2 h1:BpT4zsXiKQWoIHQzGeCDQuC4BxcWvfrsMrPuUoSNLTI=
-github.com/atomix/atomix/api v0.9.2/go.mod h1:Fz8zXQH6n28U0NTu5xctKhkNrN5RsWgX56lrMhqXlPg=
-github.com/atomix/atomix/protocols/rsm v0.5.6 h1:55u/qIIUuLakypf1d1CQ5xxj/E720f6FRGDtv1VUOSs=
-github.com/atomix/atomix/protocols/rsm v0.5.6/go.mod h1:SgkK3PVLqx1KmNjxw8OY7AY7T6t6TuPLmnP6Rr/oFAQ=
-github.com/atomix/atomix/runtime v0.9.0 h1:yDViPymoOkR5gMJkk25CZP2Tjf2j1JMAWNd/I38ws8I=
-github.com/atomix/atomix/runtime v0.9.0/go.mod h1:abB/WaP50Fm6MVi8CHRsfIOt8urDzcpQvMDFmhhlVgY=
+github.com/atomix/atomix/api v1.1.0 h1:zUbuD4yPu+jBT8NkxvDKx+m8QiRqhVmFUMgRvQoC1Tc=
+github.com/atomix/atomix/api v1.1.0/go.mod h1:Fz8zXQH6n28U0NTu5xctKhkNrN5RsWgX56lrMhqXlPg=
+github.com/atomix/atomix/protocols/rsm v1.1.0 h1:IFsU/VqoFjjRWRc+ET0B0aYqMG3+oTzDwuiYhVbBQVo=
+github.com/atomix/atomix/protocols/rsm v1.1.0/go.mod h1:TT5+SaXyrpS3CAHeGYftmi9DEybCsI8bZY/sFMJijTQ=
+github.com/atomix/atomix/runtime v1.1.0 h1:K1fUQqfngOqkXTk+1CwbPLO9STQ6gUVe6qXLlg9ageM=
+github.com/atomix/atomix/runtime v1.1.0/go.mod h1:7PtAhumBMs3TE3L/qXUSr4cNqqOWKGBFTvcHw+ZIQZ8=
 github.com/atomix/atomix/sidecar v0.4.4 h1:CjLPA3p1V83Tx2yxUGz4JLyQkeyXC8/1xXaiZVknJsY=
 github.com/atomix/atomix/sidecar v0.4.4/go.mod h1:psygbD10K+EdWS1XINexXP0QmiAd+yY2tGkIGqrVMHM=
-github.com/atomix/go-sdk v0.12.7 h1:mNj44w930GMb6Sik60LbPhcS52LLqnWvQBVmSRT4jIU=
-github.com/atomix/go-sdk v0.12.7/go.mod h1:Ua1SPP/5qms2DZrpVeH+RrIvbviIEExKEijlgZHMWng=
+github.com/atomix/go-sdk v0.13.2 h1:KeCP4S84iCbQhq/Ci+3t9eYpKD5l57EfKaBVG46bcNI=
+github.com/atomix/go-sdk v0.13.2/go.mod h1:AmNgqS80WqBj6AxdudXhMPhyEB+KW1kZIebWQpBgm9A=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/bits-and-blooms/bitset v1.3.1 h1:y+qrlmq3XsWi+xZqSaueaE8ry8Y127iMxlMfqcK8p0g=
 github.com/bits-and-blooms/bitset v1.3.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=

pkg/controller/configuration/controller.go

@@ -71,7 +71,7 @@ func (r *Reconciler) Reconcile(id controller.ID) (controller.Result, error) {
 		return controller.Result{}, nil
 	}
 
-	log.Infof("Reconciling Configuration '%s'", config.ID)
+	log.Debugf("Reconciling Configuration '%s'", config.ID)
 	log.Debug(config)
 	return r.reconcileConfiguration(ctx, config)
 }

pkg/controller/connection/controller.go

@@ -52,7 +52,7 @@ func (r *Reconciler) Reconcile(id controller.ID) (controller.Result, error) {
 	defer cancel()
 
 	connID := id.Value.(gnmi.ConnID)
-	log.Infof("Reconciling Conn '%s'", connID)
+	log.Debugf("Reconciling Conn '%s'", connID)
 	conn, ok := r.conns.Get(ctx, connID)
 	if !ok {
 		return r.deleteRelation(ctx, connID)

pkg/controller/node/controller.go

@@ -88,7 +88,7 @@ func (r *Reconciler) Reconcile(id controller.ID) (controller.Result, error) {
 	defer cancel()
 
 	onosConfigID := id.Value.(topoapi.ID)
-	log.Infof("Reconciling onos-config entity with ID: %s", onosConfigID)
+	log.Debugf("Reconciling onos-config entity with ID: %s", onosConfigID)
 	object, err := r.topo.Get(ctx, onosConfigID)
 	if err == nil {
 		//  Reconciles an onos-config entity that’s not local so the controller should requeue

pkg/controller/proposal/controller.go

@@ -94,7 +94,7 @@ func (r *Reconciler) Reconcile(id controller.ID) (controller.Result, error) {
 		return controller.Result{}, nil
 	}
 
-	log.Infof("Reconciling Proposal '%s'", proposal.ID)
+	log.Debugf("Reconciling Proposal '%s'", proposal.ID)
 	log.Debug(proposal)
 	return r.reconcileProposal(ctx, proposal)
 }
@@ -456,12 +456,10 @@ func (r *Reconciler) reconcileCommit(ctx context.Context, proposal *configapi.Pr
 			if config.Values == nil {
 				config.Values = make(map[string]*configapi.PathValue)
 			}
-			updatedChangeValues := controllerutils.AddDeleteChildren(changeValues, config.Values)
+			updatedChangeValues := controllerutils.AddDeleteChildren(proposal.TransactionIndex, changeValues, config.Values)
 			for path, updatedChangeValue := range updatedChangeValues {
 				_, _ = applyChangeToConfig(config.Values, path, updatedChangeValue)
 			}
-			config.Values = tree.PrunePathMap(config.Values, true)
-
 			config.Status.Committed.Index = proposal.TransactionIndex
 			err = r.configurations.Update(ctx, config)
 			if err != nil {
@@ -632,7 +630,7 @@ func (r *Reconciler) reconcileApply(ctx context.Context, proposal *configapi.Pro
 			changeValues = proposal.Status.RollbackValues
 		}
 
-		updatedChangeValues := controllerutils.AddDeleteChildren(changeValues, config.Values)
+		updatedChangeValues := controllerutils.AddDeleteChildren(proposal.TransactionIndex, changeValues, config.Values)
 		// Create a list of PathValue pairs from which to construct a gNMI Set for the Proposal.
 		pathValues := make([]*configapi.PathValue, 0, len(updatedChangeValues))
 		for _, changeValue := range updatedChangeValues {
@@ -741,7 +739,6 @@ func (r *Reconciler) reconcileApply(ctx context.Context, proposal *configapi.Pro
 		for path, changeValue := range updatedChangeValues {
 			config.Status.Applied.Values[path] = changeValue
 		}
-		config.Status.Applied.Values = tree.PrunePathMap(config.Status.Applied.Values, true)
 
 		if err := r.configurations.UpdateStatus(ctx, config); err != nil {
 			log.Warnf("Failed reconciling Transaction %d Proposal to target '%s'", proposal.TransactionIndex, proposal.TargetID, err)

pkg/controller/target/controller.go

@@ -51,7 +51,7 @@ func (r *Reconciler) Reconcile(id controller.ID) (controller.Result, error) {
 	defer cancel()
 
 	targetID := id.Value.(topoapi.ID)
-	log.Infof("Reconciling Target '%s'", targetID)
+	log.Debugf("Reconciling Target '%s'", targetID)
 	target, err := r.topo.Get(ctx, targetID)
 	if err != nil {
 		if !errors.IsNotFound(err) {

pkg/controller/transaction/controller.go

@@ -62,7 +62,7 @@ func (r *Reconciler) Reconcile(id controller.ID) (controller.Result, error) {
 		return controller.Result{}, nil
 	}
 
-	log.Infof("Reconciling Transaction %d", transaction.Index)
+	log.Debugf("Reconciling Transaction %d", transaction.Index)
 	log.Debug(transaction)
 	return r.reconcileTransaction(ctx, transaction)
 }

pkg/controller/utils/utils.go

@@ -23,7 +23,7 @@ func GetOnosConfigID() topoapi.ID {
 }
 
 // AddDeleteChildren adds all children of the intermediate path which is required to be deleted
-func AddDeleteChildren(changeValues map[string]*configapi.PathValue, configStore map[string]*configapi.PathValue) map[string]*configapi.PathValue {
+func AddDeleteChildren(index configapi.Index, changeValues map[string]*configapi.PathValue, configStore map[string]*configapi.PathValue) map[string]*configapi.PathValue {
 	// defining new changeValues map, where we will include old changeValues map and new pathValues to be cascading deleted
 	var updChangeValues = make(map[string]*configapi.PathValue)
 	for _, changeValue := range changeValues {
@@ -32,6 +32,7 @@ func AddDeleteChildren(changeValues map[string]*configapi.PathValue, configStore
 			for _, value := range configStore {
 				if strings.HasPrefix(value.Path, changeValue.Path) && !strings.EqualFold(value.Path, changeValue.Path) {
 					updChangeValues[value.Path] = value
+					updChangeValues[value.Path].Index = index
 					updChangeValues[value.Path].Deleted = true
 				}
 			}

pkg/controller/utils/utils_test.go

@@ -107,7 +107,7 @@ func Test_CascadingDeleteAlgorithm(t *testing.T) {
 	log.Infof("changeValues is\n%v", changeValues)
 
 	// cascading delete algorithm
-	updatedChangeValues := AddDeleteChildren(changeValues, store)
+	updatedChangeValues := AddDeleteChildren(1, changeValues, store)
 	log.Infof("updChangeValues is\n%v", updatedChangeValues)
 	assert.Equal(t, 5, len(updatedChangeValues))
 	log.Infof("updChangeValues has %d PathValues to delete", len(updatedChangeValues))