rebalancer: don't show "object is outdated" log

This patch removes logging of `OBJECT_IS_OUTDATED` errors in
the rebalancer and recovery services.

Part of #212

NO_DOC=bugfix

Author: mrForza

test/storage-luatest/storage_1_1_1_test.lua

@@ -270,3 +270,19 @@ rebalancer_recovery_group.test_no_log_spam_when_buckets_no_active = function(g)
     end)
     g.replica_2_a:start()
 end
+
+rebalancer_recovery_group.test_no_object_is_outdated_logs = function(g)
+    g.replica_1_a:exec(function(cfg)
+        ivshard.storage.internal.errinj.ERRINJ_CFG_OUTDATED_DELAY = true
+        require('fiber').new(function()
+            ivshard.storage.cfg(cfg, box.info.uuid)
+        end)
+        ivshard.storage.rebalancer_wakeup()
+    end, {global_cfg})
+    move_bucket(g.replica_3_a, g.replica_1_a,
+                get_first_storage_bucket_id(g.replica_3_a))
+    g.replica_1_a:exec(function()
+        ivshard.storage.internal.errinj.ERRINJ_CFG_OUTDATED_DELAY = false
+    end)
+    t.assert_not(g.replica_1_a:grep_log('Object is outdated'))
+end

vshard/storage/init.lua

@@ -86,6 +86,7 @@ if not M then
         total_bucket_count = 0,
         errinj = {
             ERRINJ_CFG = false,
+            ERRINJ_CFG_OUTDATED_DELAY = false,
             ERRINJ_RELOAD = false,
             ERRINJ_CFG_DELAY = false,
             ERRINJ_LONG_RECEIVE = false,
@@ -964,9 +965,11 @@ local function recovery_step_by_type(type)
                 if err == nil then
                     err = 'unknown'
                 end
-                log.info(start_format, type)
-                log.error('Error during recovery of bucket %s on replicaset '..
-                          '%s: %s', bucket_id, peer_id, err)
+                if err.name ~= 'OBJECT_IS_OUTDATED' then
+                    log.info(start_format, type)
+                    log.error('Error during recovery of bucket %s ' ..
+                              ' on replicaset %s: %s', bucket_id, peer_id, err)
+                end
                 is_step_empty = false
             end
             goto continue
@@ -1854,7 +1857,7 @@ local function bucket_send_xc(bucket_id, destination, opts, exception_guard)
                     {bucket_id, id, data}, opts)
                 bucket_generation =
                     bucket_guard_xc(bucket_generation, bucket_id, sendg)
-                if not status then
+                if not status and err.name ~= 'OBJECT_IS_OUTDATED' then
                     return status, lerror.make(err)
                 end
                 limit = consts.BUCKET_CHUNK_SIZE
@@ -1866,7 +1869,7 @@ local function bucket_send_xc(bucket_id, destination, opts, exception_guard)
     end
     status, err = master_call(replicaset, 'vshard.storage.bucket_recv',
                               {bucket_id, id, data}, opts)
-    if not status then
+    if not status and err.name ~= 'OBJECT_IS_OUTDATED' then
         return status, lerror.make(err)
     end
     while M.errinj.ERRINJ_LAST_SEND_DELAY do
@@ -1896,7 +1899,7 @@ local function bucket_send_xc(bucket_id, destination, opts, exception_guard)
     -- the same bucket is activated on the destination.
     status, err = master_call(replicaset, 'vshard.storage.bucket_recv',
                               {bucket_id, id, {}, {is_last = true}}, opts)
-    if not status then
+    if not status and err.name ~= 'OBJECT_IS_OUTDATED' then
         return status, lerror.make(err)
     end
     return true
@@ -2901,9 +2904,11 @@ local function rebalancer_service_f(service)
                 rs, 'vshard.storage.rebalancer_apply_routes', {src_routes},
                 {timeout = consts.REBALANCER_APPLY_ROUTES_TIMEOUT})
             if not status then
-                log.error(service:set_status_error(
-                    'Error during routes appying on "%s": %s. '..
-                    'Try rebalance later', rs, lerror.make(err)))
+                if err.name ~= 'OBJECT_IS_OUTDATED' then
+                    log.error(service:set_status_error(
+                        'Error during routes appying on "%s": %s. '..
+                        'Try rebalance later', rs, lerror.make(err)))
+                end
                 service:set_activity('idling')
                 lfiber.sleep(consts.REBALANCER_WORK_INTERVAL)
                 goto continue
@@ -3757,6 +3762,9 @@ local function storage_cfg_xc(cfgctx)
     lschema.cfg(new_cfg)
     lreplicaset.rebind_replicasets(new_replicasets, M.replicasets)
     lreplicaset.outdate_replicasets(M.replicasets)
+    while M.errinj.ERRINJ_CFG_OUTDATED_DELAY do
+        lfiber.sleep(0.01)
+    end
     lreplicaset.create_workers(new_replicasets)
     M.replicasets = new_replicasets
     M.this_replicaset = new_replicasets[replicaset_id]