storage: introduce box_cfg_mode

Part of #428

@TarantoolBot document
Title: vshard: `box_cfg_mode` cfg option

The option can be specified at the root level and regulates, whether
vshard calls box.cfg, when vshard.storage is invoked.

When specified to 'auto' (default), vshard configures box.cfg on its
own. When specified to 'manual', it's user's responsibility to call
box.cfg prior to vshard's configuration

Author: Serpentian

test/storage-luatest/box_cfg_mode_test.lua

@@ -0,0 +1,61 @@
+local t = require('luatest')
+local vtest = require('test.luatest_helpers.vtest')
+
+local test_group = t.group()
+
+local cfg_template = {
+    sharding = {
+        {
+            replicas = {
+                replica_1_a = {master = true},
+            },
+        },
+        {
+            replicas = {
+                replica_2_a = {master = true},
+            },
+        },
+    },
+    bucket_count = 20,
+    box_cfg_mode = 'manual',
+}
+
+local global_cfg
+
+test_group.before_all(function(g)
+    global_cfg = vtest.config_new(cfg_template)
+    vtest.cluster_new(g, global_cfg)
+    vtest.cluster_bootstrap(g, global_cfg)
+    vtest.cluster_wait_vclock_all(g)
+    vtest.cluster_rebalancer_disable(g)
+end)
+
+test_group.after_all(function(g)
+    g.cluster:stop()
+end)
+
+test_group.test_storage_disabled_error = function(g)
+    g.replica_1_a:exec(function(cfg)
+        local old_box = box.cfg
+        box.cfg = function() end
+        ilt.assert_error_msg_contains('Box must be configured', function()
+            ivshard.storage.cfg(cfg, box.info.uuid)
+        end)
+        box.cfg = old_box
+    end, {global_cfg})
+end
+
+--
+-- vtest.cluster_new does exactly, what user should do,
+-- when using 'manual' box_cfg_mode: call box.cfg prior
+-- to executing vshard.storage.cfg. So, just some basic test.
+--
+test_group.test_storage_basic = function(g)
+    g.replica_1_a:exec(function(uuid)
+        local bid = _G.get_first_bucket()
+        local ok, err = ivshard.storage.bucket_send(bid, uuid)
+        ilt.assert_equals(err, nil)
+        ilt.assert(ok)
+        _G.bucket_gc_wait()
+    end, {g.replica_2_a:replicaset_uuid()})
+end

test/unit-luatest/config_test.lua

@@ -120,4 +120,15 @@ g.test_enum = function()
         "Discovery mode must be enum {'on', 'off', 'once', nil}",
         vcfg.check, config)
     config.discovery_mode = nil
+
+    config.box_cfg_mode = nil
+    for _, v in pairs({'auto', 'manual'}) do
+        config.box_cfg_mode = v
+        t.assert(vcfg.check(config))
+    end
+    config.box_cfg_mode = 'bad'
+    t.assert_error_msg_content_equals(
+        "Box.cfg mode must be enum {'auto', 'manual', nil}",
+        vcfg.check, config)
+    config.box_cfg_mode = nil
 end

vshard/cfg.lua

@@ -376,6 +376,10 @@ local cfg_template = {
         name = 'Scheduler bucket move quota', type = 'non-negative number',
         is_optional = true, default = consts.DEFAULT_SCHED_MOVE_QUOTA
     },
+    box_cfg_mode = {
+        name = 'Box.cfg mode', type = 'enum', is_optional = true,
+        default = 'auto', enum = {'auto', 'manual'},
+    }
 }
 
 --

vshard/storage/init.lua

@@ -1957,13 +1957,24 @@ local function buckets_discovery(opts)
     return ret
 end
 
+local function storage_box_cfg(cfg)
+    if M.box_cfg_mode == 'manual' then
+        if type(box.cfg) == 'function' then
+            error(lerror.vshard(lerror.code.STORAGE_IS_DISABLED,
+                "Box must be configured, when box_cfg_mode = 'manual' is used"))
+        end
+        return true
+    end
+    return pcall(box.cfg, cfg)
+end
+
 --
 -- The only thing, that must be done to abort a master demote is
 -- a reset of read_only.
 --
 local function local_on_master_disable_abort()
     if not M.current_cfg or M.current_cfg.read_only == nil then
-        box.cfg{read_only = false}
+        storage_box_cfg({read_only = false})
     end
 end
 
@@ -1975,7 +1986,7 @@ end
 local function local_on_master_disable_prepare()
     log.info("Resigning from the replicaset master role...")
     if not M.current_cfg or M.current_cfg.read_only == nil then
-        box.cfg({read_only = true})
+        storage_box_cfg({read_only = true})
         sync(M.sync_timeout)
     end
 end
@@ -2006,7 +2017,7 @@ end
 --
 local function local_on_master_enable_abort()
     if not M.current_cfg or M.current_cfg.read_only == nil then
-        box.cfg({read_only = true})
+        storage_box_cfg({read_only = true})
     end
 end
 
@@ -2026,7 +2037,7 @@ end
 --
 local function local_on_master_enable()
     if not M.current_cfg or M.current_cfg.read_only == nil then
-        box.cfg({read_only = false})
+        storage_box_cfg({read_only = false})
     end
     M._on_master_enable:run()
     -- Start background process to collect garbage.
@@ -3396,6 +3407,8 @@ local function storage_cfg(cfg, this_replica_uuid, is_reload)
     --
     local old_sync_timeout = M.sync_timeout
     M.sync_timeout = vshard_cfg.sync_timeout
+    -- TODO: refactor this to use the context.
+    M.box_cfg_mode = vshard_cfg.box_cfg_mode
 
     if was_master and not is_master then
         local_on_master_disable_prepare()
@@ -3450,7 +3463,7 @@ local function storage_cfg(cfg, this_replica_uuid, is_reload)
                                     real_rs_uuid, this_replicaset.uuid))
             end
         end
-        local ok, err = pcall(box.cfg, box_cfg)
+        local ok, err = storage_box_cfg(box_cfg)
         while M.errinj.ERRINJ_CFG_DELAY do
             lfiber.sleep(0.01)
         end