Add list of allowed membership uris (#2300)

CHANGELOG.rst

@@ -12,6 +12,19 @@ and this project adheres to
 Unreleased
 -------------------------------------------------------------------------------
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Changed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Update ``membership`` dependency to `2.5.0 <https://github.com/tarantool/membership/releases/tag/2.5.0>`_.
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Added
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- New option ``exclude_expelled_members`` to exclude expelled
+  instances from the membership process.
+
 -------------------------------------------------------------------------------
 [2.14.0] - 2025-02-13
 -------------------------------------------------------------------------------

cartridge-scm-1.rockspec

@@ -11,7 +11,7 @@ dependencies = {
     'checks == 3.3.0-1',
     'errors == 2.2.1-1',
     'vshard == 0.1.31-1',
-    'membership == 2.4.6-1',
+    'membership == 2.5.0-1',
     'frontend-core == 8.2.2-1',
     'cartridge-metrics-role == 0.1.1',
     'graphql == 0.3.0-1',

cartridge.lua

@@ -305,6 +305,12 @@ end
 --   env `TARANTOOL_SET_COOKIE_HASH_MEMBERSHIP`,
 --   args `--set-cookie-hash-membership`)
 --
+-- @tparam ?boolean opts.exclude_expelled_members
+--   Do not add expelled members to the membership.
+--   default: `false`, overridden by
+--   env `TARANTOOL_EXCLUDE_EXPELLED_MEMBERS`,
+--   args `--exclude-expelled-members`)
+--
 -- @tparam ?boolean opts.rebalancer_mode
 --   Rebalancer mode for vshard cluster. See vshard doc for more details.
 --   env `TARANTOOL_REBALANCER_MODE`,
@@ -346,6 +352,7 @@ local function cfg(opts, box_opts)
         enable_synchro_mode = '?boolean',
         disable_raft_on_small_clusters = '?boolean',
         set_cookie_hash_membership = '?boolean',
+        exclude_expelled_members = '?boolean',
 
         transport = '?string',
         ssl_ciphers = '?string',
@@ -999,6 +1006,7 @@ local function cfg(opts, box_opts)
         enable_failover_suppressing = opts.enable_failover_suppressing,
         enable_synchro_mode = opts.enable_synchro_mode,
         disable_raft_on_small_clusters = opts.disable_raft_on_small_clusters,
+        exclude_expelled_members = opts.exclude_expelled_members,
 
         transport = opts.transport,
         ssl_ciphers = opts.ssl_ciphers,

cartridge/argparse.lua

@@ -117,6 +117,7 @@ local cluster_opts = {
     disable_raft_on_small_clusters = 'boolean', -- **boolean**
     enable_failover_suppressing = 'boolean', -- **boolean**
     set_cookie_hash_membership = 'boolean', -- **boolean**
+    exclude_expelled_members = 'boolean', -- **boolean**
 
     twophase_netbox_call_timeout = 'number', -- **number**
     twophase_upload_config_timeout = 'number', -- **number**

cartridge/confapplier.lua

@@ -7,7 +7,6 @@
 
 local log = require('log')
 local fio = require('fio')
-local fun = require('fun')
 local yaml = require('yaml').new()
 local fiber = require('fiber')
 local errors = require('errors')
@@ -61,6 +60,7 @@ vars:new('upgrade_schema', nil)
 vars:new('enable_failover_suppressing', nil)
 vars:new('enable_synchro_mode', nil)
 vars:new('disable_raft_on_small_clusters', nil)
+vars:new('exclude_expelled_members', nil)
 
 vars:new('transport', nil)
 vars:new('ssl_options', {
@@ -289,12 +289,19 @@ local function apply_config(clusterwide_config)
     set_state('ConfiguringRoles')
 
     local topology_cfg = clusterwide_config:get_readonly('topology')
-    if failover.is_leader() then
-        for _, uuid, _ in fun.filter(topology.expelled, topology_cfg.servers) do
+    local allowed_uris = {}
+    for uuid, srv in pairs(topology_cfg.servers) do
+        if srv == 'expelled' and failover.is_leader() then
             box.space._cluster.index.uuid:delete(uuid)
+        elseif srv.uri ~= nil then
+            table.insert(allowed_uris, srv.uri)
         end
     end
 
+    if vars.exclude_expelled_members then
+        membership.set_allowed_members(allowed_uris)
+    end
+
     box.cfg({
         replication_connect_quorum = 0,
         replication = topology.get_fullmesh_replication(
@@ -787,6 +794,7 @@ local function init(opts)
         enable_failover_suppressing = '?boolean',
         enable_synchro_mode = '?boolean',
         disable_raft_on_small_clusters = '?boolean',
+        exclude_expelled_members = '?boolean',
 
         transport = '?string',
         ssl_ciphers = '?string',
@@ -810,6 +818,7 @@ local function init(opts)
     vars.enable_failover_suppressing = opts.enable_failover_suppressing
     vars.enable_synchro_mode = opts.enable_synchro_mode
     vars.disable_raft_on_small_clusters = opts.disable_raft_on_small_clusters
+    vars.exclude_expelled_members = opts.exclude_expelled_members
     vars.transport = opts.transport
     vars.ssl_ciphers = opts.ssl_ciphers
     vars.ssl_server_ca_file = opts.ssl_server_ca_file

rst/cartridge_admin.rst

@@ -1914,6 +1914,25 @@ To migrate to this feature, perform next actions:
     in your ``init.lua`` file, or use environment variables. Don't forget to
     persist this setting.
 
+.. _cartridge-membership-remove-expelled:
+
+-------------------------------------------------------------------------------
+Getting rid of expelled instances in membership
+-------------------------------------------------------------------------------
+
+Sometimes, expelled instances can remain in the membership, e. g. when they
+weren't finished correctly before being expelled. Those instances **can't**
+cause any harm, but they can be annoying when reading membership members list.
+If you are being bothered by those instances in membership, you can remove them
+with the option ``cartridge.cfg({exclude_expelled_members = true})`` or with env
+`TARANTOOL_EXCLUDE_EXPELLED_MEMBERS` or with command-line argument
+`--exclude-expelled-members`.
+
+Enabling this option will remove *stopped* expelled instances from the membership list.
+Note that *alive* expelled instances and any instances that have the same
+``membership_encryption_key`` and which can ping your cluster (like newly launched
+instances which are not in the topology yet) will remain in the membership list.
+
 .. _cartridge-fix-config:
 
 -------------------------------------------------------------------------------

test/integration/expel_test.lua

@@ -1,4 +1,5 @@
 local fio = require('fio')
+local fun = require('fun')
 local t = require('luatest')
 local g = t.group()
 
@@ -14,6 +15,10 @@ g.before_all(function()
             alias = 'A',
             roles = {},
             servers = 3,
+        }, {
+            alias = 'B',
+            roles = {},
+            servers = 1,
         }},
     })
     g.cluster:start()
@@ -33,23 +38,54 @@ g.before_all(function()
     )
 
     expelled:stop()
-    g.A1:eval([[
+    g.A1:exec(function(uuid1, uuid2)
         package.loaded.cartridge.admin_edit_topology({servers = {{
-            uuid = ...,
+            uuid = uuid1,
+            expelled = true,
+        }, {
+            uuid = uuid2,
             expelled = true,
         }}})
-    ]], {expelled.instance_uuid})
-
+    end, {expelled.instance_uuid, g.cluster:server('B-1').instance_uuid})
+    g.expelled_uri = expelled.advertise_uri
     g.A1:call('package.loaded.cartridge.admin_edit_topology',
         {{servers = {{uuid = expelled.instance_uuid, expelled = true}}}})
 
+    g.standalone = helpers.Server:new({
+        alias = 'standalone',
+        workdir = fio.pathjoin(g.cluster.datadir, 'standalone'),
+        command = helpers.entrypoint('srv_basic'),
+        cluster_cookie = g.cluster.cookie,
+        advertise_port = 13300,
+        http_port = 8080,
+        replicaset_uuid = helpers.uuid('b'),
+        instance_uuid = helpers.uuid('b', 'b', 1),
+    })
+    g.standalone:start()
 end)
 
 g.after_all(function()
     g.cluster:stop()
+    g.standalone:stop()
     fio.rmtree(g.cluster.datadir)
 end)
 
+local function check_members(g, expected)
+    local to_check = fun.iter(expected):map(function(x) return x end):totable()
+    table.sort(to_check)
+    t.helpers.retrying({}, function()
+        local res = g.A1:exec(function()
+            local fun = require('fun')
+            local membership = require('membership')
+            local members = fun.iter(membership.members()):
+                map(function(x) return x end):totable()
+            table.sort(members)
+            return members
+        end)
+        t.assert_equals(res, to_check)
+    end)
+end
+
 function g.test_api()
     local ret = g.A1:eval('return box.info.replication')
     t.assert_covers(ret[1], {id = 1, uuid = g.r1_uuid}, ret)
@@ -93,4 +129,25 @@ function g.test_api()
         g.A1.net_box.space._cluster:select(),
         {{1, g.r1_uuid}, {3, g.r3_uuid}}
     )
+
+    local expected = {
+        -- second instance is space _cluster is expelled:
+        [g.cluster:server('A-1').advertise_uri] = true,
+        [g.cluster:server('A-2').advertise_uri] = true,
+        [g.cluster:server('A-3').advertise_uri] = true,
+        -- expelled, but not stopped:
+        [g.cluster:server('B-1').advertise_uri] = true,
+        -- not in the cluster:
+        [g.standalone.advertise_uri] = true,
+    }
+    table.sort(expected)
+
+    check_members(g, expected)
+
+    g.A1.env['TARANTOOL_EXCLUDE_EXPELLED_MEMBERS'] = 'true'
+    g.A1:restart()
+
+    -- now every instance except expelled and stopped should remain in membership
+    expected[g.expelled_uri] = nil
+    check_members(g, expected)
 end