• Andrew Walker's avatar
    Improve CTDB private IPs handling (#11643) · 6bd31c65
    Andrew Walker authored
    This PR makes several critical changes to how the ctdb nodes
    file (private IPs) are managed in middleware. Some types of
    issues that are addressed by this PR are:
    
    * Inconsistencies in nodes configuration file on different
      cluster nodes
    * Duplicate nodes entries being added via ctdb.shared.volume.create
      (e.g. same node twice with different IPs)
    * Lack of clear mapping between ctdb nodes and gluster peers
    
    Originally there was easy mapping of entries in the ctdb nodes
    file back to the originating gluster peer. This PR changes the
    nodes files entry from form of
    ```
    <ipaddress>
    <ipaddress>
    <ipaddress>
    ```
    
    to
    ```
    <ipaddress>#{<peer information>}
    <ipaddress>#{<peer information>}
    ```
    
    Above required a change to nodes file parsing function in ctdbd,
    which means this PR may not be backported into stable/bluefin.
    
    Currently only peer UUID is stored in the file, but in future
    we can potentially expand to include additional information.
    Store gluster node UUID in ctdb nodes file. This allows us to
    more tightly couple gluster TSP configuration with our CTDB nodes
    configuration (e.g. include peer UUID in ctdb.private.ips.query
    return).
    
    Since this change requires that the backend maintain the mapping
    between nodes and peers, the ctdb private ips API was changed to
    require caller to provide a gluster peer UUID for the nodes entry.
    
    The ctdb.shared.volume.create method originally would automatically
    append nodes file entries for missing gluster peers based on DNS
    lookup results, but this has over time proven to be somewhat less
    than reliable (users may have DNS misconfigurations that result
    in multiple nodes files entries or incorrect interfaces being used).
    
    This PR allows caller of ctdb.shared.volume.create to optionally
    include a list of private address + gluster peer UUID mappings
    to be added (if necessary) to the nodes file. This method will now
    explicitly fail in the following situations:
    
    * gluster peers that are not present in resulting file
    * peer UUIDs in payload that do not exist
    * entries in nodes file that do not map back to gluster UUID
    
    This has somewhat wide-ranging impact for how our backend APIs are
    used. For instance, if a new gluster peer is added to an existing
    TSP, then a private IP for the nodes file must also be supplied.
    
    A new cluster management service is also being added via this
    pull request to enforce consistency in how SCALE clusters are
    configured (and ensure that proper validation takes place to prevent
    issue reports on drastically misconfigured clutsers). It also
    provides a stub of an API for adding new cluster nodes (expansion).
    
    The end-goal of this API is to force the caller to provide us with
    a payload containing gluster peer, brick, and private address (nodes
    file) information for each proposed cluster node.
    Unverified
    6bd31c65
test_cluster_cleanup.py 5.01 KB
from time import sleep

import pytest
from pytest_dependency import depends

from config import CLUSTER_INFO, CLUSTER_IPS, TIMEOUTS
from utils import make_request, make_ws_request, wait_on_job
from exceptions import JobTimeOut


@pytest.mark.parametrize('ip', [
    CLUSTER_INFO['NODE_A_IP'],
    CLUSTER_INFO['NODE_B_IP'],
    CLUSTER_INFO['NODE_C_IP'],
    CLUSTER_INFO['NODE_D_IP'],
])
def test_gather_debugs_before_teardown(ip, request):
    ans = make_ws_request(ip, {'msg': 'method', 'method': 'system.debug_generate'})
    assert ans.get('error') is None, ans
    assert isinstance(ans['result'], int), ans
    try:
        status = wait_on_job(ans['result'], ip, 600)
    except JobTimeOut:
        assert False, f'Timed out waiting to generate debug on {ip!r}'
    else:
        assert status['state'] == 'SUCCESS', status


@pytest.mark.dependency(name='STOP_GVOLS')
def test_stop_all_gvols():
    ans = make_request('get', '/gluster/volume')
    assert ans.status_code == 200, ans.text

    # we will try to stop each gluster volume `retries` times waiting at least
    # 1 second between each attempt
    retries = TIMEOUTS['FUSE_OP_TIMEOUT']
    sleeptime = 1
    for i in filter(lambda x: x['status'] == 'Started', ans.json()):
        gvol = i['name']
        for retry in range(retries):
            stop = make_request('post', '/gluster/volume/stop', data={'name': gvol, 'force': True})
            if stop.status_code == 422:
                if 'Another transaction is in progress' in stop.text:
                    # another command is running in the cluster so this isn't fatal but expected
                    # so we'll sleep for `sleeptime` seconds to backoff and let cluster settle
                    sleep(sleeptime)
                else:
                    assert False, f'Failed to stop {gvol!r}: {stop.text}'
            elif stop.status_code == 200:
                break
            elif retry == retries:
                assert False, f'Retried {retries} times to stop {gvol!r} but failed.'


@pytest.mark.parametrize('ip', CLUSTER_IPS)
@pytest.mark.dependency(name='VERIFY_FUSE_UMOUNTED')
def test_verify_all_gvols_are_fuse_umounted(ip, request):
    depends(request, ['STOP_GVOLS'])

    ans = make_request('get', '/gluster/volume/list')
    assert ans.status_code == 200, ans.text

    # we will try to check if each FUSE mountpoint is umounted `retries` times waiting at
    # least `sleeptime` second between each attempt
    retries = TIMEOUTS['FUSE_OP_TIMEOUT']
    sleeptime = 1
    for i in ans.json():
        for retry in range(retries):
            # give each node a little time to actually umount the fuse volume before we claim failure
            rv = make_request('post', f'http://{ip}/api/v2.0/gluster/fuse/is_mounted', data={'name': i})
            assert rv.status_code == 200
            if not rv.json():
                break

            sleep(sleeptime)
            assert retry != retries, f'Waited {retries} seconds on FUSE mount for {i!r} to become umounted.'


@pytest.mark.dependency(name='DELETE_GVOLS')
def test_delete_gvols(request):
    depends(request, ['VERIFY_FUSE_UMOUNTED'])

    ans = make_request('get', '/gluster/volume/list')
    assert ans.status_code == 200, ans.text

    for i in ans.json():
        delete = make_request('delete', f'/gluster/volume/id/{i}')
        assert delete.status_code == 200, delete.text
        try:
            status = wait_on_job(delete.json(), CLUSTER_INFO['NODE_A_IP'], 600)
        except JobTimeOut:
            assert False, f'Timed out waiting for {i!r} to be deleted'
        else:
            assert status['state'] == 'SUCCESS', status


@pytest.mark.parametrize('ip', CLUSTER_IPS)
@pytest.mark.dependency(name='CTDB_TEARDOWN')
def test_ctdb_shared_vol_teardown(ip, request):
    payload = {'msg': 'method', 'method': 'ctdb.root_dir.teardown'}
    ans = make_ws_request(ip, payload)
    assert ans.get('error') is None, ans
    assert isinstance(ans['result'], int), ans
    try:
        status = wait_on_job(ans['result'], ip, 120)
    except JobTimeOut:
        assert False, f'Timed out waiting for ctdb shared volume to be torn down on {ip!r}'
    else:
        assert status['state'] == 'SUCCESS', status


@pytest.mark.parametrize('ip', CLUSTER_IPS)
def test_verify_ctdb_teardown(ip, request):
    depends(request, ['CTDB_TEARDOWN'])

    payload = {'msg': 'method', 'method': 'cluster.utils.state_to_be_removed'}
    ans = make_ws_request(ip, payload)
    assert ans.get('error') is None, ans
    assert isinstance(ans['result'], list), ans

    files, dirs = ans['result']
    payload = {'msg': 'method', 'method': 'filesystem.stat'}
    for _file in files:
        payload.update({'params': [_file]})
        ans = make_ws_request(ip, payload)
        assert ans.get('error'), ans
        assert '[ENOENT]' in ans['error']['reason']

    payload = {'msg': 'method', 'method': 'filesystem.listdir'}
    for _dir in dirs:
        payload.update({'params': [_dir]})
        ans = make_ws_request(ip, payload)
        assert ans.get('error') is None, ans
        assert len(ans['result']) == 0, ans['result']