salt/tests/pytests/scenarios/multimaster/test_offline_master.py

import time

import pytest


@pytest.fixture(scope="module")
def event_listener(salt_factories):
    return salt_factories.event_listener


def test_minion_hangs_on_master_failure_50814(
    event_listener,
    salt_mm_master_1,
    salt_mm_master_2,
    salt_mm_minion_1,
    mm_master_2_salt_cli,
):
    """
    Check minion handling events for the alive master when another master is dead.
    The case being checked here is described in details in issue #50814.
    """
    # Let's make sure everything works with both masters online
    event_count = 3
    while event_count:
        check_event_start_time = time.time()
        stop_time = check_event_start_time + 30
        event_tag = "myco/foo/bar/{}".format(event_count)
        ret = mm_master_2_salt_cli.run(
            "event.send", event_tag, minion_tgt=salt_mm_minion_1.id
        )
        assert ret.exitcode == 0
        assert ret.json is True
        # Let's make sure we get the event back
        mm_master_1_event_match = mm_master_2_event_match = None
        while True:
            if time.time() > stop_time:
                pytest.fail(
                    "Minion is not responding to the second master after the first "
                    "one has gone. Check #50814 for details."
                )
            if (
                mm_master_1_event_match is not None
                and mm_master_2_event_match is not None
            ):
                # We got the right event back!
                break

            time.sleep(0.5)

            if mm_master_1_event_match is None:
                events = event_listener.get_events(
                    [(salt_mm_master_1.id, event_tag)],
                    after_time=check_event_start_time,
                )
                for event in events:
                    # We got the event back!
                    if event.tag == event_tag:
                        mm_master_1_event_match = True
                        break
            if mm_master_2_event_match is None:
                events = event_listener.get_events(
                    [(salt_mm_master_2.id, event_tag)],
                    after_time=check_event_start_time,
                )
                for event in events:
                    # We got the event back!
                    if event.tag == event_tag:
                        mm_master_2_event_match = True
                        break
        event_count -= 1
        time.sleep(0.5)

    # Now, let's try this one of the masters offline
    with salt_mm_master_1.stopped():
        assert salt_mm_master_1.is_running() is False

        # Sending one event would be okay. It would hang after the second with one of the masters offline
        event_count = 1
        while event_count <= 3:
            check_event_start_time = time.time()
            stop_time = check_event_start_time + 30
            event_tag = "myco/foo/bar/{}".format(event_count)
            ret = mm_master_2_salt_cli.run(
                "event.send", event_tag, minion_tgt=salt_mm_minion_1.id
            )
            assert ret.exitcode == 0
            assert ret.json is True

            # Let's make sure we get the event back
            event_match = None
            while True:
                if time.time() > stop_time:
                    pytest.fail(
                        "Minion is not responding to the second master(events sent: %s) after the first "
                        "has gone offline. Check #50814 for details.",
                        event_count,
                    )
                if event_match is not None:
                    # We got the right event back!
                    break

                time.sleep(0.5)
                events = event_listener.get_events(
                    [(salt_mm_master_2.id, event_tag)],
                    after_time=check_event_start_time,
                )
                for event in events:
                    # We got the event back!
                    if event.tag == event_tag:
                        event_match = True
                        break
            event_count += 1
            time.sleep(0.5)