Merge pull request #227 from barneysowood/fix-test-suite

Fix test suite and move to pytest-salt-factories

.github/workflows/test.yaml

@@ -13,8 +13,6 @@ jobs:
       fail-fast: false
       matrix:
         py:
-          - "3.5"
-          - "3.6"
           - "3.7"
           - "3.8"
         netapi:
@@ -24,11 +22,6 @@ jobs:
           - "v3004.2"
           - "v3005.1"
           - "master"
-        exclude:
-          - salt: master
-            py: "3.5"
-          - salt: master
-            py: "3.6"
     steps:
       - name: Setup python for test ${{ matrix.py }}
         uses: actions/setup-python@v4

tests/conftest.py

@@ -2,7 +2,6 @@
 from __future__ import absolute_import, unicode_literals, print_function
 
 # Import python libraries
-import distutils.spawn
 import logging
 import os.path
 import shutil
@@ -11,48 +10,86 @@ import tempfile
 import textwrap
 
 # Import Salt Libraries
-import salt.client
-import salt.config
 import salt.utils.yaml as yaml
 
 # Import pytest libraries
 import pytest
-from pytestsalt.utils import SaltDaemonScriptBase, start_daemon, get_unused_localhost_port
+from pytestskipmarkers.utils import ports
+from saltfactories.utils import random_string, running_username
 
 # Import Pepper libraries
 import pepper
 import pepper.script
 
-DEFAULT_MASTER_ID = 'pytest-salt-master'
-DEFAULT_MINION_ID = 'pytest-salt-minion'
 
 log = logging.getLogger(__name__)
 
 
 @pytest.fixture(scope='session')
-def install_sshd_server():
-    if distutils.spawn.find_executable('sshd'):
-        return
-    __opts__ = salt.config.minion_config('tests/minion.conf')
-    __opts__['file_client'] = 'local'
-    caller = salt.client.Caller(mopts=__opts__)
-    caller.cmd('pkg.install', 'openssh-server')
+def sshd_config_dir(salt_factories):
+    config_dir = salt_factories.get_root_dir_for_daemon("sshd")
+    yield config_dir
+    shutil.rmtree(str(config_dir), ignore_errors=True)
 
 
-class SaltApi(SaltDaemonScriptBase):
-    '''
-    Class which runs the salt-api daemon
-    '''
-
-    def get_script_args(self):
-        return ['-l', 'quiet']
+@pytest.fixture(scope='session')
+def session_sshd_server(salt_factories, sshd_config_dir, session_master):
+    sshd_config_dict = {
+        "Protocol": "2",
+        # Turn strict modes off so that we can operate in /tmp
+        "StrictModes": "no",
+        # Logging
+        "SyslogFacility": "AUTH",
+        "LogLevel": "INFO",
+        # Authentication:
+        "LoginGraceTime": "120",
+        "PermitRootLogin": "without-password",
+        "PubkeyAuthentication": "yes",
+        # Don't read the user's ~/.rhosts and ~/.shosts files
+        "IgnoreRhosts": "yes",
+        "HostbasedAuthentication": "no",
+        # To enable empty passwords, change to yes (NOT RECOMMENDED)
+        "PermitEmptyPasswords": "no",
+        # Change to yes to enable challenge-response passwords (beware issues with
+        # some PAM modules and threads)
+        "ChallengeResponseAuthentication": "no",
+        # Change to no to disable tunnelled clear text passwords
+        "PasswordAuthentication": "no",
+        "X11Forwarding": "no",
+        "X11DisplayOffset": "10",
+        "PrintMotd": "no",
+        "PrintLastLog": "yes",
+        "TCPKeepAlive": "yes",
+        "AcceptEnv": "LANG LC_*",
+        "UsePAM": "yes",
+    }
+    factory = salt_factories.get_sshd_daemon(
+        sshd_config_dict=sshd_config_dict,
+        config_dir=sshd_config_dir,
+    )
+    with factory.started():
+        yield factory
 
-    def get_check_ports(self):
-        if 'rest_cherrypy' in self.config:
-            return [self.config['rest_cherrypy']['port']]
 
-        if 'rest_tornado' in self.config:
-            return [self.config['rest_tornado']['port']]
+@pytest.fixture(scope='session')
+def session_ssh_roster_config(session_sshd_server, session_master):
+    roster_contents = """
+    localhost:
+      host: 127.0.0.1
+      port: {}
+      user: {}
+      priv: {}
+      mine_functions:
+        test.arg: ['itworked']
+    """.format(
+        session_sshd_server.listen_port,
+        running_username(),
+        session_sshd_server.client_key
+    )
+    with pytest.helpers.temp_file(
+        "roster", roster_contents, session_master.config_dir
+    ) as roster_file:
+        yield roster_file
 
 
 @pytest.fixture(scope='session')
@@ -60,7 +97,7 @@ def salt_api_port():
     '''
     Returns an unused localhost port for the api port
     '''
-    return get_unused_localhost_port()
+    return ports.get_unused_localhost_port()
 
 
 @pytest.fixture(scope='session')
@@ -121,7 +158,7 @@ def output_file():
 
 
 @pytest.fixture(params=['/run', '/login'])
-def pepper_cli(request, session_salt_api, salt_api_port, output_file, install_sshd_server, session_sshd_server):
+def pepper_cli(request, session_salt_api, salt_api_port, output_file, session_sshd_server):
     '''
     Wrapper to invoke Pepper with common params and inside an empty env
     '''
@@ -154,6 +191,20 @@ def pepper_cli(request, session_salt_api, salt_api_port, output_file, install_ss
     return _run_pepper_cli
 
 
+@pytest.fixture(scope='session')
+def session_master_factory(request, salt_factories, session_master_config_overrides):
+    return salt_factories.salt_master_daemon(
+        random_string("master-"),
+        overrides=session_master_config_overrides
+    )
+
+
+@pytest.fixture(scope='session')
+def session_master(session_master_factory):
+    with session_master_factory.started():
+        yield session_master_factory
+
+
 @pytest.fixture(scope='session')
 def session_master_config_overrides(request, salt_api_port, salt_api_backend):
     return {
@@ -175,69 +226,40 @@ def session_master_config_overrides(request, salt_api_port, salt_api_backend):
         'token_expire': 94670856,
         'ignore_host_keys': True,
         'ssh_wipe': True,
+        'netapi_enable_clients': [
+            'local',
+            'local_async',
+            'local_subset',
+            'ssh',
+            'runner',
+            'runner_async',
+            'wheel',
+            'wheel_async',
+            'run'
+        ]
     }
 
 
 @pytest.fixture(scope='session')
-def session_api_log_prefix(master_id):
-    return 'salt-api/{0}'.format(master_id)
+def session_minion_factory(session_master_factory):
+    """Return a factory for a randomly named minion connected to master."""
+    minion_factory = session_master_factory.salt_minion_daemon(random_string("minion-"))
+    minion_factory.after_terminate(
+        pytest.helpers.remove_stale_minion_key, session_master_factory, minion_factory.id
+    )
+    return minion_factory
 
 
 @pytest.fixture(scope='session')
-def cli_api_script_name():
-    '''
-    Return the CLI script basename
-    '''
-    return 'salt-api'
-
-
-@pytest.yield_fixture(scope='session')
-def session_salt_api_before_start():
-    '''
-    This fixture should be overridden if you need to do
-    some preparation and clean up work before starting
-    the salt-api and after ending it.
-    '''
-    # Prep routines go here
-
-    # Start the salt-api
-    yield
-
-    # Clean routines go here
-
-
-@pytest.yield_fixture(scope='session')
-def session_salt_api_after_start(session_salt_api):
-    '''
-    This fixture should be overridden if you need to do
-    some preparation and clean up work after starting
-    the salt-api and before ending it.
-    '''
-    # Prep routines go here
-
-    # Resume test execution
-    yield
-
-    # Clean routines go here
+def session_minion(session_master, session_minion_factory):  # noqa
+    assert session_master.is_running()
+    with session_minion_factory.started():
+        yield session_minion_factory
 
 
 @pytest.fixture(scope='session')
-def _salt_fail_hard(request, salt_fail_hard):
-    '''
-    Return the salt fail hard value
-    '''
-    fail_hard = request.config.getoption('salt_fail_hard')
-    if fail_hard is not None:
-        # We were passed --salt-fail-hard as a CLI option
-        return fail_hard
-
-    # The salt fail hard was not passed as a CLI option
-    fail_hard = request.config.getini('salt_fail_hard')
-    if fail_hard != []:
-        # We were passed salt_fail_hard as a INI option
-        return fail_hard
-
-    return salt_fail_hard
+def session_minion_id(session_minion):
+    return session_minion.id
 
 
 @pytest.fixture(scope='session')
@@ -257,91 +279,15 @@ def salt_api_backend(request):
 
 
 @pytest.fixture(scope='session')
-def master_id(salt_master_id_counter):
-    '''
-    Returns the master id
-    '''
-    return DEFAULT_MASTER_ID + '-{0}'.format(salt_master_id_counter())
+def session_salt_api_factory(session_master_factory):
+    return session_master_factory.salt_api_daemon()
 
 
 @pytest.fixture(scope='session')
-def minion_id(salt_minion_id_counter):
-    '''
-    Returns the minion id
-    '''
-    return DEFAULT_MINION_ID + '-{0}'.format(salt_minion_id_counter())
-
-
-@pytest.fixture(scope='session')
-def session_salt_api(request,
-                     session_salt_minion,
-                     session_master_id,
-                     session_master_config,
-                     session_salt_api_before_start,  # pylint: disable=unused-argument
-                     session_api_log_prefix,
-                     cli_api_script_name,
-                     log_server,
-                     _cli_bin_dir,
-                     session_conf_dir):
-    '''
-    Returns a running salt-api
-    '''
-    return start_daemon(request,
-                        daemon_name='salt-api',
-                        daemon_id=session_master_id,
-                        daemon_log_prefix=session_api_log_prefix,
-                        daemon_cli_script_name=cli_api_script_name,
-                        daemon_config=session_master_config,
-                        daemon_config_dir=session_conf_dir,
-                        daemon_class=SaltApi,
-                        bin_dir_path=_cli_bin_dir,
-                        start_timeout=30)
-
-
-@pytest.fixture(scope='session')
-def session_sshd_config_lines(session_sshd_port):
-    '''
-    Return a list of lines which will make the sshd_config file
-    '''
-    return [
-        'Port {0}'.format(session_sshd_port),
-        'ListenAddress 127.0.0.1',
-        'Protocol 2',
-        'UsePrivilegeSeparation yes',
-        '# Turn strict modes off so that we can operate in /tmp',
-        'StrictModes no',
-        '# Logging',
-        'SyslogFacility AUTH',
-        'LogLevel INFO',
-        '# Authentication:',
-        'LoginGraceTime 120',
-        'PermitRootLogin without-password',
-        'StrictModes yes',
-        'PubkeyAuthentication yes',
-        '#AuthorizedKeysFile	%h/.ssh/authorized_keys',
-        '#AuthorizedKeysFile	key_test.pub',
-        '# Don\'t read the user\'s ~/.rhosts and ~/.shosts files',
-        'IgnoreRhosts yes',
-        '# similar for protocol version 2',
-        'HostbasedAuthentication no',
-        '#IgnoreUserKnownHosts yes',
-        '# To enable empty passwords, change to yes (NOT RECOMMENDED)',
-        'PermitEmptyPasswords no',
-        '# Change to yes to enable challenge-response passwords (beware issues with',
-        '# some PAM modules and threads)',
-        'ChallengeResponseAuthentication no',
-        '# Change to no to disable tunnelled clear text passwords',
-        'PasswordAuthentication no',
-        'X11Forwarding no',
-        'X11DisplayOffset 10',
-        'PrintMotd no',
-        'PrintLastLog yes',
-        'TCPKeepAlive yes',
-        '#UseLogin no',
-        'AcceptEnv LANG LC_*',
-        'Subsystem sftp /usr/lib/openssh/sftp-server',
-        '#UsePAM yes',
-    ]
+def session_salt_api(session_master, session_salt_api_factory):
+    assert session_master.is_running()
+    with session_salt_api_factory.started():
+        yield session_salt_api_factory
 
 
 def pytest_addoption(parser):

tests/integration/test_clients.py

@@ -1,8 +1,7 @@
 # -*- coding: utf-8 -*-
 import json
-import os
+import pathlib
 import pytest
-import sys
 
 
 def test_local_bad_opts(pepper_cli):
@@ -17,7 +16,7 @@ def test_local_bad_opts(pepper_cli):
 
 
 @pytest.mark.xfail(
-    pytest.config.getoption("--salt-api-backend") == "rest_tornado",
+    'config.getoption("--salt-api-backend") == "rest_tornado"',
     reason="timeout kwarg isnt popped until next version of salt/tornado"
 )
 def test_runner_client(pepper_cli):
@@ -30,34 +29,35 @@ def test_runner_client(pepper_cli):
 
 
 @pytest.mark.xfail(
-    pytest.config.getoption("--salt-api-backend") == "rest_tornado",
+    'config.getoption("--salt-api-backend") == "rest_tornado"',
     reason="wheelClient unimplemented for now on tornado",
 )
-def test_wheel_client_arg(pepper_cli, session_minion_id):
+def test_wheel_client_arg(pepper_cli, session_minion):
     ret = pepper_cli('--client=wheel', 'minions.connected')
-    assert ret == ['pytest-session-salt-minion-0']
+    assert ret == [session_minion.id]
 
 
 @pytest.mark.xfail(
-    pytest.config.getoption("--salt-api-backend") == "rest_tornado",
+    'config.getoption("--salt-api-backend") == "rest_tornado"',
     reason="wheelClient unimplemented for now on tornado",
 )
-def test_wheel_client_kwargs(pepper_cli, session_master_config_file):
+def test_wheel_client_kwargs(pepper_cli, session_master):
     ret = pepper_cli(
         '--client=wheel', 'config.update_config', 'file_name=pepper',
         'yaml_contents={0}'.format(json.dumps({"timeout": 5})),
     )
     assert ret == 'Wrote pepper.conf'
-    assert os.path.isfile('{0}.d/pepper.conf'.format(session_master_config_file))
+
+    default_include_dir = pathlib.Path(session_master.config['default_include']).parent
+    pepper_config = (pathlib.Path(session_master.config_dir) / default_include_dir / 'pepper.conf')
+    assert pepper_config.exists
 
 
 @pytest.mark.xfail(
-    pytest.config.getoption("--salt-api-backend") == "rest_tornado",
+    'config.getoption("--salt-api-backend") == "rest_tornado"',
     reason="sshClient unimplemented for now on tornado",
 )
-@pytest.mark.xfail(sys.version_info >= (3, 0),
-                   reason='Broken with python3 right now')
-def test_ssh_client(pepper_cli, session_roster_config, session_roster_config_file):
+def test_ssh_client(pepper_cli, session_ssh_roster_config):
     ret = pepper_cli('--client=ssh', '*', 'test.ping')
     assert ret['ssh']['localhost']['return'] is True
 

tests/integration/test_vanilla.py

@@ -9,7 +9,7 @@ def test_local(pepper_cli, session_minion_id):
 
 
 @pytest.mark.xfail(
-    pytest.config.getoption("--salt-api-backend") == "rest_tornado",
+    'config.getoption("--salt-api-backend") == "rest_tornado"',
     reason="this is broken in rest_tornado until future release",
 )
 def test_long_local(pepper_cli, session_minion_id):

tests/requirements.txt

@@ -1,11 +1,10 @@
 mock
-pytest>=3.5.0,<4.0.0
+pytest>=6.0.0
 pytest-rerunfailures
 pytest-cov
-git+https://github.com/vmware-archive/pytest-salt@master#egg=pytest-salt
-tornado==6.1
+pytest-salt-factories==0.912.2
 CherryPy
 setuptools_scm
 pyzmq<=20.0.0 ; python_version < "3.6"
 pyzmq>=17.0.0 ; python_version < "3.9"
-pyzmq>19.0.2 ; python_version >= "3.9"
+pyzmq>19.0.2 ; python_version >= "3.9"

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py{3.5,3.6,3.7,3.8}-{cherrypy,tornado}-{v3004.2,v3005.1,master},coverage,flake8
+envlist = py{3.7,3.8}-{cherrypy,tornado}-{v3004.2,v3005.1,master},coverage,flake8
 skip_missing_interpreters = true
 skipsdist = false
 
@@ -7,7 +7,9 @@ skipsdist = false
 passenv = TOXENV, CI, TRAVIS, TRAVIS_*, CODECOV_*
 deps = -r{toxinidir}/tests/requirements.txt
     v3004.2: salt<3004.2
+    v3004.2: jinja2<3.1
     v3005.1: salt<3005.1
+    py3.7-{cherrypy,tornado}-{v3004.2,v3005.1}: importlib-metadata<5.0.0
     master: git+https://github.com/saltstack/salt.git@master#egg=salt
 
 changedir = {toxinidir}
@@ -49,7 +51,7 @@ changedir = {toxinidir}/htmlcov
 commands = python -m http.server
 
 [pytest]
-addopts = --showlocals --log-file /tmp/pepper-runtests.log --no-print-logs -ra
+addopts = --showlocals --log-file /tmp/pepper-runtests.log --show-capture=no -ra
 testpaths = tests
 norecursedirs = .git .tox
 usefixtures = pepperconfig