salt/tests/support/parser/__init__.py

# -*- coding: utf-8 -*-
'''
    tests.support.parser
    ~~~~~~~~~~~~~~~~~~~~

    Salt Tests CLI access classes

    :codeauthor: Pedro Algarvio (pedro@algarvio.me)
    :copyright: Copyright 2013-2017 by the SaltStack Team, see AUTHORS for more details
    :license: Apache 2.0, see LICENSE for more details.
'''
# pylint: disable=repr-flag-used-in-string

from __future__ import absolute_import, print_function
import fnmatch
import os
import sys
import time
import signal
import shutil
import logging
import platform
import optparse
import re
import tempfile
import traceback
import subprocess
import warnings
from functools import partial
from collections import namedtuple

import tests.support.paths
from tests.support import processes
from tests.support.unit import TestLoader, TextTestRunner
from tests.support.xmlunit import HAS_XMLRUNNER, XMLTestRunner

# Import 3rd-party libs
from salt.ext import six
import salt.utils.data
import salt.utils.files
import salt.utils.path
import salt.utils.platform
import salt.utils.stringutils
import salt.utils.yaml

try:
    from tests.support.ext import console
    WIDTH, HEIGHT = console.getTerminalSize()
    PNUM = WIDTH
except Exception:  # pylint: disable=broad-except
    PNUM = 70

log = logging.getLogger(__name__)

# This is a completely random and meaningful number intended to identify our
# own signal triggering.
WEIRD_SIGNAL_NUM = -45654


def __global_logging_exception_handler(exc_type, exc_value, exc_traceback,
                                       _logger=logging.getLogger(__name__),
                                       _stderr=sys.__stderr__,
                                       _format_exception=traceback.format_exception):
    '''
    This function will log all python exceptions.
    '''
    if exc_type.__name__ == "KeyboardInterrupt":
        # Call the original sys.excepthook
        sys.__excepthook__(exc_type, exc_value, exc_traceback)
        return

    # Log the exception
    try:
        msg = (
            'An un-handled exception was caught by salt\'s testing global exception handler:\n{}: {}\n{}'.format(
                exc_type.__name__,
                exc_value,
                ''.join(_format_exception(exc_type, exc_value, exc_traceback)).strip()
            )
        )
    except Exception:  # pylint: disable=broad-except
        msg = (
            'An un-handled exception was caught by salt-testing\'s global exception handler:\n{}: {}\n'
            '(UNABLE TO FORMAT TRACEBACK)'.format(
                exc_type.__name__,
                exc_value,
            )
        )
    try:
        _logger(__name__).error(msg)
    except Exception:  # pylint: disable=broad-except
        # Python is shutting down and logging has been set to None already
        try:
            _stderr.write(msg + '\n')
        except Exception:  # pylint: disable=broad-except
            # We have also lost reference to sys.__stderr__ ?!
            print(msg)

    # Call the original sys.excepthook
    try:
        sys.__excepthook__(exc_type, exc_value, exc_traceback)
    except Exception:  # pylint: disable=broad-except
        # Python is shutting down and sys has been set to None already
        pass


# Set our own exception handler as the one to use
sys.excepthook = __global_logging_exception_handler

TestsuiteResult = namedtuple('TestsuiteResult', ['header', 'errors', 'skipped', 'failures', 'passed'])
TestResult = namedtuple('TestResult', ['id', 'reason'])


def print_header(header, sep='~', top=True, bottom=True, inline=False,
                 centered=False, width=PNUM):
    '''
    Allows some pretty printing of headers on the console, either with a
    "ruler" on bottom and/or top, inline, centered, etc.
    '''
    if top and not inline:
        print(sep * width)

    if centered and not inline:
        fmt = u'{0:^{width}}'
    elif inline and not centered:
        fmt = u'{0:{sep}<{width}}'
    elif inline and centered:
        fmt = u'{0:{sep}^{width}}'
    else:
        fmt = u'{0}'
    print(fmt.format(header, sep=sep, width=width))

    if bottom and not inline:
        print(sep * width)


class SaltTestingParser(optparse.OptionParser):
    support_docker_execution = False
    support_destructive_tests_selection = False
    support_expensive_tests_selection = False
    source_code_basedir = None

    _known_interpreters = {
        'salttest/arch': 'python2',
        'salttest/centos-5': 'python2.6',
        'salttest/centos-6': 'python2.6',
        'salttest/debian-7': 'python2.7',
        'salttest/opensuse-12.3': 'python2.7',
        'salttest/ubuntu-12.04': 'python2.7',
        'salttest/ubuntu-12.10': 'python2.7',
        'salttest/ubuntu-13.04': 'python2.7',
        'salttest/ubuntu-13.10': 'python2.7',
        'salttest/py3': 'python3'
    }

    def __init__(self, testsuite_directory, *args, **kwargs):
        if kwargs.pop('html_output_from_env', None) is not None or \
                kwargs.pop('html_output_dir', None) is not None:
            warnings.warn(
                'The unit tests HTML support was removed from {0}. Please '
                'stop passing \'html_output_dir\' or \'html_output_from_env\' '
                'as arguments to {0}'.format(self.__class__.__name__),
                category=DeprecationWarning,
                stacklevel=2
            )

        # Get XML output settings
        xml_output_dir_env_var = kwargs.pop(
            'xml_output_from_env',
            'XML_TESTS_OUTPUT_DIR'
        )
        xml_output_dir = kwargs.pop('xml_output_dir', None)

        if xml_output_dir_env_var in os.environ:
            xml_output_dir = os.environ.get(xml_output_dir_env_var)
        if not xml_output_dir:
            xml_output_dir = os.path.join(
                tempfile.gettempdir() if platform.system() != 'Darwin' else '/tmp',
                'xml-tests-output'
            )
        self.xml_output_dir = xml_output_dir

        # Get the desired logfile to use while running tests
        self.tests_logfile = kwargs.pop('tests_logfile', None)

        optparse.OptionParser.__init__(self, *args, **kwargs)
        self.testsuite_directory = testsuite_directory
        self.testsuite_results = []

        self.test_selection_group = optparse.OptionGroup(
            self,
            'Tests Selection Options',
            'Select which tests are to be executed'
        )
        if self.support_destructive_tests_selection is True:
            self.test_selection_group.add_option(
                '--run-destructive',
                action='store_true',
                default=False,
                help=('Run destructive tests. These tests can include adding '
                      'or removing users from your system for example. '
                      'Default: %default')
            )
        if self.support_expensive_tests_selection is True:
            self.test_selection_group.add_option(
                '--run-expensive',
                action='store_true',
                default=False,
                help=('Run expensive tests. Expensive tests are any tests that, '
                      'once configured, cost money to run, such as creating or '
                      'destroying cloud instances on a cloud provider.')
            )

        self.test_selection_group.add_option(
            '-n',
            '--name',
            dest='name',
            action='append',
            default=[],
            help=('Specific test name to run. A named test is the module path '
                  'relative to the tests directory')
        )
        self.test_selection_group.add_option(
            '--names-file',
            dest='names_file',
            default=None,
            help=('The location of a newline delimited file of test names to '
                  'run')
        )
        self.test_selection_group.add_option(
            '--from-filenames',
            dest='from_filenames',
            action='append',
            default=None,
            help=('Pass a comma-separated list of file paths, and any '
                  'unit/integration test module which corresponds to the '
                  'specified file(s) will be run. For example, a path of '
                  'salt/modules/git.py would result in unit.modules.test_git '
                  'and integration.modules.test_git being run. Absolute paths '
                  'are assumed to be files containing relative paths, one per '
                  'line. Providing the paths in a file can help get around '
                  'shell character limits when the list of files is long.')
        )
        self.test_selection_group.add_option(
            '--filename-map',
            dest='filename_map',
            default=None,
            help=('Path to a YAML file mapping paths/path globs to a list '
                  'of test names to run. See tests/filename_map.yml '
                  'for example usage (when --from-filenames is used, this '
                  'map file will be the default one used).')
        )
        self.add_option_group(self.test_selection_group)

        if self.support_docker_execution is True:
            self.docked_selection_group = optparse.OptionGroup(
                self,
                'Docked Tests Execution',
                'Run the tests suite under a Docker container. This allows, '
                'for example, to run destructive tests on your machine '
                'without actually breaking it in any way.'
            )
            self.docked_selection_group.add_option(
                '--docked',
                default=None,
                metavar='CONTAINER',
                help='Run the tests suite in the chosen Docker container'
            )
            self.docked_selection_group.add_option(
                '--docked-interpreter',
                default=None,
                metavar='PYTHON_INTERPRETER',
                help='The python binary name to use when calling the tests '
                     'suite.'
            )
            self.docked_selection_group.add_option(
                '--docked-skip-delete',
                default=False,
                action='store_true',
                help='Skip docker container deletion on exit. Default: False'
            )
            self.docked_selection_group.add_option(
                '--docked-skip-delete-on-errors',
                default=False,
                action='store_true',
                help='Skip docker container deletion on exit if errors '
                     'occurred. Default: False'
            )
            self.docked_selection_group.add_option(
                '--docker-binary',
                help='The docker binary on the host system. Default: %default',
                default='/usr/bin/docker',
            )
            self.add_option_group(self.docked_selection_group)

        self.output_options_group = optparse.OptionGroup(
            self, 'Output Options'
        )
        self.output_options_group.add_option(
            '-F',
            '--fail-fast',
            dest='failfast',
            default=False,
            action='store_true',
            help='Stop on first failure'
        )
        self.output_options_group.add_option(
            '-v',
            '--verbose',
            dest='verbosity',
            default=1,
            action='count',
            help='Verbose test runner output'
        )
        self.output_options_group.add_option(
            '--output-columns',
            default=PNUM,
            type=int,
            help='Number of maximum columns to use on the output'
        )
        self.output_options_group.add_option(
            '--tests-logfile',
            default=self.tests_logfile,
            help='The path to the tests suite logging logfile'
        )
        if self.xml_output_dir is not None:
            self.output_options_group.add_option(
                '-x',
                '--xml',
                '--xml-out',
                dest='xml_out',
                default=False,
                help='XML test runner output(Output directory: {0})'.format(
                    self.xml_output_dir
                )
            )
        self.output_options_group.add_option(
            '--no-report',
            default=False,
            action='store_true',
            help='Do NOT show the overall tests result'
        )
        self.add_option_group(self.output_options_group)

        self.fs_cleanup_options_group = optparse.OptionGroup(
            self, 'File system cleanup Options'
        )
        self.fs_cleanup_options_group.add_option(
            '--clean',
            dest='clean',
            default=True,
            action='store_true',
            help=('Clean up test environment before and after running the '
                  'tests suite (default behaviour)')
        )
        self.fs_cleanup_options_group.add_option(
            '--no-clean',
            dest='clean',
            action='store_false',
            help=('Don\'t clean up test environment before and after the '
                  'tests suite execution (speed up test process)')
        )
        self.add_option_group(self.fs_cleanup_options_group)
        self.setup_additional_options()

    @staticmethod
    def _expand_paths(paths):
        '''
        Expand any comma-separated lists of paths, and return a set of all
        paths to ensure there are no duplicates.
        '''
        ret = set()
        for path in paths:
            for item in [x.strip() for x in path.split(',')]:
                if not item:
                    continue
                elif os.path.isabs(item):
                    try:
                        with salt.utils.files.fopen(item, 'rb') as fp_:
                            for line in fp_:
                                line = salt.utils.stringutils.to_unicode(line.strip())
                                if os.path.isabs(line):
                                    log.warning(
                                        'Invalid absolute path %s in %s, '
                                        'ignoring', line, item
                                    )
                                else:
                                    ret.add(line)
                    except (IOError, OSError) as exc:
                        log.error('Failed to read from %s: %s', item, exc)
                else:
                    ret.add(item)
        return ret

    @property
    def _test_mods(self):
        '''
        Use the test_mods generator to get all of the test module names, and
        then store them in a set so that further references to this attribute
        will not need to re-walk the test dir.
        '''
        try:
            return self.__test_mods
        except AttributeError:
            self.__test_mods = set(tests.support.paths.list_test_mods())
            return self.__test_mods

    def _map_files(self, files):
        '''
        Map the passed paths to test modules, returning a set of the mapped
        module names.
        '''
        ret = set()

        if self.options.filename_map is not None:
            try:
                with salt.utils.files.fopen(self.options.filename_map) as fp_:
                    filename_map = salt.utils.yaml.safe_load(fp_)
            except Exception as exc:  # pylint: disable=broad-except
                raise RuntimeError(
                    'Failed to load filename map: {0}'.format(exc)
                )
        else:
            filename_map = {}

        def _add(comps):
            '''
            Helper to add unit and integration tests matching a given mod path
            '''
            mod_relname = '.'.join(comps)
            ret.update(
                x for x in
                ['.'.join(('unit', mod_relname)),
                 '.'.join(('integration', mod_relname)),
                 '.'.join(('multimaster', mod_relname))]
                if x in self._test_mods
            )

        # First, try a path match
        for path in files:
            match = re.match(r'^(salt/|tests/(unit|integration|multimaster)/)(.+\.py)$', path)
            if match:
                comps = match.group(3).split('/')

                # Find matches for a source file
                if match.group(1) == 'salt/':
                    if comps[-1] == '__init__.py':
                        comps.pop(-1)
                        comps[-1] = 'test_' + comps[-1]
                    else:
                        comps[-1] = 'test_{0}'.format(comps[-1][:-3])

                    # Direct name matches
                    _add(comps)

                    # State matches for execution modules of the same name
                    # (e.g. unit.states.test_archive if
                    # unit.modules.test_archive is being run)
                    try:
                        if comps[-2] == 'modules':
                            comps[-2] = 'states'
                            _add(comps)
                    except IndexError:
                        # Not an execution module. This is either directly in
                        # the salt/ directory, or salt/something/__init__.py
                        pass

                # Make sure to run a test module if it's been modified
                elif match.group(1).startswith('tests/'):
                    comps.insert(0, match.group(2))
                    if fnmatch.fnmatch(comps[-1], 'test_*.py'):
                        comps[-1] = comps[-1][:-3]
                        test_name = '.'.join(comps)
                        if test_name in self._test_mods:
                            ret.add(test_name)

        # Next, try the filename_map
        for path_expr in filename_map:
            for filename in files:
                if salt.utils.stringutils.expr_match(filename, path_expr):
                    ret.update(filename_map[path_expr])
                    break

        if any(x.startswith('integration.proxy.') for x in ret):
            # Ensure that the salt-proxy daemon is started for these tests.
            self.options.proxy = True

        if any(x.startswith('integration.ssh.') for x in ret):
            # Ensure that an ssh daemon is started for these tests.
            self.options.ssh = True

        return ret

    def parse_args(self, args=None, values=None):
        self.options, self.args = optparse.OptionParser.parse_args(self, args, values)

        file_names = []
        if self.options.names_file:
            with open(self.options.names_file, 'rb') as fp_:  # pylint: disable=resource-leakage
                for line in fp_.readlines():
                    if six.PY2:
                        file_names.append(line.strip())
                    else:
                        file_names.append(
                            line.decode(__salt_system_encoding__).strip())

        if self.args:
            for fpath in self.args:
                if os.path.isfile(fpath) and \
                        fpath.endswith('.py') and \
                        os.path.basename(fpath).startswith('test_'):
                    if fpath in file_names:
                        self.options.name.append(fpath)
                    continue
                self.exit(status=1, msg='\'{}\' is not a valid test module\n'.format(fpath))

        if self.options.from_filenames is not None:
            self.options.from_filenames = self._expand_paths(self.options.from_filenames)

            # Locate the default map file if one was not passed
            if self.options.filename_map is None:
                self.options.filename_map = salt.utils.path.join(
                    tests.support.paths.TESTS_DIR,
                    'filename_map.yml'
                )

            self.options.name.extend(self._map_files(self.options.from_filenames))

        if self.options.name and file_names:
            self.options.name = list(set(self.options.name).intersection(file_names))
        elif file_names:
            self.options.name = file_names

        print_header(u'', inline=True, width=self.options.output_columns)
        self.pre_execution_cleanup()

        if self.support_docker_execution and self.options.docked is not None:
            if self.source_code_basedir is None:
                raise RuntimeError(
                    'You need to define the \'source_code_basedir\' attribute '
                    'in \'{0}\'.'.format(self.__class__.__name__)
                )

            if '/' not in self.options.docked:
                self.options.docked = 'salttest/{0}'.format(
                    self.options.docked
                )

            if self.options.docked_interpreter is None:
                self.options.docked_interpreter = self._known_interpreters.get(
                    self.options.docked, 'python'
                )

            # No more processing should be done. We'll exit with the return
            # code we get from the docker container execution
            self.exit(self.run_suite_in_docker())

        # Validate options after checking that we're not goint to execute the
        # tests suite under a docker container
        self._validate_options()

        print(' * Current Directory: {0}'.format(os.getcwd()))
        print(' * Test suite is running under PID {0}'.format(os.getpid()))

        self._setup_logging()
        try:
            return (self.options, self.args)
        finally:
            print_header(u'', inline=True, width=self.options.output_columns)

    def setup_additional_options(self):
        '''
        Subclasses should add additional options in this overridden method
        '''

    def _validate_options(self):
        '''
        Validate the default available options
        '''
        if self.xml_output_dir is not None and self.options.xml_out and HAS_XMLRUNNER is False:
            self.error(
                '\'--xml\' is not available. The xmlrunner library is not '
                'installed.'
            )

        if self.options.xml_out:
            # Override any environment setting with the passed value
            self.xml_output_dir = self.options.xml_out

        if self.xml_output_dir is not None and self.options.xml_out:
            if not os.path.isdir(self.xml_output_dir):
                os.makedirs(self.xml_output_dir)
            os.environ['TESTS_XML_OUTPUT_DIR'] = self.xml_output_dir
            print(
                ' * Generated unit test XML reports will be stored '
                'at {0!r}'.format(self.xml_output_dir)
            )

        self.validate_options()

        if self.support_destructive_tests_selection and not os.environ.get('DESTRUCTIVE_TESTS', None):
            # Set the required environment variable in order to know if
            # destructive tests should be executed or not.
            os.environ['DESTRUCTIVE_TESTS'] = str(self.options.run_destructive)

        if self.support_expensive_tests_selection and not os.environ.get('EXPENSIVE_TESTS', None):
            # Set the required environment variable in order to know if
            # expensive tests should be executed or not.
            os.environ['EXPENSIVE_TESTS'] = str(self.options.run_expensive)

    def validate_options(self):
        '''
        Validate the provided options. Override this method to run your own
        validation procedures.
        '''

    def _setup_logging(self):
        '''
        Setup python's logging system to work with/for the tests suite
        '''
        # Setup tests logging
        formatter = logging.Formatter(
            '%(asctime)s,%(msecs)03.0f [%(name)-5s:%(lineno)-4d]'
            '[%(levelname)-8s] %(message)s',
            datefmt='%H:%M:%S'
        )
        if not hasattr(logging, 'TRACE'):
            logging.TRACE = 5
            logging.addLevelName(logging.TRACE, 'TRACE')
        if not hasattr(logging, 'GARBAGE'):
            logging.GARBAGE = 1
            logging.addLevelName(logging.GARBAGE, 'GARBAGE')

        # Default logging level: ERROR
        logging.root.setLevel(logging.NOTSET)

        log_levels_to_evaluate = [
            logging.ERROR,  # Default log level
        ]
        if self.options.tests_logfile:
            filehandler = logging.FileHandler(
                mode='w',           # Not preserved between re-runs
                filename=self.options.tests_logfile,
                encoding='utf-8',
            )
            # The logs of the file are the most verbose possible
            filehandler.setLevel(logging.DEBUG)
            filehandler.setFormatter(formatter)
            logging.root.addHandler(filehandler)
            log_levels_to_evaluate.append(logging.DEBUG)

            print(' * Logging tests on {0}'.format(self.options.tests_logfile))

        # With greater verbosity we can also log to the console
        if self.options.verbosity >= 2:
            consolehandler = logging.StreamHandler(sys.stderr)
            consolehandler.setFormatter(formatter)
            if self.options.verbosity >= 6:     # -vvvvv
                logging_level = logging.GARBAGE
            elif self.options.verbosity == 5:   # -vvvv
                logging_level = logging.TRACE
            elif self.options.verbosity == 4:   # -vvv
                logging_level = logging.DEBUG
            elif self.options.verbosity == 3:   # -vv
                logging_level = logging.INFO
            else:
                logging_level = logging.ERROR
            log_levels_to_evaluate.append(logging_level)
            os.environ['TESTS_LOG_LEVEL'] = str(self.options.verbosity)  # future lint: disable=blacklisted-function
            consolehandler.setLevel(logging_level)
            logging.root.addHandler(consolehandler)
            log.info('Runtests logging has been setup')

        os.environ['TESTS_MIN_LOG_LEVEL_NAME'] = logging.getLevelName(min(log_levels_to_evaluate))

    def pre_execution_cleanup(self):
        '''
        Run any initial clean up operations. If sub-classed, don't forget to
        call SaltTestingParser.pre_execution_cleanup(self) from the overridden
        method.
        '''
        if self.options.clean is True:
            for path in (self.xml_output_dir,):
                if path is None:
                    continue
                if os.path.isdir(path):
                    shutil.rmtree(path)

    def run_suite(self, path, display_name, suffix='test_*.py',
                  load_from_name=False, additional_test_dirs=None, failfast=False):
        '''
        Execute a unit test suite
        '''
        loaded_custom = False
        loader = TestLoader()
        try:
            if load_from_name:
                tests = loader.loadTestsFromName(display_name)
            else:
                if additional_test_dirs is None or self.testsuite_directory.startswith(path):
                    tests = loader.discover(path, suffix, self.testsuite_directory)
                else:
                    tests = loader.discover(path, suffix)
                    loaded_custom = True
        except (AttributeError, ImportError):
            print('Could not locate test \'{0}\'. Exiting.'.format(display_name))
            sys.exit(1)

        if additional_test_dirs and not loaded_custom:
            for test_dir in additional_test_dirs:
                additional_tests = loader.discover(test_dir, suffix, test_dir)
                tests.addTests(additional_tests)

        header = '{0} Tests'.format(display_name)
        print_header('Starting {0}'.format(header),
                     width=self.options.output_columns)

        if self.options.xml_out:
            runner = XMLTestRunner(
                stream=sys.stdout,
                output=self.xml_output_dir,
                verbosity=self.options.verbosity,
                failfast=failfast,
            ).run(tests)
        else:
            runner = TextTestRunner(
                stream=sys.stdout,
                verbosity=self.options.verbosity,
                failfast=failfast
            ).run(tests)

        errors = []
        skipped = []
        failures = []
        for testcase, reason in runner.errors:
            errors.append(TestResult(testcase.id(), reason))
        for testcase, reason in runner.skipped:
            skipped.append(TestResult(testcase.id(), reason))
        for testcase, reason in runner.failures:
            failures.append(TestResult(testcase.id(), reason))
        self.testsuite_results.append(
            TestsuiteResult(header,
                            errors,
                            skipped,
                            failures,
                            runner.testsRun - len(errors + skipped + failures))
        )
        success = runner.wasSuccessful()
        del loader
        del runner
        return success

    def print_overall_testsuite_report(self):
        '''
        Print a nicely formatted report about the test suite results
        '''
        print()
        print_header(
            u'  Overall Tests Report  ', sep=u'=', centered=True, inline=True,
            width=self.options.output_columns
        )

        failures = errors = skipped = passed = 0
        no_problems_found = True
        for results in self.testsuite_results:
            failures += len(results.failures)
            errors += len(results.errors)
            skipped += len(results.skipped)
            passed += results.passed

            if not results.failures and not results.errors and not results.skipped:
                continue

            no_problems_found = False

            print_header(
                u'*** {0}  '.format(results.header), sep=u'*', inline=True,
                width=self.options.output_columns
            )
            if results.skipped:
                print_header(
                    u' --------  Skipped Tests  ', sep='-', inline=True,
                    width=self.options.output_columns
                )
                maxlen = len(
                    max([testcase.id for testcase in results.skipped], key=len)
                )
                fmt = u'   -> {0: <{maxlen}}  ->  {1}'
                for testcase in results.skipped:
                    print(fmt.format(testcase.id, testcase.reason, maxlen=maxlen))
                print_header(u' ', sep='-', inline=True,
                             width=self.options.output_columns)

            if results.errors:
                print_header(
                    u' --------  Tests with Errors  ', sep='-', inline=True,
                    width=self.options.output_columns
                )
                for testcase in results.errors:
                    print_header(
                        u'   -> {0}  '.format(testcase.id),
                        sep=u'.', inline=True,
                        width=self.options.output_columns
                    )
                    for line in testcase.reason.rstrip().splitlines():
                        print('       {0}'.format(line.rstrip()))
                    print_header(u'   ', sep=u'.', inline=True,
                                 width=self.options.output_columns)
                print_header(u' ', sep='-', inline=True,
                             width=self.options.output_columns)

            if results.failures:
                print_header(
                    u' --------  Failed Tests  ', sep='-', inline=True,
                    width=self.options.output_columns
                )
                for testcase in results.failures:
                    print_header(
                        u'   -> {0}  '.format(testcase.id),
                        sep=u'.', inline=True,
                        width=self.options.output_columns
                    )
                    for line in testcase.reason.rstrip().splitlines():
                        print('       {0}'.format(line.rstrip()))
                    print_header(u'   ', sep=u'.', inline=True,
                                 width=self.options.output_columns)
                print_header(u' ', sep='-', inline=True,
                             width=self.options.output_columns)

        if no_problems_found:
            print_header(
                u'***  No Problems Found While Running Tests  ',
                sep=u'*', inline=True, width=self.options.output_columns
            )

        print_header(u'', sep=u'=', inline=True,
                     width=self.options.output_columns)
        total = sum([passed, skipped, errors, failures])
        print(
            '{0} (total={1}, skipped={2}, passed={3}, failures={4}, '
            'errors={5}) '.format(
                (errors or failures) and 'FAILED' or 'OK',
                total, skipped, passed, failures, errors
            )
        )
        print_header(
            '  Overall Tests Report  ', sep='=', centered=True, inline=True,
            width=self.options.output_columns
        )

    def post_execution_cleanup(self):
        '''
        Run any final clean-up operations.  If sub-classed, don't forget to
        call SaltTestingParser.post_execution_cleanup(self) from the overridden
        method.
        '''

    def finalize(self, exit_code=0):
        '''
        Run the finalization procedures. Show report, clean-up file-system, etc
        '''
        # Collect any child processes still laying around
        children = processes.collect_child_processes(os.getpid())
        if self.options.no_report is False:
            self.print_overall_testsuite_report()
        self.post_execution_cleanup()
        # Brute force approach to terminate this process and its children
        if children:
            log.info('Terminating test suite child processes: %s', children)
            processes.terminate_process(children=children, kill_children=True)
            children = processes.collect_child_processes(os.getpid())
            if children:
                log.info('Second run at terminating test suite child processes: %s', children)
                processes.terminate_process(children=children, kill_children=True)
        exit_msg = 'Test suite execution finalized with exit code: {}'.format(exit_code)
        log.info(exit_msg)
        self.exit(status=exit_code, msg=exit_msg + '\n')

    def run_suite_in_docker(self):
        '''
        Run the tests suite in a Docker container
        '''
        def stop_running_docked_container(cid, signum=None, frame=None):
            # Allow some time for the container to stop if it's going to be
            # stopped by docker or any signals docker might have received
            time.sleep(0.5)

            print_header('', inline=True, width=self.options.output_columns)

            # Let's check if, in fact, the container is stopped
            scode_call = subprocess.Popen(
                [self.options.docker_binary, 'inspect', '--format={{.State.Running}}', cid],
                env=os.environ.copy(),
                close_fds=True,
                stdout=subprocess.PIPE
            )
            scode_call.wait()
            parsed_scode = scode_call.stdout.read().strip()
            if six.PY3:
                parsed_scode = parsed_scode.decode(__salt_system_encoding__)
            if parsed_scode != 'false':
                # If the container is still running, let's make sure it
                # properly stops
                sys.stdout.write(' * Making sure the container is stopped. CID: ')
                sys.stdout.flush()

                stop_call = subprocess.Popen(
                    [self.options.docker_binary, 'stop', '--time=15', cid],
                    env=os.environ.copy(),
                    close_fds=True,
                    stdout=subprocess.PIPE
                )
                stop_call.wait()
                output = stop_call.stdout.read().strip()
                if six.PY3:
                    output = output.decode(__salt_system_encoding__)
                print(output)
                sys.stdout.flush()
                time.sleep(0.5)

            # Let's get the container's exit code. We can't trust on Popen's
            # returncode because it's not reporting the proper one? Still
            # haven't narrowed it down why.
            sys.stdout.write(' * Container exit code: ')
            sys.stdout.flush()
            rcode_call = subprocess.Popen(
                [self.options.docker_binary, 'inspect', '--format={{.State.ExitCode}}', cid],
                env=os.environ.copy(),
                close_fds=True,
                stdout=subprocess.PIPE
            )
            rcode_call.wait()
            parsed_rcode = rcode_call.stdout.read().strip()
            if six.PY3:
                parsed_rcode = parsed_rcode.decode(__salt_system_encoding__)
            try:
                returncode = int(parsed_rcode)
            except ValueError:
                returncode = -1
            print(parsed_rcode)
            sys.stdout.flush()

            if self.options.docked_skip_delete is False and \
                    (self.options.docked_skip_delete_on_errors is False or
                     (self.options.docked_skip_delete_on_error and returncode == 0)):
                sys.stdout.write(' * Cleaning Up Temporary Docker Container. CID: ')
                sys.stdout.flush()
                cleanup_call = subprocess.Popen(
                    [self.options.docker_binary, 'rm', cid],
                    env=os.environ.copy(),
                    close_fds=True,
                    stdout=subprocess.PIPE
                )
                cleanup_call.wait()
                output = cleanup_call.stdout.read().strip()
                if six.PY3:
                    output = output.decode(__salt_system_encoding__)
                print(output)

            if 'DOCKER_CIDFILE' not in os.environ:
                # The CID file was not created "from the outside", so delete it
                os.unlink(cidfile)

            print_header('', inline=True, width=self.options.output_columns)
            # Finally, EXIT!
            sys.exit(returncode)

        # Let's start the Docker container and run the tests suite there
        if '/' not in self.options.docked:
            container = 'salttest/{0}'.format(self.options.docked)
        else:
            container = self.options.docked

        calling_args = [self.options.docked_interpreter,
                        '/salt-source/tests/runtests.py']
        for option in self._get_all_options():
            if option.dest is None:
                # For example --version
                continue

            if option.dest and (option.dest in ('verbosity',) or
                                option.dest.startswith('docked')):
                # We don't need to pass any docker related arguments inside the
                # container, and verbose will be handled bellow
                continue

            default = self.defaults.get(option.dest)
            value = getattr(self.options, option.dest, default)

            if default == value:
                # This is the default value, no need to pass the option to the
                # parser
                continue

            if option.action.startswith('store_'):
                calling_args.append(option.get_opt_string())

            elif option.action == 'append':
                for val in value is not None and value or default:
                    calling_args.extend([option.get_opt_string(), str(val)])
            elif option.action == 'count':
                calling_args.extend([option.get_opt_string()] * value)
            else:
                calling_args.extend(
                    [option.get_opt_string(),
                     str(value is not None and value or default)]
                )

        if not self.options.run_destructive:
            calling_args.append('--run-destructive')

        if self.options.verbosity > 1:
            calling_args.append(
                '-{0}'.format('v' * (self.options.verbosity - 1))
            )

        sys.stdout.write(' * Docker command: {0}\n'.format(' '.join(calling_args)))
        sys.stdout.write(' * Running the tests suite under the {0!r} docker '
                         'container. CID: '.format(container))
        sys.stdout.flush()

        cidfile = os.environ.get(
            'DOCKER_CIDFILE',
            tempfile.mktemp(prefix='docked-testsuite-', suffix='.cid')
        )
        call = subprocess.Popen(
            [self.options.docker_binary,
             'run',
             # '--rm=true', Do not remove the container automatically, we need
             #              to get information back, even for stopped containers
             '--tty',
             '--interactive',
             '-v',
             '{0}:/salt-source'.format(self.source_code_basedir),
             '-w',
             '/salt-source',
             '-e',
             'SHELL=/bin/sh',
             '-e',
             'COLUMNS={0}'.format(WIDTH),
             '-e',
             'LINES={0}'.format(HEIGHT),
             '--cidfile={0}'.format(cidfile),
             container,
             # We need to pass the runtests.py arguments as a single string so
             # that the start-me-up.sh script can handle them properly
             ' '.join(calling_args),
             ],
            env=os.environ.copy(),
            close_fds=True,
        )

        cid = None
        cid_printed = terminating = exiting = False
        signal_handler_installed = signalled = False

        time.sleep(0.25)

        while True:
            try:
                time.sleep(0.15)
                if cid_printed is False:
                    with open(cidfile) as cidfile_fd:  # pylint: disable=resource-leakage
                        cid = cidfile_fd.read()
                        if cid:
                            print(cid)
                            sys.stdout.flush()
                            cid_printed = True
                            # Install our signal handler to properly shutdown
                            # the docker container
                            for sig in (signal.SIGTERM, signal.SIGINT,
                                        signal.SIGHUP, signal.SIGQUIT):
                                signal.signal(
                                    sig,
                                    partial(stop_running_docked_container, cid)
                                )
                            signal_handler_installed = True

                if exiting:
                    break
                elif terminating and not exiting:
                    exiting = True
                    call.kill()
                    break
                elif signalled and not terminating:
                    terminating = True
                    call.terminate()
                else:
                    call.poll()
                    if call.returncode is not None:
                        # Finished
                        break
            except KeyboardInterrupt:
                print('Caught CTRL-C, exiting...')
                signalled = True
                call.send_signal(signal.SIGINT)

        call.wait()
        time.sleep(0.25)

        # Finish up
        if signal_handler_installed:
            stop_running_docked_container(
                cid,
                signum=(signal.SIGINT if signalled else WEIRD_SIGNAL_NUM)
            )
        else:
            sys.exit(call.returncode)


class SaltTestcaseParser(SaltTestingParser):
    '''
    Option parser to run one or more ``unittest.case.TestCase``, ie, no
    discovery involved.
    '''
    def __init__(self, *args, **kwargs):
        SaltTestingParser.__init__(self, None, *args, **kwargs)
        self.usage = '%prog [options]'
        self.option_groups.remove(self.test_selection_group)
        if self.has_option('--xml-out'):
            self.remove_option('--xml-out')

    def get_prog_name(self):
        return '{0} {1}'.format(sys.executable.split(os.sep)[-1], sys.argv[0])

    def run_testcase(self, testcase):
        '''
        Run one or more ``unittest.case.TestCase``
        '''
        header = ''
        loader = TestLoader()
        if isinstance(testcase, list):
            for case in testcase:
                tests = loader.loadTestsFromTestCase(case)
        else:
            tests = loader.loadTestsFromTestCase(testcase)

        if not isinstance(testcase, list):
            header = '{0} Tests'.format(testcase.__name__)
            print_header('Starting {0}'.format(header),
                         width=self.options.output_columns)

        runner = TextTestRunner(
            verbosity=self.options.verbosity,
            failfast=self.options.failfast,
        ).run(tests)
        self.testsuite_results.append((header, runner))
        return runner.wasSuccessful()