salt/tests/unit/utils/test_stringutils.py

# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals
import re
import sys
import textwrap

# Import Salt libs
from tests.support.mock import patch
from tests.support.unit import TestCase, LOREM_IPSUM
import salt.utils.stringutils

# Import 3rd-party libs
from salt.ext import six
from salt.ext.six.moves import builtins, range  # pylint: disable=redefined-builtin

UNICODE = '中国語 (繁体)'
STR = BYTES = UNICODE.encode('utf-8')
# This is an example of a unicode string with й constructed using two separate
# code points. Do not modify it.
EGGS = '\u044f\u0438\u0306\u0446\u0430'

LATIN1_UNICODE = 'räksmörgås'
LATIN1_BYTES = LATIN1_UNICODE.encode('latin-1')

DOUBLE_TXT = '''\
# set variable identifying the chroot you work in (used in the prompt below)
if [ -z "$debian_chroot" ] && [ -r /etc/debian_chroot ]; then
    debian_chroot=$(cat /etc/debian_chroot)
fi
'''

SINGLE_TXT = '''\
# set variable identifying the chroot you work in (used in the prompt below)
if [ -z '$debian_chroot' ] && [ -r /etc/debian_chroot ]; then
    debian_chroot=$(cat /etc/debian_chroot)
fi
'''

SINGLE_DOUBLE_TXT = '''\
# set variable identifying the chroot you work in (used in the prompt below)
if [ -z '$debian_chroot' ] && [ -r /etc/debian_chroot ]; then
    debian_chroot=$(cat /etc/debian_chroot)
fi

# set variable identifying the chroot you work in (used in the prompt below)
if [ -z "$debian_chroot" ] && [ -r /etc/debian_chroot ]; then
    debian_chroot=$(cat /etc/debian_chroot)
fi
'''

SINGLE_DOUBLE_SAME_LINE_TXT = '''\
# set variable identifying the chroot you work in (used in the prompt below)
if [ -z '$debian_chroot' ] && [ -r "/etc/debian_chroot" ]; then
    debian_chroot=$(cat /etc/debian_chroot)
fi
'''

MATCH = '''\
# set variable identifying the chroot you work in (used in the prompt below)
if [ -z '$debian_chroot' ] && [ -r /etc/debian_chroot ]; then
    debian_chroot=$(cat /etc/debian_chroot)
fi


# set variable identifying the chroot you work in (used in the prompt below)
if [ -z "$debian_chroot" ] && [ -r /etc/debian_chroot ]; then
    debian_chroot=$(cat /etc/debian_chroot)
fi


# set variable identifying the chroot you work in (used in the prompt below)
if [ -z "$debian_chroot" ] && [ -r /etc/debian_chroot ]; then
    debian_chroot=$(cat /etc/debian_chroot)
fi


# set variable identifying the chroot you work in (used in the prompt below)
if [ -z '$debian_chroot' ] && [ -r /etc/debian_chroot ]; then
    debian_chroot=$(cat /etc/debian_chroot)
fi


# set variable identifying the chroot you work in (used in the prompt below)
if [ -z '$debian_chroot' ] && [ -r "/etc/debian_chroot" ]; then
    debian_chroot=$(cat /etc/debian_chroot)
fi
'''


class TestBuildWhitespaceRegex(TestCase):

    def test_single_quotes(self):
        regex = salt.utils.stringutils.build_whitespace_split_regex(SINGLE_TXT)
        self.assertTrue(re.search(regex, MATCH))

    def test_double_quotes(self):
        regex = salt.utils.stringutils.build_whitespace_split_regex(DOUBLE_TXT)
        self.assertTrue(re.search(regex, MATCH))

    def test_single_and_double_quotes(self):
        regex = salt.utils.stringutils.build_whitespace_split_regex(SINGLE_DOUBLE_TXT)
        self.assertTrue(re.search(regex, MATCH))

    def test_issue_2227(self):
        regex = salt.utils.stringutils.build_whitespace_split_regex(SINGLE_DOUBLE_SAME_LINE_TXT)
        self.assertTrue(re.search(regex, MATCH))


class StringutilsTestCase(TestCase):
    def test_contains_whitespace(self):
        does_contain_whitespace = 'A brown fox jumped over the red hen.'
        does_not_contain_whitespace = 'Abrownfoxjumpedovertheredhen.'

        self.assertTrue(salt.utils.stringutils.contains_whitespace(does_contain_whitespace))
        self.assertFalse(salt.utils.stringutils.contains_whitespace(does_not_contain_whitespace))

    def test_to_num(self):
        self.assertEqual(7, salt.utils.stringutils.to_num('7'))
        self.assertIsInstance(salt.utils.stringutils.to_num('7'), int)
        self.assertEqual(7, salt.utils.stringutils.to_num('7.0'))
        self.assertIsInstance(salt.utils.stringutils.to_num('7.0'), float)
        self.assertEqual(salt.utils.stringutils.to_num('Seven'), 'Seven')
        self.assertIsInstance(salt.utils.stringutils.to_num('Seven'), six.text_type)

    def test_to_none(self):
        self.assertIsNone(salt.utils.stringutils.to_none(''))
        self.assertIsNone(salt.utils.stringutils.to_none('  '))
        # Ensure that we do not inadvertently convert certain strings or 0 to None
        self.assertIsNotNone(salt.utils.stringutils.to_none('None'))
        self.assertIsNotNone(salt.utils.stringutils.to_none(0))

    def test_is_binary(self):
        self.assertFalse(salt.utils.stringutils.is_binary(LOREM_IPSUM))
        # Also test bytestring
        self.assertFalse(
            salt.utils.stringutils.is_binary(
                salt.utils.stringutils.is_binary(LOREM_IPSUM)
            )
        )

        zero_str = '{0}{1}'.format(LOREM_IPSUM, '\0')
        self.assertTrue(salt.utils.stringutils.is_binary(zero_str))
        # Also test bytestring
        self.assertTrue(
            salt.utils.stringutils.is_binary(
                salt.utils.stringutils.to_bytes(zero_str)
            )
        )

        # To to ensure safe exit if str passed doesn't evaluate to True
        self.assertFalse(salt.utils.stringutils.is_binary(''))
        self.assertFalse(salt.utils.stringutils.is_binary(b''))

        nontext = 3 * (''.join([chr(x) for x in range(1, 32) if x not in (8, 9, 10, 12, 13)]))
        almost_bin_str = '{0}{1}'.format(LOREM_IPSUM[:100], nontext[:42])
        self.assertFalse(salt.utils.stringutils.is_binary(almost_bin_str))
        # Also test bytestring
        self.assertFalse(
            salt.utils.stringutils.is_binary(
                salt.utils.stringutils.to_bytes(almost_bin_str)
            )
        )

        bin_str = almost_bin_str + '\x01'
        self.assertTrue(salt.utils.stringutils.is_binary(bin_str))
        # Also test bytestring
        self.assertTrue(
            salt.utils.stringutils.is_binary(
                salt.utils.stringutils.to_bytes(bin_str)
            )
        )

    def test_to_str(self):
        for x in (123, (1, 2, 3), [1, 2, 3], {1: 23}, None):
            self.assertRaises(TypeError, salt.utils.stringutils.to_str, x)
        if six.PY3:
            self.assertEqual(salt.utils.stringutils.to_str('plugh'), 'plugh')
            self.assertEqual(salt.utils.stringutils.to_str('áéíóúý', 'utf-8'), 'áéíóúý')
            self.assertEqual(salt.utils.stringutils.to_str(BYTES, 'utf-8'), UNICODE)
            self.assertEqual(salt.utils.stringutils.to_str(bytearray(BYTES), 'utf-8'), UNICODE)
            # Test situation when a minion returns incorrect utf-8 string because of... million reasons
            ut2 = b'\x9c'
            self.assertRaises(UnicodeDecodeError, salt.utils.stringutils.to_str, ut2, 'utf-8')
            self.assertEqual(salt.utils.stringutils.to_str(ut2, 'utf-8', 'replace'), u'\ufffd')
            self.assertRaises(UnicodeDecodeError, salt.utils.stringutils.to_str, bytearray(ut2), 'utf-8')
            self.assertEqual(salt.utils.stringutils.to_str(bytearray(ut2), 'utf-8', 'replace'), u'\ufffd')
        else:
            self.assertEqual(salt.utils.stringutils.to_str('plugh'), str('plugh'))  # future lint: disable=blacklisted-function
            self.assertEqual(salt.utils.stringutils.to_str('áéíóúý', 'utf-8'), 'áéíóúý'.encode('utf-8'))
            self.assertEqual(salt.utils.stringutils.to_str(UNICODE, 'utf-8'), STR)
            self.assertEqual(salt.utils.stringutils.to_str(bytearray(STR), 'utf-8'), STR)

            # Test utf-8 fallback with Windows default codepage
            with patch.object(builtins, '__salt_system_encoding__', 'CP1252'):
                self.assertEqual(salt.utils.stringutils.to_str('Ψ'), 'Ψ'.encode('utf-8'))

    def test_to_bytes(self):
        for x in (123, (1, 2, 3), [1, 2, 3], {1: 23}, None):
            self.assertRaises(TypeError, salt.utils.stringutils.to_bytes, x)
        if six.PY3:
            self.assertEqual(salt.utils.stringutils.to_bytes('xyzzy'), b'xyzzy')
            self.assertEqual(salt.utils.stringutils.to_bytes(BYTES), BYTES)
            self.assertEqual(salt.utils.stringutils.to_bytes(bytearray(BYTES)), BYTES)
            self.assertEqual(salt.utils.stringutils.to_bytes(UNICODE, 'utf-8'), BYTES)

            # Test utf-8 fallback with ascii default encoding
            with patch.object(builtins, '__salt_system_encoding__', 'ascii'):
                self.assertEqual(salt.utils.stringutils.to_bytes('Ψ'), b'\xce\xa8')
        else:
            self.assertEqual(salt.utils.stringutils.to_bytes('xyzzy'), 'xyzzy')
            self.assertEqual(salt.utils.stringutils.to_bytes(BYTES), BYTES)
            self.assertEqual(salt.utils.stringutils.to_bytes(bytearray(BYTES)), BYTES)
            self.assertEqual(salt.utils.stringutils.to_bytes(UNICODE, 'utf-8'), BYTES)

    def test_to_unicode(self):
        self.assertEqual(
            salt.utils.stringutils.to_unicode(
                EGGS,
                normalize=True
            ),
            'яйца'
        )
        self.assertNotEqual(
            salt.utils.stringutils.to_unicode(
                EGGS,
                normalize=False
            ),
            'яйца'
        )

        self.assertEqual(
            salt.utils.stringutils.to_unicode(
                LATIN1_BYTES, encoding='latin-1'
            ),
            LATIN1_UNICODE
        )

        if six.PY3:
            self.assertEqual(salt.utils.stringutils.to_unicode('plugh'), 'plugh')
            self.assertEqual(salt.utils.stringutils.to_unicode('áéíóúý'), 'áéíóúý')
            self.assertEqual(salt.utils.stringutils.to_unicode(BYTES, 'utf-8'), UNICODE)
            self.assertEqual(salt.utils.stringutils.to_unicode(bytearray(BYTES), 'utf-8'), UNICODE)
        else:
            self.assertEqual(salt.utils.stringutils.to_unicode(str('xyzzy'), 'utf-8'), 'xyzzy')  # future lint: disable=blacklisted-function
            self.assertEqual(salt.utils.stringutils.to_unicode(BYTES, 'utf-8'), UNICODE)

            # Test that unicode chars are decoded properly even when using
            # locales which are not UTF-8 compatible
            with patch.object(builtins, '__salt_system_encoding__', 'ascii'):
                self.assertEqual(salt.utils.stringutils.to_unicode('Ψ'.encode('utf-8')), 'Ψ')
            with patch.object(builtins, '__salt_system_encoding__', 'CP1252'):
                self.assertEqual(salt.utils.stringutils.to_unicode('Ψ'.encode('utf-8')), 'Ψ')

    def test_to_unicode_multi_encoding(self):
        result = salt.utils.stringutils.to_unicode(LATIN1_BYTES, encoding=('utf-8', 'latin1'))
        assert result == LATIN1_UNICODE

    def test_build_whitespace_split_regex(self):
        # With 3.7+,  re.escape only escapes special characters, no longer
        # escaping all characters other than ASCII letters, numbers and
        # underscores.  This includes commas.
        if sys.version_info >= (3, 7):
            expected_regex = '(?m)^(?:[\\s]+)?Lorem(?:[\\s]+)?ipsum(?:[\\s]+)?dolor(?:[\\s]+)?sit(?:[\\s]+)?amet,' \
                             '(?:[\\s]+)?$'
        else:
            expected_regex = '(?m)^(?:[\\s]+)?Lorem(?:[\\s]+)?ipsum(?:[\\s]+)?dolor(?:[\\s]+)?sit(?:[\\s]+)?amet\\,' \
                             '(?:[\\s]+)?$'
        ret = salt.utils.stringutils.build_whitespace_split_regex(' '.join(LOREM_IPSUM.split()[:5]))
        self.assertEqual(ret, expected_regex)

    def test_get_context(self):
        expected_context = textwrap.dedent('''\
            ---
            Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque eget urna a arcu lacinia sagittis.
            Sed scelerisque, lacus eget malesuada vestibulum, justo diam facilisis tortor, in sodales dolor
            [...]
            ---''')
        ret = salt.utils.stringutils.get_context(LOREM_IPSUM, 1, num_lines=1)
        self.assertEqual(ret, expected_context)

    def test_get_context_has_enough_context(self):
        template = '1\n2\n3\n4\n5\n6\n7\n8\n9\na\nb\nc\nd\ne\nf'
        context = salt.utils.stringutils.get_context(template, 8)
        expected = '---\n[...]\n3\n4\n5\n6\n7\n8\n9\na\nb\nc\nd\n[...]\n---'
        self.assertEqual(expected, context)

    def test_get_context_at_top_of_file(self):
        template = '1\n2\n3\n4\n5\n6\n7\n8\n9\na\nb\nc\nd\ne\nf'
        context = salt.utils.stringutils.get_context(template, 1)
        expected = '---\n1\n2\n3\n4\n5\n6\n[...]\n---'
        self.assertEqual(expected, context)

    def test_get_context_at_bottom_of_file(self):
        template = '1\n2\n3\n4\n5\n6\n7\n8\n9\na\nb\nc\nd\ne\nf'
        context = salt.utils.stringutils.get_context(template, 15)
        expected = '---\n[...]\na\nb\nc\nd\ne\nf\n---'
        self.assertEqual(expected, context)

    def test_get_context_2_context_lines(self):
        template = '1\n2\n3\n4\n5\n6\n7\n8\n9\na\nb\nc\nd\ne\nf'
        context = salt.utils.stringutils.get_context(template, 8, num_lines=2)
        expected = '---\n[...]\n6\n7\n8\n9\na\n[...]\n---'
        self.assertEqual(expected, context)

    def test_get_context_with_marker(self):
        template = '1\n2\n3\n4\n5\n6\n7\n8\n9\na\nb\nc\nd\ne\nf'
        context = salt.utils.stringutils.get_context(template, 8, num_lines=2, marker=' <---')
        expected = '---\n[...]\n6\n7\n8 <---\n9\na\n[...]\n---'
        self.assertEqual(expected, context)

    def test_expr_match(self):
        val = 'foo/bar/baz'
        # Exact match
        self.assertTrue(salt.utils.stringutils.expr_match(val, val))
        # Glob match
        self.assertTrue(salt.utils.stringutils.expr_match(val, 'foo/*/baz'))
        # Glob non-match
        self.assertFalse(salt.utils.stringutils.expr_match(val, 'foo/*/bar'))
        # Regex match
        self.assertTrue(salt.utils.stringutils.expr_match(val, r'foo/\w+/baz'))
        # Regex non-match
        self.assertFalse(salt.utils.stringutils.expr_match(val, r'foo/\w/baz'))

    def test_check_whitelist_blacklist(self):
        '''
        Ensure that whitelist matching works on both PY2 and PY3
        '''
        whitelist = ['one/two/three', r'web[0-9]']
        blacklist = ['four/five/six', r'web[5-9]']

        # Tests with string whitelist/blacklist
        self.assertFalse(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web_one',
                whitelist=whitelist[1],
                blacklist=None,
            )
        )
        self.assertFalse(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web_one',
                whitelist=whitelist[1],
                blacklist=[],
            )
        )
        self.assertTrue(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web1',
                whitelist=whitelist[1],
                blacklist=None,
            )
        )
        self.assertTrue(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web1',
                whitelist=whitelist[1],
                blacklist=[],
            )
        )
        self.assertFalse(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web5',
                whitelist=None,
                blacklist=blacklist[1],
            )
        )
        self.assertFalse(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web5',
                whitelist=[],
                blacklist=blacklist[1],
            )
        )
        self.assertTrue(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web_five',
                whitelist=None,
                blacklist=blacklist[1],
            )
        )
        self.assertTrue(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web_five',
                whitelist=[],
                blacklist=blacklist[1],
            )
        )
        self.assertFalse(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web5',
                whitelist=whitelist[1],
                blacklist=blacklist[1],
            )
        )
        self.assertTrue(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web4',
                whitelist=whitelist[1],
                blacklist=blacklist[1],
            )
        )

        # Tests with list whitelist/blacklist
        self.assertFalse(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web_one',
                whitelist=whitelist,
                blacklist=None,
            )
        )
        self.assertFalse(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web_one',
                whitelist=whitelist,
                blacklist=[],
            )
        )
        self.assertTrue(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web1',
                whitelist=whitelist,
                blacklist=None,
            )
        )
        self.assertTrue(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web1',
                whitelist=whitelist,
                blacklist=[],
            )
        )
        self.assertFalse(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web5',
                whitelist=None,
                blacklist=blacklist,
            )
        )
        self.assertFalse(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web5',
                whitelist=[],
                blacklist=blacklist,
            )
        )
        self.assertTrue(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web_five',
                whitelist=None,
                blacklist=blacklist,
            )
        )
        self.assertTrue(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web_five',
                whitelist=[],
                blacklist=blacklist,
            )
        )
        self.assertFalse(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web5',
                whitelist=whitelist,
                blacklist=blacklist,
            )
        )
        self.assertTrue(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web4',
                whitelist=whitelist,
                blacklist=blacklist,
            )
        )

        # Tests with set whitelist/blacklist
        self.assertFalse(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web_one',
                whitelist=set(whitelist),
                blacklist=None,
            )
        )
        self.assertFalse(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web_one',
                whitelist=set(whitelist),
                blacklist=set(),
            )
        )
        self.assertTrue(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web1',
                whitelist=set(whitelist),
                blacklist=None,
            )
        )
        self.assertTrue(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web1',
                whitelist=set(whitelist),
                blacklist=set(),
            )
        )
        self.assertFalse(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web5',
                whitelist=None,
                blacklist=set(blacklist),
            )
        )
        self.assertFalse(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web5',
                whitelist=set(),
                blacklist=set(blacklist),
            )
        )
        self.assertTrue(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web_five',
                whitelist=None,
                blacklist=set(blacklist),
            )
        )
        self.assertTrue(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web_five',
                whitelist=set(),
                blacklist=set(blacklist),
            )
        )
        self.assertFalse(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web5',
                whitelist=set(whitelist),
                blacklist=set(blacklist),
            )
        )
        self.assertTrue(
            salt.utils.stringutils.check_whitelist_blacklist(
                'web4',
                whitelist=set(whitelist),
                blacklist=set(blacklist),
            )
        )

        # Test with invalid type for whitelist/blacklist
        self.assertRaises(
            TypeError,
            salt.utils.stringutils.check_whitelist_blacklist,
            'foo', whitelist=123
        )
        self.assertRaises(
            TypeError,
            salt.utils.stringutils.check_whitelist_blacklist,
            'foo', blacklist=123
        )