salt/tests/unit/utils/test_data.py

# -*- coding: utf-8 -*-
"""
Tests for salt.utils.data
"""

# Import Python libs
from __future__ import absolute_import, print_function, unicode_literals

import logging

# Import Salt libs
import salt.utils.data
import salt.utils.stringutils
from salt.ext import six

# Import 3rd party libs
from salt.ext.six.moves import (  # pylint: disable=import-error,redefined-builtin
    builtins,
)
from salt.utils.odict import OrderedDict
from tests.support.mock import patch
from tests.support.unit import LOREM_IPSUM, TestCase

log = logging.getLogger(__name__)
_b = lambda x: x.encode("utf-8")
_s = lambda x: salt.utils.stringutils.to_str(x, normalize=True)
# Some randomized data that will not decode
BYTES = b"1\x814\x10"

# This is an example of a unicode string with й constructed using two separate
# code points. Do not modify it.
EGGS = "\u044f\u0438\u0306\u0446\u0430"


class DataTestCase(TestCase):
    test_data = [
        "unicode_str",
        _b("питон"),
        123,
        456.789,
        True,
        False,
        None,
        EGGS,
        BYTES,
        [123, 456.789, _b("спам"), True, False, None, EGGS, BYTES],
        (987, 654.321, _b("яйца"), EGGS, None, (True, EGGS, BYTES)),
        {
            _b("str_key"): _b("str_val"),
            None: True,
            123: 456.789,
            EGGS: BYTES,
            _b("subdict"): {
                "unicode_key": EGGS,
                _b("tuple"): (123, "hello", _b("world"), True, EGGS, BYTES),
                _b("list"): [456, _b("спам"), False, EGGS, BYTES],
            },
        },
        OrderedDict([(_b("foo"), "bar"), (123, 456), (EGGS, BYTES)]),
    ]

    def test_sorted_ignorecase(self):
        test_list = ["foo", "Foo", "bar", "Bar"]
        expected_list = ["bar", "Bar", "foo", "Foo"]
        self.assertEqual(salt.utils.data.sorted_ignorecase(test_list), expected_list)

    def test_mysql_to_dict(self):
        test_mysql_output = [
            "+----+------+-----------+------+---------+------+-------+------------------+",
            "| Id | User | Host      | db   | Command | Time | State | Info             |",
            "+----+------+-----------+------+---------+------+-------+------------------+",
            "|  7 | root | localhost | NULL | Query   |    0 | init  | show processlist |",
            "+----+------+-----------+------+---------+------+-------+------------------+",
        ]

        ret = salt.utils.data.mysql_to_dict(test_mysql_output, "Info")
        expected_dict = {
            "show processlist": {
                "Info": "show processlist",
                "db": "NULL",
                "State": "init",
                "Host": "localhost",
                "Command": "Query",
                "User": "root",
                "Time": 0,
                "Id": 7,
            }
        }

        self.assertDictEqual(ret, expected_dict)

    def test_subdict_match(self):
        test_two_level_dict = {"foo": {"bar": "baz"}}
        test_two_level_comb_dict = {"foo": {"bar": "baz:woz"}}
        test_two_level_dict_and_list = {
            "abc": ["def", "ghi", {"lorem": {"ipsum": [{"dolor": "sit"}]}}],
        }
        test_three_level_dict = {"a": {"b": {"c": "v"}}}

        self.assertTrue(
            salt.utils.data.subdict_match(test_two_level_dict, "foo:bar:baz")
        )
        # In test_two_level_comb_dict, 'foo:bar' corresponds to 'baz:woz', not
        # 'baz'. This match should return False.
        self.assertFalse(
            salt.utils.data.subdict_match(test_two_level_comb_dict, "foo:bar:baz")
        )
        # This tests matching with the delimiter in the value part (in other
        # words, that the path 'foo:bar' corresponds to the string 'baz:woz').
        self.assertTrue(
            salt.utils.data.subdict_match(test_two_level_comb_dict, "foo:bar:baz:woz")
        )
        # This would match if test_two_level_comb_dict['foo']['bar'] was equal
        # to 'baz:woz:wiz', or if there was more deep nesting. But it does not,
        # so this should return False.
        self.assertFalse(
            salt.utils.data.subdict_match(
                test_two_level_comb_dict, "foo:bar:baz:woz:wiz"
            )
        )
        # This tests for cases when a key path corresponds to a list. The
        # value part 'ghi' should be successfully matched as it is a member of
        # the list corresponding to key path 'abc'. It is somewhat a
        # duplication of a test within test_traverse_dict_and_list, but
        # salt.utils.data.subdict_match() does more than just invoke
        # salt.utils.traverse_list_and_dict() so this particular assertion is a
        # sanity check.
        self.assertTrue(
            salt.utils.data.subdict_match(test_two_level_dict_and_list, "abc:ghi")
        )
        # This tests the use case of a dict embedded in a list, embedded in a
        # list, embedded in a dict. This is a rather absurd case, but it
        # confirms that match recursion works properly.
        self.assertTrue(
            salt.utils.data.subdict_match(
                test_two_level_dict_and_list, "abc:lorem:ipsum:dolor:sit"
            )
        )
        # Test four level dict match for reference
        self.assertTrue(salt.utils.data.subdict_match(test_three_level_dict, "a:b:c:v"))
        # Test regression in 2015.8 where 'a:c:v' would match 'a:b:c:v'
        self.assertFalse(salt.utils.data.subdict_match(test_three_level_dict, "a:c:v"))
        # Test wildcard match
        self.assertTrue(salt.utils.data.subdict_match(test_three_level_dict, "a:*:c:v"))

    def test_subdict_match_with_wildcards(self):
        """
        Tests subdict matching when wildcards are used in the expression
        """
        data = {"a": {"b": {"ç": "d", "é": ["eff", "gee", "8ch"], "ĩ": {"j": "k"}}}}
        assert salt.utils.data.subdict_match(data, "*:*:*:*")
        assert salt.utils.data.subdict_match(data, "a:*:*:*")
        assert salt.utils.data.subdict_match(data, "a:b:*:*")
        assert salt.utils.data.subdict_match(data, "a:b:ç:*")
        assert salt.utils.data.subdict_match(data, "a:b:*:d")
        assert salt.utils.data.subdict_match(data, "a:*:ç:d")
        assert salt.utils.data.subdict_match(data, "*:b:ç:d")
        assert salt.utils.data.subdict_match(data, "*:*:ç:d")
        assert salt.utils.data.subdict_match(data, "*:*:*:d")
        assert salt.utils.data.subdict_match(data, "a:*:*:d")
        assert salt.utils.data.subdict_match(data, "a:b:*:ef*")
        assert salt.utils.data.subdict_match(data, "a:b:*:g*")
        assert salt.utils.data.subdict_match(data, "a:b:*:j:*")
        assert salt.utils.data.subdict_match(data, "a:b:*:j:k")
        assert salt.utils.data.subdict_match(data, "a:b:*:*:k")
        assert salt.utils.data.subdict_match(data, "a:b:*:*:*")

    def test_traverse_dict(self):
        test_two_level_dict = {"foo": {"bar": "baz"}}

        self.assertDictEqual(
            {"not_found": "nope"},
            salt.utils.data.traverse_dict(
                test_two_level_dict, "foo:bar:baz", {"not_found": "nope"}
            ),
        )
        self.assertEqual(
            "baz",
            salt.utils.data.traverse_dict(
                test_two_level_dict, "foo:bar", {"not_found": "not_found"}
            ),
        )

    def test_traverse_dict_and_list(self):
        test_two_level_dict = {"foo": {"bar": "baz"}}
        test_two_level_dict_and_list = {
            "foo": ["bar", "baz", {"lorem": {"ipsum": [{"dolor": "sit"}]}}]
        }

        # Check traversing too far: salt.utils.data.traverse_dict_and_list() returns
        # the value corresponding to a given key path, and baz is a value
        # corresponding to the key path foo:bar.
        self.assertDictEqual(
            {"not_found": "nope"},
            salt.utils.data.traverse_dict_and_list(
                test_two_level_dict, "foo:bar:baz", {"not_found": "nope"}
            ),
        )
        # Now check to ensure that foo:bar corresponds to baz
        self.assertEqual(
            "baz",
            salt.utils.data.traverse_dict_and_list(
                test_two_level_dict, "foo:bar", {"not_found": "not_found"}
            ),
        )
        # Check traversing too far
        self.assertDictEqual(
            {"not_found": "nope"},
            salt.utils.data.traverse_dict_and_list(
                test_two_level_dict_and_list, "foo:bar", {"not_found": "nope"}
            ),
        )
        # Check index 1 (2nd element) of list corresponding to path 'foo'
        self.assertEqual(
            "baz",
            salt.utils.data.traverse_dict_and_list(
                test_two_level_dict_and_list, "foo:1", {"not_found": "not_found"}
            ),
        )
        # Traverse a couple times into dicts embedded in lists
        self.assertEqual(
            "sit",
            salt.utils.data.traverse_dict_and_list(
                test_two_level_dict_and_list,
                "foo:lorem:ipsum:dolor",
                {"not_found": "not_found"},
            ),
        )

        # Traverse and match integer key in a nested dict
        # https://github.com/saltstack/salt/issues/56444
        self.assertEqual(
            "it worked",
            salt.utils.data.traverse_dict_and_list(
                {"foo": {1234: "it worked"}}, "foo:1234", "it didn't work",
            ),
        )
        # Make sure that we properly return the default value when the initial
        # attempt fails and YAML-loading the target key doesn't change its
        # value.
        self.assertEqual(
            "default",
            salt.utils.data.traverse_dict_and_list(
                {"foo": {"baz": "didn't work"}}, "foo:bar", "default",
            ),
        )

    def test_compare_dicts(self):
        ret = salt.utils.data.compare_dicts(old={"foo": "bar"}, new={"foo": "bar"})
        self.assertEqual(ret, {})

        ret = salt.utils.data.compare_dicts(old={"foo": "bar"}, new={"foo": "woz"})
        expected_ret = {"foo": {"new": "woz", "old": "bar"}}
        self.assertDictEqual(ret, expected_ret)

    def test_compare_lists_no_change(self):
        ret = salt.utils.data.compare_lists(
            old=[1, 2, 3, "a", "b", "c"], new=[1, 2, 3, "a", "b", "c"]
        )
        expected = {}
        self.assertDictEqual(ret, expected)

    def test_compare_lists_changes(self):
        ret = salt.utils.data.compare_lists(
            old=[1, 2, 3, "a", "b", "c"], new=[1, 2, 4, "x", "y", "z"]
        )
        expected = {"new": [4, "x", "y", "z"], "old": [3, "a", "b", "c"]}
        self.assertDictEqual(ret, expected)

    def test_compare_lists_changes_new(self):
        ret = salt.utils.data.compare_lists(old=[1, 2, 3], new=[1, 2, 3, "x", "y", "z"])
        expected = {"new": ["x", "y", "z"]}
        self.assertDictEqual(ret, expected)

    def test_compare_lists_changes_old(self):
        ret = salt.utils.data.compare_lists(old=[1, 2, 3, "a", "b", "c"], new=[1, 2, 3])
        expected = {"old": ["a", "b", "c"]}
        self.assertDictEqual(ret, expected)

    def test_decode(self):
        """
        Companion to test_decode_to_str, they should both be kept up-to-date
        with one another.

        NOTE: This uses the lambda "_b" defined above in the global scope,
        which encodes a string to a bytestring, assuming utf-8.
        """
        expected = [
            "unicode_str",
            "питон",
            123,
            456.789,
            True,
            False,
            None,
            "яйца",
            BYTES,
            [123, 456.789, "спам", True, False, None, "яйца", BYTES],
            (987, 654.321, "яйца", "яйца", None, (True, "яйца", BYTES)),
            {
                "str_key": "str_val",
                None: True,
                123: 456.789,
                "яйца": BYTES,
                "subdict": {
                    "unicode_key": "яйца",
                    "tuple": (123, "hello", "world", True, "яйца", BYTES),
                    "list": [456, "спам", False, "яйца", BYTES],
                },
            },
            OrderedDict([("foo", "bar"), (123, 456), ("яйца", BYTES)]),
        ]

        ret = salt.utils.data.decode(
            self.test_data,
            keep=True,
            normalize=True,
            preserve_dict_class=True,
            preserve_tuples=True,
        )
        self.assertEqual(ret, expected)

        # The binary data in the data structure should fail to decode, even
        # using the fallback, and raise an exception.
        self.assertRaises(
            UnicodeDecodeError,
            salt.utils.data.decode,
            self.test_data,
            keep=False,
            normalize=True,
            preserve_dict_class=True,
            preserve_tuples=True,
        )

        # Now munge the expected data so that we get what we would expect if we
        # disable preservation of dict class and tuples
        expected[10] = [987, 654.321, "яйца", "яйца", None, [True, "яйца", BYTES]]
        expected[11]["subdict"]["tuple"] = [123, "hello", "world", True, "яйца", BYTES]
        expected[12] = {"foo": "bar", 123: 456, "яйца": BYTES}

        ret = salt.utils.data.decode(
            self.test_data,
            keep=True,
            normalize=True,
            preserve_dict_class=False,
            preserve_tuples=False,
        )
        self.assertEqual(ret, expected)

        # Now test single non-string, non-data-structure items, these should
        # return the same value when passed to this function
        for item in (123, 4.56, True, False, None):
            log.debug("Testing decode of %s", item)
            self.assertEqual(salt.utils.data.decode(item), item)

        # Test single strings (not in a data structure)
        self.assertEqual(salt.utils.data.decode("foo"), "foo")
        self.assertEqual(salt.utils.data.decode(_b("bar")), "bar")
        self.assertEqual(salt.utils.data.decode(EGGS, normalize=True), "яйца")
        self.assertEqual(salt.utils.data.decode(EGGS, normalize=False), EGGS)

        # Test binary blob
        self.assertEqual(salt.utils.data.decode(BYTES, keep=True), BYTES)
        self.assertRaises(UnicodeDecodeError, salt.utils.data.decode, BYTES, keep=False)

    def test_circular_refs_dicts(self):
        test_dict = {"key": "value", "type": "test1"}
        test_dict["self"] = test_dict
        ret = salt.utils.data._remove_circular_refs(ob=test_dict)
        self.assertDictEqual(ret, {"key": "value", "type": "test1", "self": None})

    def test_circular_refs_lists(self):
        test_list = {
            "foo": [],
        }
        test_list["foo"].append((test_list,))
        ret = salt.utils.data._remove_circular_refs(ob=test_list)
        self.assertDictEqual(ret, {"foo": [(None,)]})

    def test_circular_refs_tuple(self):
        test_dup = {"foo": "string 1", "bar": "string 1", "ham": 1, "spam": 1}
        ret = salt.utils.data._remove_circular_refs(ob=test_dup)
        self.assertDictEqual(
            ret, {"foo": "string 1", "bar": "string 1", "ham": 1, "spam": 1}
        )

    def test_decode_to_str(self):
        """
        Companion to test_decode, they should both be kept up-to-date with one
        another.

        NOTE: This uses the lambda "_s" defined above in the global scope,
        which converts the string/bytestring to a str type.
        """
        expected = [
            _s("unicode_str"),
            _s("питон"),
            123,
            456.789,
            True,
            False,
            None,
            _s("яйца"),
            BYTES,
            [123, 456.789, _s("спам"), True, False, None, _s("яйца"), BYTES],
            (987, 654.321, _s("яйца"), _s("яйца"), None, (True, _s("яйца"), BYTES)),
            {
                _s("str_key"): _s("str_val"),
                None: True,
                123: 456.789,
                _s("яйца"): BYTES,
                _s("subdict"): {
                    _s("unicode_key"): _s("яйца"),
                    _s("tuple"): (
                        123,
                        _s("hello"),
                        _s("world"),
                        True,
                        _s("яйца"),
                        BYTES,
                    ),
                    _s("list"): [456, _s("спам"), False, _s("яйца"), BYTES],
                },
            },
            OrderedDict([(_s("foo"), _s("bar")), (123, 456), (_s("яйца"), BYTES)]),
        ]

        ret = salt.utils.data.decode(
            self.test_data,
            keep=True,
            normalize=True,
            preserve_dict_class=True,
            preserve_tuples=True,
            to_str=True,
        )
        self.assertEqual(ret, expected)

        if six.PY3:
            # The binary data in the data structure should fail to decode, even
            # using the fallback, and raise an exception.
            self.assertRaises(
                UnicodeDecodeError,
                salt.utils.data.decode,
                self.test_data,
                keep=False,
                normalize=True,
                preserve_dict_class=True,
                preserve_tuples=True,
                to_str=True,
            )

        # Now munge the expected data so that we get what we would expect if we
        # disable preservation of dict class and tuples
        expected[10] = [
            987,
            654.321,
            _s("яйца"),
            _s("яйца"),
            None,
            [True, _s("яйца"), BYTES],
        ]
        expected[11][_s("subdict")][_s("tuple")] = [
            123,
            _s("hello"),
            _s("world"),
            True,
            _s("яйца"),
            BYTES,
        ]
        expected[12] = {_s("foo"): _s("bar"), 123: 456, _s("яйца"): BYTES}

        ret = salt.utils.data.decode(
            self.test_data,
            keep=True,
            normalize=True,
            preserve_dict_class=False,
            preserve_tuples=False,
            to_str=True,
        )
        self.assertEqual(ret, expected)

        # Now test single non-string, non-data-structure items, these should
        # return the same value when passed to this function
        for item in (123, 4.56, True, False, None):
            log.debug("Testing decode of %s", item)
            self.assertEqual(salt.utils.data.decode(item, to_str=True), item)

        # Test single strings (not in a data structure)
        self.assertEqual(salt.utils.data.decode("foo", to_str=True), _s("foo"))
        self.assertEqual(salt.utils.data.decode(_b("bar"), to_str=True), _s("bar"))

        # Test binary blob
        self.assertEqual(salt.utils.data.decode(BYTES, keep=True, to_str=True), BYTES)
        if six.PY3:
            self.assertRaises(
                UnicodeDecodeError,
                salt.utils.data.decode,
                BYTES,
                keep=False,
                to_str=True,
            )

    def test_decode_fallback(self):
        """
        Test fallback to utf-8
        """
        with patch.object(builtins, "__salt_system_encoding__", "ascii"):
            self.assertEqual(salt.utils.data.decode(_b("яйца")), "яйца")

    def test_encode(self):
        """
        NOTE: This uses the lambda "_b" defined above in the global scope,
        which encodes a string to a bytestring, assuming utf-8.
        """
        expected = [
            _b("unicode_str"),
            _b("питон"),
            123,
            456.789,
            True,
            False,
            None,
            _b(EGGS),
            BYTES,
            [123, 456.789, _b("спам"), True, False, None, _b(EGGS), BYTES],
            (987, 654.321, _b("яйца"), _b(EGGS), None, (True, _b(EGGS), BYTES)),
            {
                _b("str_key"): _b("str_val"),
                None: True,
                123: 456.789,
                _b(EGGS): BYTES,
                _b("subdict"): {
                    _b("unicode_key"): _b(EGGS),
                    _b("tuple"): (123, _b("hello"), _b("world"), True, _b(EGGS), BYTES),
                    _b("list"): [456, _b("спам"), False, _b(EGGS), BYTES],
                },
            },
            OrderedDict([(_b("foo"), _b("bar")), (123, 456), (_b(EGGS), BYTES)]),
        ]

        # Both keep=True and keep=False should work because the BYTES data is
        # already bytes.
        ret = salt.utils.data.encode(
            self.test_data, keep=True, preserve_dict_class=True, preserve_tuples=True
        )
        self.assertEqual(ret, expected)
        ret = salt.utils.data.encode(
            self.test_data, keep=False, preserve_dict_class=True, preserve_tuples=True
        )
        self.assertEqual(ret, expected)

        # Now munge the expected data so that we get what we would expect if we
        # disable preservation of dict class and tuples
        expected[10] = [
            987,
            654.321,
            _b("яйца"),
            _b(EGGS),
            None,
            [True, _b(EGGS), BYTES],
        ]
        expected[11][_b("subdict")][_b("tuple")] = [
            123,
            _b("hello"),
            _b("world"),
            True,
            _b(EGGS),
            BYTES,
        ]
        expected[12] = {_b("foo"): _b("bar"), 123: 456, _b(EGGS): BYTES}

        ret = salt.utils.data.encode(
            self.test_data, keep=True, preserve_dict_class=False, preserve_tuples=False
        )
        self.assertEqual(ret, expected)
        ret = salt.utils.data.encode(
            self.test_data, keep=False, preserve_dict_class=False, preserve_tuples=False
        )
        self.assertEqual(ret, expected)

        # Now test single non-string, non-data-structure items, these should
        # return the same value when passed to this function
        for item in (123, 4.56, True, False, None):
            log.debug("Testing encode of %s", item)
            self.assertEqual(salt.utils.data.encode(item), item)

        # Test single strings (not in a data structure)
        self.assertEqual(salt.utils.data.encode("foo"), _b("foo"))
        self.assertEqual(salt.utils.data.encode(_b("bar")), _b("bar"))

        # Test binary blob, nothing should happen even when keep=False since
        # the data is already bytes
        self.assertEqual(salt.utils.data.encode(BYTES, keep=True), BYTES)
        self.assertEqual(salt.utils.data.encode(BYTES, keep=False), BYTES)

    def test_encode_keep(self):
        """
        Whereas we tested the keep argument in test_decode, it is much easier
        to do a more comprehensive test of keep in its own function where we
        can force the encoding.
        """
        unicode_str = "питон"
        encoding = "ascii"

        # Test single string
        self.assertEqual(
            salt.utils.data.encode(unicode_str, encoding, keep=True), unicode_str
        )
        self.assertRaises(
            UnicodeEncodeError,
            salt.utils.data.encode,
            unicode_str,
            encoding,
            keep=False,
        )

        data = [
            unicode_str,
            [b"foo", [unicode_str], {b"key": unicode_str}, (unicode_str,)],
            {
                b"list": [b"foo", unicode_str],
                b"dict": {b"key": unicode_str},
                b"tuple": (b"foo", unicode_str),
            },
            ([b"foo", unicode_str], {b"key": unicode_str}, (unicode_str,)),
        ]

        # Since everything was a bytestring aside from the bogus data, the
        # return data should be identical. We don't need to test recursive
        # decoding, that has already been tested in test_encode.
        self.assertEqual(
            salt.utils.data.encode(data, encoding, keep=True, preserve_tuples=True),
            data,
        )
        self.assertRaises(
            UnicodeEncodeError,
            salt.utils.data.encode,
            data,
            encoding,
            keep=False,
            preserve_tuples=True,
        )

        for index, _ in enumerate(data):
            self.assertEqual(
                salt.utils.data.encode(
                    data[index], encoding, keep=True, preserve_tuples=True
                ),
                data[index],
            )
            self.assertRaises(
                UnicodeEncodeError,
                salt.utils.data.encode,
                data[index],
                encoding,
                keep=False,
                preserve_tuples=True,
            )

    def test_encode_fallback(self):
        """
        Test fallback to utf-8
        """
        with patch.object(builtins, "__salt_system_encoding__", "ascii"):
            self.assertEqual(salt.utils.data.encode("яйца"), _b("яйца"))
        with patch.object(builtins, "__salt_system_encoding__", "CP1252"):
            self.assertEqual(salt.utils.data.encode("Ψ"), _b("Ψ"))

    def test_repack_dict(self):
        list_of_one_element_dicts = [
            {"dict_key_1": "dict_val_1"},
            {"dict_key_2": "dict_val_2"},
            {"dict_key_3": "dict_val_3"},
        ]
        expected_ret = {
            "dict_key_1": "dict_val_1",
            "dict_key_2": "dict_val_2",
            "dict_key_3": "dict_val_3",
        }
        ret = salt.utils.data.repack_dictlist(list_of_one_element_dicts)
        self.assertDictEqual(ret, expected_ret)

        # Try with yaml
        yaml_key_val_pair = "- key1: val1"
        ret = salt.utils.data.repack_dictlist(yaml_key_val_pair)
        self.assertDictEqual(ret, {"key1": "val1"})

        # Make sure we handle non-yaml junk data
        ret = salt.utils.data.repack_dictlist(LOREM_IPSUM)
        self.assertDictEqual(ret, {})

    def test_stringify(self):
        self.assertRaises(TypeError, salt.utils.data.stringify, 9)
        self.assertEqual(
            salt.utils.data.stringify(
                ["one", "two", str("three"), 4, 5]
            ),  # future lint: disable=blacklisted-function
            ["one", "two", "three", "4", "5"],
        )

    def test_json_query(self):
        # Raises exception if jmespath module is not found
        with patch("salt.utils.data.jmespath", None):
            self.assertRaisesRegex(
                RuntimeError, "requires jmespath", salt.utils.data.json_query, {}, "@"
            )

        # Test search
        user_groups = {
            "user1": {"groups": ["group1", "group2", "group3"]},
            "user2": {"groups": ["group1", "group2"]},
            "user3": {"groups": ["group3"]},
        }
        expression = "*.groups[0]"
        primary_groups = ["group1", "group1", "group3"]
        self.assertEqual(
            sorted(salt.utils.data.json_query(user_groups, expression)), primary_groups
        )


class FilterFalseyTestCase(TestCase):
    """
    Test suite for salt.utils.data.filter_falsey
    """

    def test_nop(self):
        """
        Test cases where nothing will be done.
        """
        # Test with dictionary without recursion
        old_dict = {
            "foo": "bar",
            "bar": {"baz": {"qux": "quux"}},
            "baz": ["qux", {"foo": "bar"}],
        }
        new_dict = salt.utils.data.filter_falsey(old_dict)
        self.assertEqual(old_dict, new_dict)
        # Check returned type equality
        self.assertIs(type(old_dict), type(new_dict))
        # Test dictionary with recursion
        new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3)
        self.assertEqual(old_dict, new_dict)
        # Test with list
        old_list = ["foo", "bar"]
        new_list = salt.utils.data.filter_falsey(old_list)
        self.assertEqual(old_list, new_list)
        # Check returned type equality
        self.assertIs(type(old_list), type(new_list))
        # Test with set
        old_set = set(["foo", "bar"])
        new_set = salt.utils.data.filter_falsey(old_set)
        self.assertEqual(old_set, new_set)
        # Check returned type equality
        self.assertIs(type(old_set), type(new_set))
        # Test with OrderedDict
        old_dict = OrderedDict(
            [
                ("foo", "bar"),
                ("bar", OrderedDict([("qux", "quux")])),
                ("baz", ["qux", OrderedDict([("foo", "bar")])]),
            ]
        )
        new_dict = salt.utils.data.filter_falsey(old_dict)
        self.assertEqual(old_dict, new_dict)
        self.assertIs(type(old_dict), type(new_dict))
        # Test excluding int
        old_list = [0]
        new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type(0)])
        self.assertEqual(old_list, new_list)
        # Test excluding str (or unicode) (or both)
        old_list = [""]
        new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type("")])
        self.assertEqual(old_list, new_list)
        # Test excluding list
        old_list = [[]]
        new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type([])])
        self.assertEqual(old_list, new_list)
        # Test excluding dict
        old_list = [{}]
        new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type({})])
        self.assertEqual(old_list, new_list)

    def test_filter_dict_no_recurse(self):
        """
        Test filtering a dictionary without recursing.
        This will only filter out key-values where the values are falsey.
        """
        old_dict = {
            "foo": None,
            "bar": {"baz": {"qux": None, "quux": "", "foo": []}},
            "baz": ["qux"],
            "qux": {},
            "quux": [],
        }
        new_dict = salt.utils.data.filter_falsey(old_dict)
        expect_dict = {
            "bar": {"baz": {"qux": None, "quux": "", "foo": []}},
            "baz": ["qux"],
        }
        self.assertEqual(expect_dict, new_dict)
        self.assertIs(type(expect_dict), type(new_dict))

    def test_filter_dict_recurse(self):
        """
        Test filtering a dictionary with recursing.
        This will filter out any key-values where the values are falsey or when
        the values *become* falsey after filtering their contents (in case they
        are lists or dicts).
        """
        old_dict = {
            "foo": None,
            "bar": {"baz": {"qux": None, "quux": "", "foo": []}},
            "baz": ["qux"],
            "qux": {},
            "quux": [],
        }
        new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3)
        expect_dict = {"baz": ["qux"]}
        self.assertEqual(expect_dict, new_dict)
        self.assertIs(type(expect_dict), type(new_dict))

    def test_filter_list_no_recurse(self):
        """
        Test filtering a list without recursing.
        This will only filter out items which are falsey.
        """
        old_list = ["foo", None, [], {}, 0, ""]
        new_list = salt.utils.data.filter_falsey(old_list)
        expect_list = ["foo"]
        self.assertEqual(expect_list, new_list)
        self.assertIs(type(expect_list), type(new_list))
        # Ensure nested values are *not* filtered out.
        old_list = [
            "foo",
            ["foo"],
            ["foo", None],
            {"foo": 0},
            {"foo": "bar", "baz": []},
            [{"foo": ""}],
        ]
        new_list = salt.utils.data.filter_falsey(old_list)
        self.assertEqual(old_list, new_list)
        self.assertIs(type(old_list), type(new_list))

    def test_filter_list_recurse(self):
        """
        Test filtering a list with recursing.
        This will filter out any items which are falsey, or which become falsey
        after filtering their contents (in case they are lists or dicts).
        """
        old_list = [
            "foo",
            ["foo"],
            ["foo", None],
            {"foo": 0},
            {"foo": "bar", "baz": []},
            [{"foo": ""}],
        ]
        new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3)
        expect_list = ["foo", ["foo"], ["foo"], {"foo": "bar"}]
        self.assertEqual(expect_list, new_list)
        self.assertIs(type(expect_list), type(new_list))

    def test_filter_set_no_recurse(self):
        """
        Test filtering a set without recursing.
        Note that a set cannot contain unhashable types, so recursion is not possible.
        """
        old_set = set(["foo", None, 0, ""])
        new_set = salt.utils.data.filter_falsey(old_set)
        expect_set = set(["foo"])
        self.assertEqual(expect_set, new_set)
        self.assertIs(type(expect_set), type(new_set))

    def test_filter_ordereddict_no_recurse(self):
        """
        Test filtering an OrderedDict without recursing.
        """
        old_dict = OrderedDict(
            [
                ("foo", None),
                (
                    "bar",
                    OrderedDict(
                        [
                            (
                                "baz",
                                OrderedDict([("qux", None), ("quux", ""), ("foo", [])]),
                            )
                        ]
                    ),
                ),
                ("baz", ["qux"]),
                ("qux", {}),
                ("quux", []),
            ]
        )
        new_dict = salt.utils.data.filter_falsey(old_dict)
        expect_dict = OrderedDict(
            [
                (
                    "bar",
                    OrderedDict(
                        [
                            (
                                "baz",
                                OrderedDict([("qux", None), ("quux", ""), ("foo", [])]),
                            )
                        ]
                    ),
                ),
                ("baz", ["qux"]),
            ]
        )
        self.assertEqual(expect_dict, new_dict)
        self.assertIs(type(expect_dict), type(new_dict))

    def test_filter_ordereddict_recurse(self):
        """
        Test filtering an OrderedDict with recursing.
        """
        old_dict = OrderedDict(
            [
                ("foo", None),
                (
                    "bar",
                    OrderedDict(
                        [
                            (
                                "baz",
                                OrderedDict([("qux", None), ("quux", ""), ("foo", [])]),
                            )
                        ]
                    ),
                ),
                ("baz", ["qux"]),
                ("qux", {}),
                ("quux", []),
            ]
        )
        new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3)
        expect_dict = OrderedDict([("baz", ["qux"])])
        self.assertEqual(expect_dict, new_dict)
        self.assertIs(type(expect_dict), type(new_dict))

    def test_filter_list_recurse_limit(self):
        """
        Test filtering a list with recursing, but with a limited depth.
        Note that the top-level is always processed, so a recursion depth of 2
        means that two *additional* levels are processed.
        """
        old_list = [None, [None, [None, [None]]]]
        new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=2)
        self.assertEqual([[[[None]]]], new_list)

    def test_filter_dict_recurse_limit(self):
        """
        Test filtering a dict with recursing, but with a limited depth.
        Note that the top-level is always processed, so a recursion depth of 2
        means that two *additional* levels are processed.
        """
        old_dict = {
            "one": None,
            "foo": {"two": None, "bar": {"three": None, "baz": {"four": None}}},
        }
        new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=2)
        self.assertEqual({"foo": {"bar": {"baz": {"four": None}}}}, new_dict)

    def test_filter_exclude_types(self):
        """
        Test filtering a list recursively, but also ignoring (i.e. not filtering)
        out certain types that can be falsey.
        """
        # Ignore int, unicode
        old_list = [
            "foo",
            ["foo"],
            ["foo", None],
            {"foo": 0},
            {"foo": "bar", "baz": []},
            [{"foo": ""}],
        ]
        new_list = salt.utils.data.filter_falsey(
            old_list, recurse_depth=3, ignore_types=[type(0), type("")]
        )
        self.assertEqual(
            ["foo", ["foo"], ["foo"], {"foo": 0}, {"foo": "bar"}, [{"foo": ""}]],
            new_list,
        )
        # Ignore list
        old_list = [
            "foo",
            ["foo"],
            ["foo", None],
            {"foo": 0},
            {"foo": "bar", "baz": []},
            [{"foo": ""}],
        ]
        new_list = salt.utils.data.filter_falsey(
            old_list, recurse_depth=3, ignore_types=[type([])]
        )
        self.assertEqual(
            ["foo", ["foo"], ["foo"], {"foo": "bar", "baz": []}, []], new_list
        )
        # Ignore dict
        old_list = [
            "foo",
            ["foo"],
            ["foo", None],
            {"foo": 0},
            {"foo": "bar", "baz": []},
            [{"foo": ""}],
        ]
        new_list = salt.utils.data.filter_falsey(
            old_list, recurse_depth=3, ignore_types=[type({})]
        )
        self.assertEqual(["foo", ["foo"], ["foo"], {}, {"foo": "bar"}, [{}]], new_list)
        # Ignore NoneType
        old_list = [
            "foo",
            ["foo"],
            ["foo", None],
            {"foo": 0},
            {"foo": "bar", "baz": []},
            [{"foo": ""}],
        ]
        new_list = salt.utils.data.filter_falsey(
            old_list, recurse_depth=3, ignore_types=[type(None)]
        )
        self.assertEqual(["foo", ["foo"], ["foo", None], {"foo": "bar"}], new_list)


class FilterRecursiveDiff(TestCase):
    """
    Test suite for salt.utils.data.recursive_diff
    """

    def test_list_equality(self):
        """
        Test cases where equal lists are compared.
        """
        test_list = [0, 1, 2]
        self.assertEqual({}, salt.utils.data.recursive_diff(test_list, test_list))

        test_list = [[0], [1], [0, 1, 2]]
        self.assertEqual({}, salt.utils.data.recursive_diff(test_list, test_list))

    def test_dict_equality(self):
        """
        Test cases where equal dicts are compared.
        """
        test_dict = {"foo": "bar", "bar": {"baz": {"qux": "quux"}}, "frop": 0}
        self.assertEqual({}, salt.utils.data.recursive_diff(test_dict, test_dict))

    def test_ordereddict_equality(self):
        """
        Test cases where equal OrderedDicts are compared.
        """
        test_dict = OrderedDict(
            [
                ("foo", "bar"),
                ("bar", OrderedDict([("baz", OrderedDict([("qux", "quux")]))])),
                ("frop", 0),
            ]
        )
        self.assertEqual({}, salt.utils.data.recursive_diff(test_dict, test_dict))

    def test_mixed_equality(self):
        """
        Test cases where mixed nested lists and dicts are compared.
        """
        test_data = {
            "foo": "bar",
            "baz": [0, 1, 2],
            "bar": {"baz": [{"qux": "quux"}, {"froop", 0}]},
        }
        self.assertEqual({}, salt.utils.data.recursive_diff(test_data, test_data))

    def test_set_equality(self):
        """
        Test cases where equal sets are compared.
        """
        test_set = set([0, 1, 2, 3, "foo"])
        self.assertEqual({}, salt.utils.data.recursive_diff(test_set, test_set))

        # This is a bit of an oddity, as python seems to sort the sets in memory
        # so both sets end up with the same ordering (0..3).
        set_one = set([0, 1, 2, 3])
        set_two = set([3, 2, 1, 0])
        self.assertEqual({}, salt.utils.data.recursive_diff(set_one, set_two))

    def test_tuple_equality(self):
        """
        Test cases where equal tuples are compared.
        """
        test_tuple = (0, 1, 2, 3, "foo")
        self.assertEqual({}, salt.utils.data.recursive_diff(test_tuple, test_tuple))

    def test_list_inequality(self):
        """
        Test cases where two inequal lists are compared.
        """
        list_one = [0, 1, 2]
        list_two = ["foo", "bar", "baz"]
        expected_result = {"old": list_one, "new": list_two}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(list_one, list_two)
        )
        expected_result = {"new": list_one, "old": list_two}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(list_two, list_one)
        )

        list_one = [0, "foo", 1, "bar"]
        list_two = [1, "foo", 1, "qux"]
        expected_result = {"old": [0, "bar"], "new": [1, "qux"]}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(list_one, list_two)
        )
        expected_result = {"new": [0, "bar"], "old": [1, "qux"]}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(list_two, list_one)
        )

        list_one = [0, 1, [2, 3]]
        list_two = [0, 1, ["foo", "bar"]]
        expected_result = {"old": [[2, 3]], "new": [["foo", "bar"]]}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(list_one, list_two)
        )
        expected_result = {"new": [[2, 3]], "old": [["foo", "bar"]]}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(list_two, list_one)
        )

    def test_dict_inequality(self):
        """
        Test cases where two inequal dicts are compared.
        """
        dict_one = {"foo": 1, "bar": 2, "baz": 3}
        dict_two = {"foo": 2, 1: "bar", "baz": 3}
        expected_result = {"old": {"foo": 1, "bar": 2}, "new": {"foo": 2, 1: "bar"}}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(dict_one, dict_two)
        )
        expected_result = {"new": {"foo": 1, "bar": 2}, "old": {"foo": 2, 1: "bar"}}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(dict_two, dict_one)
        )

        dict_one = {"foo": {"bar": {"baz": 1}}}
        dict_two = {"foo": {"qux": {"baz": 1}}}
        expected_result = {"old": dict_one, "new": dict_two}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(dict_one, dict_two)
        )
        expected_result = {"new": dict_one, "old": dict_two}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(dict_two, dict_one)
        )

    def test_ordereddict_inequality(self):
        """
        Test cases where two inequal OrderedDicts are compared.
        """
        odict_one = OrderedDict([("foo", "bar"), ("bar", "baz")])
        odict_two = OrderedDict([("bar", "baz"), ("foo", "bar")])
        expected_result = {"old": odict_one, "new": odict_two}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(odict_one, odict_two)
        )

    def test_set_inequality(self):
        """
        Test cases where two inequal sets are compared.
        Tricky as the sets are compared zipped, so shuffled sets of equal values
        are considered different.
        """
        set_one = set([0, 1, 2, 4])
        set_two = set([0, 1, 3, 4])
        expected_result = {"old": set([2]), "new": set([3])}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(set_one, set_two)
        )
        expected_result = {"new": set([2]), "old": set([3])}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(set_two, set_one)
        )

        # It is unknown how different python versions will store sets in memory.
        # Python 2.7 seems to sort it (i.e. set_one below becomes {0, 1, 'foo', 'bar'}
        # However Python 3.6.8 stores it differently each run.
        # So just test for "not equal" here.
        set_one = set([0, "foo", 1, "bar"])
        set_two = set(["foo", 1, "bar", 2])
        expected_result = {}
        self.assertNotEqual(
            expected_result, salt.utils.data.recursive_diff(set_one, set_two)
        )

    def test_mixed_inequality(self):
        """
        Test cases where two mixed dicts/iterables that are different are compared.
        """
        dict_one = {"foo": [1, 2, 3]}
        dict_two = {"foo": [3, 2, 1]}
        expected_result = {"old": {"foo": [1, 3]}, "new": {"foo": [3, 1]}}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(dict_one, dict_two)
        )
        expected_result = {"new": {"foo": [1, 3]}, "old": {"foo": [3, 1]}}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(dict_two, dict_one)
        )

        list_one = [1, 2, {"foo": ["bar", {"foo": 1, "bar": 2}]}]
        list_two = [3, 4, {"foo": ["qux", {"foo": 1, "bar": 2}]}]
        expected_result = {
            "old": [1, 2, {"foo": ["bar"]}],
            "new": [3, 4, {"foo": ["qux"]}],
        }
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(list_one, list_two)
        )
        expected_result = {
            "new": [1, 2, {"foo": ["bar"]}],
            "old": [3, 4, {"foo": ["qux"]}],
        }
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(list_two, list_one)
        )

        mixed_one = {"foo": set([0, 1, 2]), "bar": [0, 1, 2]}
        mixed_two = {"foo": set([1, 2, 3]), "bar": [1, 2, 3]}
        expected_result = {
            "old": {"foo": set([0]), "bar": [0, 1, 2]},
            "new": {"foo": set([3]), "bar": [1, 2, 3]},
        }
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(mixed_one, mixed_two)
        )
        expected_result = {
            "new": {"foo": set([0]), "bar": [0, 1, 2]},
            "old": {"foo": set([3]), "bar": [1, 2, 3]},
        }
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(mixed_two, mixed_one)
        )

    def test_tuple_inequality(self):
        """
        Test cases where two tuples that are different are compared.
        """
        tuple_one = (1, 2, 3)
        tuple_two = (3, 2, 1)
        expected_result = {"old": (1, 3), "new": (3, 1)}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(tuple_one, tuple_two)
        )

    def test_list_vs_set(self):
        """
        Test case comparing a list with a set, will be compared unordered.
        """
        mixed_one = [1, 2, 3]
        mixed_two = set([3, 2, 1])
        expected_result = {}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(mixed_one, mixed_two)
        )
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(mixed_two, mixed_one)
        )

    def test_dict_vs_ordereddict(self):
        """
        Test case comparing a dict with an ordereddict, will be compared unordered.
        """
        test_dict = {"foo": "bar", "bar": "baz"}
        test_odict = OrderedDict([("foo", "bar"), ("bar", "baz")])
        self.assertEqual({}, salt.utils.data.recursive_diff(test_dict, test_odict))
        self.assertEqual({}, salt.utils.data.recursive_diff(test_odict, test_dict))

        test_odict2 = OrderedDict([("bar", "baz"), ("foo", "bar")])
        self.assertEqual({}, salt.utils.data.recursive_diff(test_dict, test_odict2))
        self.assertEqual({}, salt.utils.data.recursive_diff(test_odict2, test_dict))

    def test_list_ignore_ignored(self):
        """
        Test case comparing two lists with ignore-list supplied (which is not used
        when comparing lists).
        """
        list_one = [1, 2, 3]
        list_two = [3, 2, 1]
        expected_result = {"old": [1, 3], "new": [3, 1]}
        self.assertEqual(
            expected_result,
            salt.utils.data.recursive_diff(list_one, list_two, ignore_keys=[1, 3]),
        )

    def test_dict_ignore(self):
        """
        Test case comparing two dicts with ignore-list supplied.
        """
        dict_one = {"foo": 1, "bar": 2, "baz": 3}
        dict_two = {"foo": 3, "bar": 2, "baz": 1}
        expected_result = {"old": {"baz": 3}, "new": {"baz": 1}}
        self.assertEqual(
            expected_result,
            salt.utils.data.recursive_diff(dict_one, dict_two, ignore_keys=["foo"]),
        )

    def test_ordereddict_ignore(self):
        """
        Test case comparing two OrderedDicts with ignore-list supplied.
        """
        odict_one = OrderedDict([("foo", 1), ("bar", 2), ("baz", 3)])
        odict_two = OrderedDict([("baz", 1), ("bar", 2), ("foo", 3)])
        # The key 'foo' will be ignored, which means the key from the other OrderedDict
        # will always be considered "different" since OrderedDicts are compared ordered.
        expected_result = {
            "old": OrderedDict([("baz", 3)]),
            "new": OrderedDict([("baz", 1)]),
        }
        self.assertEqual(
            expected_result,
            salt.utils.data.recursive_diff(odict_one, odict_two, ignore_keys=["foo"]),
        )

    def test_dict_vs_ordereddict_ignore(self):
        """
        Test case comparing a dict with an OrderedDict with ignore-list supplied.
        """
        dict_one = {"foo": 1, "bar": 2, "baz": 3}
        odict_two = OrderedDict([("foo", 3), ("bar", 2), ("baz", 1)])
        expected_result = {"old": {"baz": 3}, "new": OrderedDict([("baz", 1)])}
        self.assertEqual(
            expected_result,
            salt.utils.data.recursive_diff(dict_one, odict_two, ignore_keys=["foo"]),
        )

    def test_mixed_nested_ignore(self):
        """
        Test case comparing mixed, nested items with ignore-list supplied.
        """
        dict_one = {"foo": [1], "bar": {"foo": 1, "bar": 2}, "baz": 3}
        dict_two = {"foo": [2], "bar": {"foo": 3, "bar": 2}, "baz": 1}
        expected_result = {"old": {"baz": 3}, "new": {"baz": 1}}
        self.assertEqual(
            expected_result,
            salt.utils.data.recursive_diff(dict_one, dict_two, ignore_keys=["foo"]),
        )

    def test_ordered_dict_unequal_length(self):
        """
        Test case comparing two OrderedDicts of unequal length.
        """
        odict_one = OrderedDict([("foo", 1), ("bar", 2), ("baz", 3)])
        odict_two = OrderedDict([("foo", 1), ("bar", 2)])
        expected_result = {"old": OrderedDict([("baz", 3)]), "new": {}}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(odict_one, odict_two)
        )

    def test_list_unequal_length(self):
        """
        Test case comparing two lists of unequal length.
        """
        list_one = [1, 2, 3]
        list_two = [1, 2, 3, 4]
        expected_result = {"old": [], "new": [4]}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(list_one, list_two)
        )

    def test_set_unequal_length(self):
        """
        Test case comparing two sets of unequal length.
        This does not do anything special, as it is unordered.
        """
        set_one = set([1, 2, 3])
        set_two = set([4, 3, 2, 1])
        expected_result = {"old": set([]), "new": set([4])}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(set_one, set_two)
        )

    def test_tuple_unequal_length(self):
        """
        Test case comparing two tuples of unequal length.
        This should be the same as comparing two ordered lists.
        """
        tuple_one = (1, 2, 3)
        tuple_two = (1, 2, 3, 4)
        expected_result = {"old": (), "new": (4,)}
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(tuple_one, tuple_two)
        )

    def test_list_unordered(self):
        """
        Test case comparing two lists unordered.
        """
        list_one = [1, 2, 3, 4]
        list_two = [4, 3, 2]
        expected_result = {"old": [1], "new": []}
        self.assertEqual(
            expected_result,
            salt.utils.data.recursive_diff(list_one, list_two, ignore_order=True),
        )

    def test_mixed_nested_unordered(self):
        """
        Test case comparing nested dicts/lists unordered.
        """
        dict_one = {"foo": {"bar": [1, 2, 3]}, "bar": [{"foo": 4}, 0]}
        dict_two = {"foo": {"bar": [3, 2, 1]}, "bar": [0, {"foo": 4}]}
        expected_result = {}
        self.assertEqual(
            expected_result,
            salt.utils.data.recursive_diff(dict_one, dict_two, ignore_order=True),
        )
        expected_result = {
            "old": {"foo": {"bar": [1, 3]}, "bar": [{"foo": 4}, 0]},
            "new": {"foo": {"bar": [3, 1]}, "bar": [0, {"foo": 4}]},
        }
        self.assertEqual(
            expected_result, salt.utils.data.recursive_diff(dict_one, dict_two)
        )

    def test_ordered_dict_unordered(self):
        """
        Test case comparing OrderedDicts unordered.
        """
        odict_one = OrderedDict([("foo", 1), ("bar", 2), ("baz", 3)])
        odict_two = OrderedDict([("baz", 3), ("bar", 2), ("foo", 1)])
        expected_result = {}
        self.assertEqual(
            expected_result,
            salt.utils.data.recursive_diff(odict_one, odict_two, ignore_order=True),
        )

    def test_ignore_missing_keys_dict(self):
        """
        Test case ignoring missing keys on a comparison of dicts.
        """
        dict_one = {"foo": 1, "bar": 2, "baz": 3}
        dict_two = {"bar": 3}
        expected_result = {"old": {"bar": 2}, "new": {"bar": 3}}
        self.assertEqual(
            expected_result,
            salt.utils.data.recursive_diff(
                dict_one, dict_two, ignore_missing_keys=True
            ),
        )

    def test_ignore_missing_keys_ordered_dict(self):
        """
        Test case not ignoring missing keys on a comparison of OrderedDicts.
        """
        odict_one = OrderedDict([("foo", 1), ("bar", 2), ("baz", 3)])
        odict_two = OrderedDict([("bar", 3)])
        expected_result = {"old": odict_one, "new": odict_two}
        self.assertEqual(
            expected_result,
            salt.utils.data.recursive_diff(
                odict_one, odict_two, ignore_missing_keys=True
            ),
        )

    def test_ignore_missing_keys_recursive(self):
        """
        Test case ignoring missing keys on a comparison of nested dicts.
        """
        dict_one = {"foo": {"bar": 2, "baz": 3}}
        dict_two = {"foo": {"baz": 3}}
        expected_result = {}
        self.assertEqual(
            expected_result,
            salt.utils.data.recursive_diff(
                dict_one, dict_two, ignore_missing_keys=True
            ),
        )
        # Compare from dict-in-dict
        dict_two = {}
        self.assertEqual(
            expected_result,
            salt.utils.data.recursive_diff(
                dict_one, dict_two, ignore_missing_keys=True
            ),
        )
        # Compare from dict-in-list
        dict_one = {"foo": ["bar", {"baz": 3}]}
        dict_two = {"foo": ["bar", {}]}
        self.assertEqual(
            expected_result,
            salt.utils.data.recursive_diff(
                dict_one, dict_two, ignore_missing_keys=True
            ),
        )