gitfs.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801
  1. """
  2. Base classes for gitfs/git_pillar integration tests
  3. """
  4. import copy
  5. import errno
  6. import logging
  7. import os
  8. import pprint
  9. import shutil
  10. import tempfile
  11. import textwrap
  12. import salt.utils.files
  13. import salt.utils.path
  14. import salt.utils.platform
  15. import salt.utils.yaml
  16. from salt.fileserver import gitfs
  17. from salt.pillar import git_pillar
  18. from saltfactories.utils.ports import get_unused_localhost_port
  19. from saltfactories.utils.processes.bases import FactoryDaemonScriptBase
  20. from saltfactories.utils.processes.helpers import start_daemon, terminate_process
  21. from saltfactories.utils.processes.sshd import SshdDaemon
  22. from tests.support.case import ModuleCase
  23. from tests.support.helpers import patched_environ, requires_system_grains
  24. from tests.support.mixins import (
  25. AdaptedConfigurationTestCaseMixin,
  26. LoaderModuleMockMixin,
  27. SaltReturnAssertsMixin,
  28. )
  29. from tests.support.mock import patch
  30. from tests.support.runtests import RUNTIME_VARS
  31. from tests.support.unit import SkipTest
  32. try:
  33. import psutil
  34. except ImportError:
  35. pass
  36. log = logging.getLogger(__name__)
  37. USERNAME = "gitpillaruser"
  38. PASSWORD = "saltrules"
  39. _OPTS = {
  40. "__role": "minion",
  41. "environment": None,
  42. "pillarenv": None,
  43. "hash_type": "sha256",
  44. "file_roots": {},
  45. "state_top": "top.sls",
  46. "state_top_saltenv": None,
  47. "renderer": "yaml_jinja",
  48. "renderer_whitelist": [],
  49. "renderer_blacklist": [],
  50. "pillar_merge_lists": False,
  51. "git_pillar_base": "master",
  52. "git_pillar_branch": "master",
  53. "git_pillar_env": "",
  54. "git_pillar_fallback": "",
  55. "git_pillar_root": "",
  56. "git_pillar_ssl_verify": True,
  57. "git_pillar_global_lock": True,
  58. "git_pillar_user": "",
  59. "git_pillar_password": "",
  60. "git_pillar_insecure_auth": False,
  61. "git_pillar_privkey": "",
  62. "git_pillar_pubkey": "",
  63. "git_pillar_passphrase": "",
  64. "git_pillar_refspecs": [
  65. "+refs/heads/*:refs/remotes/origin/*",
  66. "+refs/tags/*:refs/tags/*",
  67. ],
  68. "git_pillar_includes": True,
  69. }
  70. PROC_TIMEOUT = 10
  71. class UwsgiDaemon(FactoryDaemonScriptBase):
  72. def __init__(self, *args, **kwargs):
  73. config_dir = kwargs.pop("config_dir")
  74. check_port = kwargs.pop("check_port")
  75. super().__init__(*args, **kwargs)
  76. self.config_dir = config_dir
  77. self.check_port = check_port
  78. def get_log_prefix(self):
  79. return "[uWSGI] "
  80. def get_base_script_args(self):
  81. """
  82. Returns any additional arguments to pass to the CLI script
  83. """
  84. return ["--yaml", os.path.join(self.config_dir, "uwsgi.yml")]
  85. def get_check_ports(self):
  86. """
  87. Return a list of ports to check against to ensure the daemon is running
  88. """
  89. return [self.check_port]
  90. class NginxDaemon(FactoryDaemonScriptBase):
  91. def __init__(self, *args, **kwargs):
  92. config_dir = kwargs.pop("config_dir")
  93. check_port = kwargs.pop("check_port")
  94. super().__init__(*args, **kwargs)
  95. self.config_dir = config_dir
  96. self.check_port = check_port
  97. def get_log_prefix(self):
  98. return "[Nginx] "
  99. def get_base_script_args(self):
  100. """
  101. Returns any additional arguments to pass to the CLI script
  102. """
  103. return ["-c", os.path.join(self.config_dir, "nginx.conf")]
  104. def get_check_ports(self):
  105. """
  106. Return a list of ports to check against to ensure the daemon is running
  107. """
  108. return [self.check_port]
  109. class SaltClientMixin(ModuleCase):
  110. client = None
  111. @classmethod
  112. @requires_system_grains
  113. def setUpClass(cls, grains=None): # pylint: disable=arguments-differ
  114. # Cent OS 6 has too old a version of git to handle the make_repo code, as
  115. # it lacks the -c option for git itself.
  116. make_repo = getattr(cls, "make_repo", None)
  117. if (
  118. callable(make_repo)
  119. and grains["os_family"] == "RedHat"
  120. and grains["osmajorrelease"] < 7
  121. ):
  122. raise SkipTest("RHEL < 7 has too old a version of git to run these tests")
  123. # Late import
  124. import salt.client
  125. mopts = AdaptedConfigurationTestCaseMixin.get_config(
  126. "master", from_scratch=True
  127. )
  128. cls.user = mopts["user"]
  129. cls.client = salt.client.get_local_client(mopts=mopts)
  130. @classmethod
  131. def tearDownClass(cls):
  132. cls.client = None
  133. @classmethod
  134. def cls_run_function(cls, function, *args, **kwargs):
  135. orig = cls.client.cmd("minion", function, arg=args, timeout=300, kwarg=kwargs)
  136. return orig["minion"]
  137. class SSHDMixin(SaltClientMixin, SaltReturnAssertsMixin):
  138. """
  139. Functions to stand up an SSHD server to serve up git repos for tests.
  140. """
  141. sshd_proc = None
  142. prep_states_ran = False
  143. known_hosts_setup = False
  144. @classmethod
  145. def setUpClass(cls): # pylint: disable=arguments-differ
  146. super().setUpClass()
  147. try:
  148. log.info("%s: prep_server()", cls.__name__)
  149. cls.sshd_bin = salt.utils.path.which("sshd")
  150. cls.sshd_config_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
  151. cls.sshd_config = os.path.join(cls.sshd_config_dir, "sshd_config")
  152. cls.sshd_port = get_unused_localhost_port(cached_seconds=120)
  153. cls.url = "ssh://{username}@127.0.0.1:{port}/~/repo.git".format(
  154. username=cls.username, port=cls.sshd_port
  155. )
  156. cls.url_extra_repo = "ssh://{username}@127.0.0.1:{port}/~/extra_repo.git".format(
  157. username=cls.username, port=cls.sshd_port
  158. )
  159. home = "/root/.ssh"
  160. cls.ext_opts = {
  161. "url": cls.url,
  162. "url_extra_repo": cls.url_extra_repo,
  163. "privkey_nopass": os.path.join(home, cls.id_rsa_nopass),
  164. "pubkey_nopass": os.path.join(home, cls.id_rsa_nopass + ".pub"),
  165. "privkey_withpass": os.path.join(home, cls.id_rsa_withpass),
  166. "pubkey_withpass": os.path.join(home, cls.id_rsa_withpass + ".pub"),
  167. "passphrase": cls.passphrase,
  168. }
  169. if cls.prep_states_ran is False:
  170. ret = cls.cls_run_function(
  171. "state.apply",
  172. mods="git_pillar.ssh",
  173. pillar={
  174. "git_pillar": {
  175. "git_ssh": cls.git_ssh,
  176. "id_rsa_nopass": cls.id_rsa_nopass,
  177. "id_rsa_withpass": cls.id_rsa_withpass,
  178. "sshd_bin": cls.sshd_bin,
  179. "sshd_port": cls.sshd_port,
  180. "sshd_config_dir": cls.sshd_config_dir,
  181. "master_user": cls.user,
  182. "user": cls.username,
  183. }
  184. },
  185. )
  186. assert next(iter(ret.values()))["result"] is True
  187. cls.prep_states_ran = True
  188. log.info("%s: States applied", cls.__name__)
  189. if cls.sshd_proc is not None:
  190. if not psutil.pid_exists(cls.sshd_proc.pid):
  191. log.info(
  192. "%s: sshd started but appears to be dead now. Will try to restart it.",
  193. cls.__name__,
  194. )
  195. cls.sshd_proc = None
  196. if cls.sshd_proc is None:
  197. cls.sshd_proc = start_daemon(
  198. cls.sshd_bin,
  199. SshdDaemon,
  200. config_dir=cls.sshd_config_dir,
  201. serve_port=cls.sshd_port,
  202. )
  203. log.info("%s: sshd started", cls.__name__)
  204. except AssertionError:
  205. cls.tearDownClass()
  206. raise
  207. if cls.known_hosts_setup is False:
  208. known_hosts_ret = cls.cls_run_function(
  209. "ssh.set_known_host",
  210. user=cls.user,
  211. hostname="127.0.0.1",
  212. port=cls.sshd_port,
  213. enc="ssh-rsa",
  214. fingerprint="fd:6f:7f:5d:06:6b:f2:06:0d:26:93:9e:5a:b5:19:46",
  215. hash_known_hosts=False,
  216. fingerprint_hash_type="md5",
  217. )
  218. if "error" in known_hosts_ret:
  219. cls.tearDownClass()
  220. raise AssertionError(
  221. "Failed to add key to {} user's known_hosts "
  222. "file: {}".format(cls.master_opts["user"], known_hosts_ret["error"])
  223. )
  224. cls.known_hosts_setup = True
  225. @classmethod
  226. def tearDownClass(cls):
  227. if cls.sshd_proc is not None:
  228. log.info(
  229. "[%s] Stopping %s",
  230. cls.sshd_proc.get_log_prefix(),
  231. cls.sshd_proc.__class__.__name__,
  232. )
  233. terminate_process(cls.sshd_proc.pid, kill_children=True, slow_stop=True)
  234. log.info(
  235. "[%s] %s stopped",
  236. cls.sshd_proc.get_log_prefix(),
  237. cls.sshd_proc.__class__.__name__,
  238. )
  239. cls.sshd_proc = None
  240. if cls.prep_states_ran:
  241. ret = cls.cls_run_function(
  242. "state.single", "user.absent", name=cls.username, purge=True
  243. )
  244. try:
  245. if ret and "minion" in ret:
  246. ret_data = next(iter(ret["minion"].values()))
  247. if not ret_data["result"]:
  248. log.warning("Failed to delete test account %s", cls.username)
  249. except KeyError:
  250. log.warning(
  251. "Failed to delete test account. Salt return:\n%s",
  252. pprint.pformat(ret),
  253. )
  254. cls.prep_states_ran = False
  255. cls.known_hosts_setup = False
  256. shutil.rmtree(cls.sshd_config_dir, ignore_errors=True)
  257. ssh_dir = os.path.expanduser("~/.ssh")
  258. for filename in (
  259. cls.id_rsa_nopass,
  260. cls.id_rsa_nopass + ".pub",
  261. cls.id_rsa_withpass,
  262. cls.id_rsa_withpass + ".pub",
  263. cls.git_ssh,
  264. ):
  265. try:
  266. os.remove(os.path.join(ssh_dir, filename))
  267. except OSError as exc:
  268. if exc.errno != errno.ENOENT:
  269. raise
  270. super().tearDownClass()
  271. class WebserverMixin(SaltClientMixin, SaltReturnAssertsMixin):
  272. """
  273. Functions to stand up an nginx + uWSGI + git-http-backend webserver to
  274. serve up git repos for tests.
  275. """
  276. nginx_proc = uwsgi_proc = None
  277. prep_states_ran = False
  278. @classmethod
  279. def setUpClass(cls): # pylint: disable=arguments-differ
  280. """
  281. Set up all the webserver paths. Designed to be run once in a
  282. setUpClass function.
  283. """
  284. super().setUpClass()
  285. cls.root_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
  286. cls.config_dir = os.path.join(cls.root_dir, "config")
  287. cls.nginx_conf = os.path.join(cls.config_dir, "nginx.conf")
  288. cls.uwsgi_conf = os.path.join(cls.config_dir, "uwsgi.yml")
  289. cls.git_dir = os.path.join(cls.root_dir, "git")
  290. cls.repo_dir = os.path.join(cls.git_dir, "repos")
  291. cls.venv_dir = os.path.join(cls.root_dir, "venv")
  292. cls.uwsgi_bin = os.path.join(cls.venv_dir, "bin", "uwsgi")
  293. cls.nginx_port = cls.uwsgi_port = get_unused_localhost_port(cached_seconds=120)
  294. cls.uwsgi_port = get_unused_localhost_port(cached_seconds=120)
  295. cls.url = "http://127.0.0.1:{port}/repo.git".format(port=cls.nginx_port)
  296. cls.url_extra_repo = "http://127.0.0.1:{port}/extra_repo.git".format(
  297. port=cls.nginx_port
  298. )
  299. cls.ext_opts = {"url": cls.url, "url_extra_repo": cls.url_extra_repo}
  300. # Add auth params if present (if so this will trigger the spawned
  301. # server to turn on HTTP basic auth).
  302. for credential_param in ("user", "password"):
  303. if hasattr(cls, credential_param):
  304. cls.ext_opts[credential_param] = getattr(cls, credential_param)
  305. auth_enabled = hasattr(cls, "username") and hasattr(cls, "password")
  306. pillar = {
  307. "git_pillar": {
  308. "config_dir": cls.config_dir,
  309. "git_dir": cls.git_dir,
  310. "venv_dir": cls.venv_dir,
  311. "root_dir": cls.root_dir,
  312. "nginx_port": cls.nginx_port,
  313. "uwsgi_port": cls.uwsgi_port,
  314. "auth_enabled": auth_enabled,
  315. }
  316. }
  317. # Different libexec dir for git backend on Debian and FreeBSD-based systems
  318. if salt.utils.platform.is_freebsd():
  319. git_core = "/usr/local/libexec/git-core"
  320. else:
  321. git_core = "/usr/libexec/git-core"
  322. if not os.path.exists(git_core):
  323. git_core = "/usr/lib/git-core"
  324. if not os.path.exists(git_core):
  325. cls.tearDownClass()
  326. raise AssertionError(
  327. "{} not found. Either git is not installed, or the test "
  328. "class needs to be updated.".format(git_core)
  329. )
  330. pillar["git_pillar"]["git-http-backend"] = os.path.join(
  331. git_core, "git-http-backend"
  332. )
  333. try:
  334. if cls.prep_states_ran is False:
  335. ret = cls.cls_run_function(
  336. "state.apply", mods="git_pillar.http", pillar=pillar
  337. )
  338. assert next(iter(ret.values()))["result"] is True
  339. cls.prep_states_ran = True
  340. log.info("%s: States applied", cls.__name__)
  341. if cls.uwsgi_proc is not None:
  342. if not psutil.pid_exists(cls.uwsgi_proc.pid):
  343. log.warning(
  344. "%s: uWsgi started but appears to be dead now. Will try to restart it.",
  345. cls.__name__,
  346. )
  347. cls.uwsgi_proc = None
  348. if cls.uwsgi_proc is None:
  349. cls.uwsgi_proc = start_daemon(
  350. cls.uwsgi_bin,
  351. UwsgiDaemon,
  352. config_dir=cls.config_dir,
  353. check_port=cls.uwsgi_port,
  354. )
  355. log.info("%s: %s started", cls.__name__, cls.uwsgi_bin)
  356. if cls.nginx_proc is not None:
  357. if not psutil.pid_exists(cls.nginx_proc.pid):
  358. log.warning(
  359. "%s: nginx started but appears to be dead now. Will try to restart it.",
  360. cls.__name__,
  361. )
  362. cls.nginx_proc = None
  363. if cls.nginx_proc is None:
  364. cls.nginx_proc = start_daemon(
  365. "nginx",
  366. NginxDaemon,
  367. config_dir=cls.config_dir,
  368. check_port=cls.nginx_port,
  369. )
  370. log.info("%s: nginx started", cls.__name__)
  371. except AssertionError:
  372. cls.tearDownClass()
  373. raise
  374. @classmethod
  375. def tearDownClass(cls):
  376. if cls.nginx_proc is not None:
  377. log.info(
  378. "[%s] Stopping %s",
  379. cls.nginx_proc.get_log_prefix(),
  380. cls.nginx_proc.__class__.__name__,
  381. )
  382. terminate_process(cls.nginx_proc.pid, kill_children=True, slow_stop=True)
  383. log.info(
  384. "[%s] %s stopped",
  385. cls.nginx_proc.get_log_prefix(),
  386. cls.nginx_proc.__class__.__name__,
  387. )
  388. cls.nginx_proc = None
  389. if cls.uwsgi_proc is not None:
  390. log.info(
  391. "[%s] Stopping %s",
  392. cls.uwsgi_proc.get_log_prefix(),
  393. cls.uwsgi_proc.__class__.__name__,
  394. )
  395. terminate_process(cls.uwsgi_proc.pid, kill_children=True, slow_stop=True)
  396. log.info(
  397. "[%s] %s stopped",
  398. cls.uwsgi_proc.get_log_prefix(),
  399. cls.uwsgi_proc.__class__.__name__,
  400. )
  401. cls.uwsgi_proc = None
  402. shutil.rmtree(cls.root_dir, ignore_errors=True)
  403. cls.prep_states_ran = False
  404. super().tearDownClass()
  405. class GitTestBase(ModuleCase):
  406. """
  407. Base class for all gitfs/git_pillar tests. Must be subclassed and paired
  408. with either SSHDMixin or WebserverMixin to provide the server.
  409. """
  410. maxDiff = None
  411. git_opts = '-c user.name="Foo Bar" -c user.email=foo@bar.com'
  412. ext_opts = {}
  413. def make_repo(self, root_dir, user="root"):
  414. raise NotImplementedError()
  415. class GitFSTestBase(GitTestBase, LoaderModuleMockMixin):
  416. """
  417. Base class for all gitfs tests
  418. """
  419. @requires_system_grains
  420. def setup_loader_modules(self, grains): # pylint: disable=W0221
  421. return {gitfs: {"__opts__": copy.copy(_OPTS), "__grains__": grains}}
  422. def make_repo(self, root_dir, user="root"):
  423. raise NotImplementedError()
  424. class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin):
  425. """
  426. Base class for all git_pillar tests
  427. """
  428. bare_repo = bare_repo_backup = bare_extra_repo = bare_extra_repo_backup = None
  429. admin_repo = admin_repo_backup = admin_extra_repo = admin_extra_repo_backup = None
  430. @requires_system_grains
  431. def setup_loader_modules(self, grains): # pylint: disable=W0221
  432. return {git_pillar: {"__opts__": copy.copy(_OPTS), "__grains__": grains}}
  433. def get_pillar(self, ext_pillar_conf):
  434. """
  435. Run git_pillar with the specified configuration
  436. """
  437. cachedir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
  438. self.addCleanup(shutil.rmtree, cachedir, ignore_errors=True)
  439. ext_pillar_opts = {"optimization_order": [0, 1, 2]}
  440. ext_pillar_opts.update(
  441. salt.utils.yaml.safe_load(
  442. ext_pillar_conf.format(
  443. cachedir=cachedir,
  444. extmods=os.path.join(cachedir, "extmods"),
  445. **self.ext_opts
  446. )
  447. )
  448. )
  449. with patch.dict(git_pillar.__opts__, ext_pillar_opts):
  450. return git_pillar.ext_pillar(
  451. "minion", {}, *ext_pillar_opts["ext_pillar"][0]["git"]
  452. )
  453. def make_repo(self, root_dir, user="root"):
  454. log.info("Creating test Git repo....")
  455. self.bare_repo = os.path.join(root_dir, "repo.git")
  456. self.bare_repo_backup = "{}.backup".format(self.bare_repo)
  457. self.admin_repo = os.path.join(root_dir, "admin")
  458. self.admin_repo_backup = "{}.backup".format(self.admin_repo)
  459. for dirname in (self.bare_repo, self.admin_repo):
  460. shutil.rmtree(dirname, ignore_errors=True)
  461. if os.path.exists(self.bare_repo_backup) and os.path.exists(
  462. self.admin_repo_backup
  463. ):
  464. shutil.copytree(self.bare_repo_backup, self.bare_repo)
  465. shutil.copytree(self.admin_repo_backup, self.admin_repo)
  466. return
  467. # Create bare repo
  468. self.run_function("git.init", [self.bare_repo], user=user, bare=True)
  469. # Clone bare repo
  470. self.run_function("git.clone", [self.admin_repo], url=self.bare_repo, user=user)
  471. def _push(branch, message):
  472. self.run_function("git.add", [self.admin_repo, "."], user=user)
  473. self.run_function(
  474. "git.commit",
  475. [self.admin_repo, message],
  476. user=user,
  477. git_opts=self.git_opts,
  478. )
  479. self.run_function(
  480. "git.push", [self.admin_repo], remote="origin", ref=branch, user=user,
  481. )
  482. with salt.utils.files.fopen(
  483. os.path.join(self.admin_repo, "top.sls"), "w"
  484. ) as fp_:
  485. fp_.write(
  486. textwrap.dedent(
  487. """\
  488. base:
  489. '*':
  490. - foo
  491. """
  492. )
  493. )
  494. with salt.utils.files.fopen(
  495. os.path.join(self.admin_repo, "foo.sls"), "w"
  496. ) as fp_:
  497. fp_.write(
  498. textwrap.dedent(
  499. """\
  500. branch: master
  501. mylist:
  502. - master
  503. mydict:
  504. master: True
  505. nested_list:
  506. - master
  507. nested_dict:
  508. master: True
  509. """
  510. )
  511. )
  512. # Add another file to be referenced using git_pillar_includes
  513. with salt.utils.files.fopen(
  514. os.path.join(self.admin_repo, "bar.sls"), "w"
  515. ) as fp_:
  516. fp_.write("included_pillar: True\n")
  517. # Add another file in subdir
  518. os.mkdir(os.path.join(self.admin_repo, "subdir"))
  519. with salt.utils.files.fopen(
  520. os.path.join(self.admin_repo, "subdir", "bar.sls"), "w"
  521. ) as fp_:
  522. fp_.write("from_subdir: True\n")
  523. _push("master", "initial commit")
  524. # Do the same with different values for "dev" branch
  525. self.run_function("git.checkout", [self.admin_repo], user=user, opts="-b dev")
  526. # The bar.sls shouldn't be in any branch but master
  527. self.run_function("git.rm", [self.admin_repo, "bar.sls"], user=user)
  528. with salt.utils.files.fopen(
  529. os.path.join(self.admin_repo, "top.sls"), "w"
  530. ) as fp_:
  531. fp_.write(
  532. textwrap.dedent(
  533. """\
  534. dev:
  535. '*':
  536. - foo
  537. """
  538. )
  539. )
  540. with salt.utils.files.fopen(
  541. os.path.join(self.admin_repo, "foo.sls"), "w"
  542. ) as fp_:
  543. fp_.write(
  544. textwrap.dedent(
  545. """\
  546. branch: dev
  547. mylist:
  548. - dev
  549. mydict:
  550. dev: True
  551. nested_list:
  552. - dev
  553. nested_dict:
  554. dev: True
  555. """
  556. )
  557. )
  558. _push("dev", "add dev branch")
  559. # Create just a top file in a separate repo, to be mapped to the base
  560. # env and referenced using git_pillar_includes
  561. self.run_function(
  562. "git.checkout", [self.admin_repo], user=user, opts="-b top_only"
  563. )
  564. # The top.sls should be the only file in this branch
  565. self.run_function(
  566. "git.rm",
  567. [self.admin_repo, "foo.sls", os.path.join("subdir", "bar.sls")],
  568. user=user,
  569. )
  570. with salt.utils.files.fopen(
  571. os.path.join(self.admin_repo, "top.sls"), "w"
  572. ) as fp_:
  573. fp_.write(
  574. textwrap.dedent(
  575. """\
  576. base:
  577. '*':
  578. - bar
  579. """
  580. )
  581. )
  582. _push("top_only", "add top_only branch")
  583. # Create just another top file in a separate repo, to be mapped to the base
  584. # env and including mounted.bar
  585. self.run_function(
  586. "git.checkout", [self.admin_repo], user=user, opts="-b top_mounted"
  587. )
  588. # The top.sls should be the only file in this branch
  589. with salt.utils.files.fopen(
  590. os.path.join(self.admin_repo, "top.sls"), "w"
  591. ) as fp_:
  592. fp_.write(
  593. textwrap.dedent(
  594. """\
  595. base:
  596. '*':
  597. - mounted.bar
  598. """
  599. )
  600. )
  601. _push("top_mounted", "add top_mounted branch")
  602. shutil.copytree(self.bare_repo, self.bare_repo_backup)
  603. shutil.copytree(self.admin_repo, self.admin_repo_backup)
  604. log.info("Test Git repo created.")
  605. def make_extra_repo(self, root_dir, user="root"):
  606. log.info("Creating extra test Git repo....")
  607. self.bare_extra_repo = os.path.join(root_dir, "extra_repo.git")
  608. self.bare_extra_repo_backup = "{}.backup".format(self.bare_extra_repo)
  609. self.admin_extra_repo = os.path.join(root_dir, "admin_extra")
  610. self.admin_extra_repo_backup = "{}.backup".format(self.admin_extra_repo)
  611. for dirname in (self.bare_extra_repo, self.admin_extra_repo):
  612. shutil.rmtree(dirname, ignore_errors=True)
  613. if os.path.exists(self.bare_extra_repo_backup) and os.path.exists(
  614. self.admin_extra_repo_backup
  615. ):
  616. shutil.copytree(self.bare_extra_repo_backup, self.bare_extra_repo)
  617. shutil.copytree(self.admin_extra_repo_backup, self.admin_extra_repo)
  618. return
  619. # Create bare extra repo
  620. self.run_function("git.init", [self.bare_extra_repo], user=user, bare=True)
  621. # Clone bare repo
  622. self.run_function(
  623. "git.clone", [self.admin_extra_repo], url=self.bare_extra_repo, user=user
  624. )
  625. def _push(branch, message):
  626. self.run_function("git.add", [self.admin_extra_repo, "."], user=user)
  627. self.run_function(
  628. "git.commit",
  629. [self.admin_extra_repo, message],
  630. user=user,
  631. git_opts=self.git_opts,
  632. )
  633. self.run_function(
  634. "git.push",
  635. [self.admin_extra_repo],
  636. remote="origin",
  637. ref=branch,
  638. user=user,
  639. )
  640. with salt.utils.files.fopen(
  641. os.path.join(self.admin_extra_repo, "top.sls"), "w"
  642. ) as fp_:
  643. fp_.write(
  644. textwrap.dedent(
  645. """\
  646. "{{saltenv}}":
  647. '*':
  648. - motd
  649. - nowhere.foo
  650. """
  651. )
  652. )
  653. with salt.utils.files.fopen(
  654. os.path.join(self.admin_extra_repo, "motd.sls"), "w"
  655. ) as fp_:
  656. fp_.write(
  657. textwrap.dedent(
  658. """\
  659. motd: The force will be with you. Always.
  660. """
  661. )
  662. )
  663. _push("master", "initial commit")
  664. shutil.copytree(self.bare_extra_repo, self.bare_extra_repo_backup)
  665. shutil.copytree(self.admin_extra_repo, self.admin_extra_repo_backup)
  666. log.info("Extra test Git repo created.")
  667. @classmethod
  668. def tearDownClass(cls):
  669. super().tearDownClass()
  670. for dirname in (
  671. cls.admin_repo,
  672. cls.admin_repo_backup,
  673. cls.admin_extra_repo,
  674. cls.admin_extra_repo_backup,
  675. cls.bare_repo,
  676. cls.bare_repo_backup,
  677. cls.bare_extra_repo,
  678. cls.bare_extra_repo_backup,
  679. ):
  680. if dirname is not None:
  681. shutil.rmtree(dirname, ignore_errors=True)
  682. class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin):
  683. """
  684. Base class for GitPython and Pygit2 SSH tests
  685. """
  686. id_rsa_nopass = id_rsa_withpass = None
  687. git_ssh = "/tmp/git_ssh"
  688. def setUp(self):
  689. """
  690. Create the SSH server and user, and create the git repo
  691. """
  692. log.info("%s.setUp() started...", self.__class__.__name__)
  693. super().setUp()
  694. root_dir = os.path.expanduser("~{}".format(self.username))
  695. if root_dir.startswith("~"):
  696. raise AssertionError(
  697. "Unable to resolve homedir for user '{}'".format(self.username)
  698. )
  699. self.make_repo(root_dir, user=self.username)
  700. self.make_extra_repo(root_dir, user=self.username)
  701. log.info("%s.setUp() complete.", self.__class__.__name__)
  702. def get_pillar(self, ext_pillar_conf):
  703. """
  704. Wrap the parent class' get_pillar() func in logic that temporarily
  705. changes the GIT_SSH to use our custom script, ensuring that the
  706. passphraselsess key is used to auth without needing to modify the root
  707. user's ssh config file.
  708. """
  709. with patched_environ(GIT_SSH=self.git_ssh):
  710. return super().get_pillar(ext_pillar_conf)
  711. class GitPillarHTTPTestBase(GitPillarTestBase, WebserverMixin):
  712. """
  713. Base class for GitPython and Pygit2 HTTP tests
  714. """
  715. def setUp(self):
  716. """
  717. Create and start the webserver, and create the git repo
  718. """
  719. log.info("%s.setUp() started...", self.__class__.__name__)
  720. super().setUp()
  721. self.make_repo(self.repo_dir)
  722. self.make_extra_repo(self.repo_dir)
  723. log.info("%s.setUp() complete", self.__class__.__name__)