testprogram.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958
  1. # -*- coding: utf-8 -*-
  2. '''
  3. Classes for starting/stopping/status salt daemons, auxiliary
  4. scripts, generic commands.
  5. '''
  6. from __future__ import absolute_import
  7. import atexit
  8. import copy
  9. from datetime import datetime, timedelta
  10. import errno
  11. import getpass
  12. import logging
  13. import os
  14. import shutil
  15. import signal
  16. import socket
  17. import subprocess
  18. import sys
  19. import tempfile
  20. import time
  21. import salt.utils.files
  22. import salt.utils.platform
  23. import salt.utils.process
  24. import salt.utils.psutil_compat as psutils
  25. import salt.utils.yaml
  26. import salt.defaults.exitcodes as exitcodes
  27. from salt.ext import six
  28. from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin
  29. from tests.support.unit import TestCase
  30. from tests.support.runtests import RUNTIME_VARS
  31. from tests.support.processes import terminate_process, terminate_process_list
  32. from tests.support.cli_scripts import ScriptPathMixin
  33. log = logging.getLogger(__name__)
  34. if 'TimeoutError' not in __builtins__:
  35. class TimeoutError(OSError):
  36. '''Compatibility exception with python3'''
  37. __builtins__['TimeoutError'] = TimeoutError
  38. class TestProgramMeta(type):
  39. '''
  40. Stack all inherited config_attrs and dirtree dirs from the base classes.
  41. '''
  42. def __new__(mcs, name, bases, attrs):
  43. config_vals = {}
  44. config_attrs = set()
  45. dirtree = set()
  46. for base in bases:
  47. config_vals.update(getattr(base, 'config_vals', {}))
  48. config_attrs.update(getattr(base, 'config_attrs', {}))
  49. dirtree.update(getattr(base, 'dirtree', []))
  50. config_vals.update(attrs.get('config_vals', {}))
  51. attrs['config_vals'] = config_vals
  52. config_attrs.update(attrs.get('config_attrs', {}))
  53. attrs['config_attrs'] = config_attrs
  54. dirtree.update(attrs.get('dirtree', []))
  55. attrs['dirtree'] = dirtree
  56. return super(TestProgramMeta, mcs).__new__(mcs, name, bases, attrs)
  57. # pylint: disable=too-many-instance-attributes
  58. class TestProgram(six.with_metaclass(TestProgramMeta, object)):
  59. '''
  60. Set up an arbitrary executable to run.
  61. :attribute dirtree: An iterable of directories to be created
  62. '''
  63. empty_config = ''
  64. config_file = ''
  65. config_attrs = set([
  66. 'name',
  67. 'test_dir',
  68. 'config_dirs',
  69. ])
  70. config_vals = {
  71. }
  72. config_base = ''
  73. config_dir = os.path.join('etc')
  74. configs = {}
  75. config_types = (str, six.string_types,)
  76. dirtree = [
  77. '&config_dirs',
  78. ]
  79. @staticmethod
  80. def config_caster(cfg):
  81. return str(cfg)
  82. def __init__(self, program=None, name=None, env=None, shell=False, parent_dir=None, clean_on_exit=True, **kwargs):
  83. self.program = program or getattr(self, 'program', None)
  84. self.name = name or getattr(self, 'name', '')
  85. self.env = env or {}
  86. self.shell = shell
  87. self._parent_dir = parent_dir or None
  88. self.clean_on_exit = clean_on_exit
  89. self._root_dir = kwargs.pop('root_dir', self.name)
  90. self.config_dir = kwargs.pop('config_dir', copy.copy(self.config_dir))
  91. config_attrs = copy.copy(self.config_attrs)
  92. config_attrs.update(kwargs.pop('config_attrs', set()))
  93. self.config_attrs = config_attrs
  94. config_vals = copy.copy(self.config_vals)
  95. config_vals.update(kwargs.pop('config_vals', {}))
  96. self.config_vals = config_vals
  97. config_base = copy.deepcopy(self.config_base)
  98. config_base = self.config_merge(config_base, kwargs.pop('config_base', self.config_types[0]()))
  99. self.config_base = config_base
  100. configs = copy.deepcopy(self.configs)
  101. for cname, cinfo in kwargs.pop('configs', {}).items():
  102. target = configs.setdefault(cname, {})
  103. if 'path' in cinfo:
  104. target['path'] = cinfo['path']
  105. if 'map' in cinfo:
  106. target_map = target.setdefault('map', self.config_types[0]())
  107. target_map = self.config_merge(target_map, cinfo['map'])
  108. target['map'] = target_map
  109. self.configs = configs
  110. if not self.name:
  111. if not self.program:
  112. raise ValueError('"{0}" object must specify "program" parameter'.format(self.__class__.__name__))
  113. self.name = os.path.basename(self.program)
  114. self.process = None
  115. self.created_parent_dir = False
  116. self._setup_done = False
  117. dirtree = set(self.dirtree)
  118. dirtree.update(kwargs.pop('dirtree', []))
  119. self.dirtree = dirtree
  120. # Register the exit clean-up before making anything needing clean-up
  121. atexit.register(self.cleanup)
  122. def __enter__(self):
  123. pass
  124. def __exit__(self, typ, value, traceback):
  125. pass
  126. @property
  127. def test_dir(self):
  128. '''Directory that will contains all of the static and dynamic files for the daemon'''
  129. return os.path.join(self.parent_dir, self._root_dir)
  130. def config_file_get(self, config):
  131. '''Get the filename (viz. path) to the configuration file'''
  132. cfgf = self.configs[config].get('path')
  133. if cfgf:
  134. cfgf.format(**self.config_subs())
  135. else:
  136. cfgf = os.path.join(self.config_dir, config)
  137. return cfgf
  138. def config_dir_get(self, config):
  139. '''Get the parent directory for the configuration file'''
  140. return os.path.dirname(self.config_file_get(config))
  141. @property
  142. def config_dirs(self):
  143. '''Return a list of configuration directories'''
  144. cdirs = [self.config_dir_get(config) for config in self.configs.keys()]
  145. return cdirs
  146. def abs_path(self, path):
  147. '''Absolute path of file including the test_dir'''
  148. return os.path.join(self.test_dir, path)
  149. @property
  150. def start_pid(self):
  151. '''PID of the called script prior to daemonizing.'''
  152. return self.process.pid if self.process else None
  153. @property
  154. def parent_dir(self):
  155. '''
  156. Directory that contains everything generated for running scripts - possibly
  157. for multiple scripts.
  158. '''
  159. if self._parent_dir is None:
  160. self.created_parent_dir = True
  161. self._parent_dir = tempfile.mkdtemp(prefix='salt-testdaemon-')
  162. else:
  163. self._parent_dir = os.path.abspath(os.path.normpath(self._parent_dir))
  164. if not os.path.exists(self._parent_dir):
  165. self.created_parent_dir = True
  166. os.makedirs(self._parent_dir)
  167. elif not os.path.isdir(self._parent_dir):
  168. raise ValueError('Parent path "{0}" exists but is not a directory'.format(self._parent_dir))
  169. return self._parent_dir
  170. def config_write(self, config):
  171. '''Write out the config to a file'''
  172. if not config:
  173. return
  174. cpath = self.abs_path(self.config_file_get(config))
  175. with salt.utils.files.fopen(cpath, 'w') as cfo:
  176. cfg = self.config_stringify(config)
  177. log.debug('Writing configuration for {0} to {1}:\n{2}'.format(self.name, cpath, cfg))
  178. cfo.write(cfg)
  179. cfo.flush()
  180. def configs_write(self):
  181. '''Write all configuration files'''
  182. for config in self.configs:
  183. self.config_write(config)
  184. def config_type(self, config):
  185. '''Check if a configuration is an acceptable type.'''
  186. return isinstance(config, self.config_types)
  187. def config_cast(self, config):
  188. '''Cast a configuration to the internal expected type.'''
  189. if not self.config_type(config):
  190. config = self.config_caster(config)
  191. return config
  192. def config_subs(self):
  193. '''Get the substitution values for use to generate the config'''
  194. subs = dict([(attr, getattr(self, attr, None)) for attr in self.config_attrs])
  195. for key, val in self.config_vals.items():
  196. subs[key] = val.format(**subs)
  197. return subs
  198. def config_stringify(self, config):
  199. '''Get the configuration as a string'''
  200. cfg = self.config_get(config)
  201. cfg.format(**self.config_subs())
  202. return cfg
  203. def config_merge(self, base, overrides):
  204. '''Merge two configuration hunks'''
  205. base = self.config_cast(base)
  206. overrides = self.config_cast(overrides)
  207. return ''.join([base, overrides])
  208. def config_get(self, config):
  209. '''Get the configuration data'''
  210. return self.configs[config]
  211. def config_set(self, config, val):
  212. '''Set the configuration data'''
  213. self.configs[config] = val
  214. def make_dirtree(self):
  215. '''Create directory structure.'''
  216. subdirs = []
  217. for branch in self.dirtree:
  218. log.debug('checking dirtree: {0}'.format(branch))
  219. if not branch:
  220. continue
  221. if isinstance(branch, six.string_types) and branch[0] == '&':
  222. log.debug('Looking up dirtree branch "{0}"'.format(branch))
  223. try:
  224. dirattr = getattr(self, branch[1:], None)
  225. log.debug('dirtree "{0}" => "{1}"'.format(branch, dirattr))
  226. except AttributeError:
  227. raise ValueError(
  228. 'Unable to find dirtree attribute "{0}" on object "{1}.name = {2}: {3}"'.format(
  229. branch, self.__class__.__name__, self.name, dir(self),
  230. )
  231. )
  232. if not dirattr:
  233. continue
  234. if isinstance(dirattr, six.string_types):
  235. subdirs.append(dirattr)
  236. elif hasattr(dirattr, '__iter__'):
  237. subdirs.extend(dirattr)
  238. else:
  239. raise TypeError("Branch type of {0} in dirtree is unhandled".format(branch))
  240. elif isinstance(branch, six.string_types):
  241. subdirs.append(branch)
  242. else:
  243. raise TypeError("Branch type of {0} in dirtree is unhandled".format(branch))
  244. for subdir in subdirs:
  245. path = self.abs_path(subdir)
  246. if not os.path.exists(path):
  247. log.debug('make_dirtree: {0}'.format(path))
  248. os.makedirs(path)
  249. def setup(self, *args, **kwargs):
  250. '''Create any scaffolding for run-time'''
  251. # unused
  252. _ = args, kwargs
  253. if not self._setup_done:
  254. self.make_dirtree()
  255. self.configs_write()
  256. self._setup_done = True
  257. def cleanup(self, *args, **kwargs):
  258. ''' Clean out scaffolding of setup() and any run-time generated files.'''
  259. # Unused for now
  260. _ = (args, kwargs)
  261. if self.process:
  262. try:
  263. self.process.kill()
  264. self.process.wait()
  265. except OSError:
  266. pass
  267. if os.path.exists(self.test_dir):
  268. shutil.rmtree(self.test_dir)
  269. if self.created_parent_dir and os.path.exists(self.parent_dir):
  270. shutil.rmtree(self.parent_dir)
  271. def run(
  272. self,
  273. args=None,
  274. catch_stderr=False,
  275. with_retcode=False,
  276. timeout=None,
  277. raw=False,
  278. env=None,
  279. verbatim_args=False,
  280. verbatim_env=False,
  281. ):
  282. '''
  283. Execute a command possibly using a supplied environment.
  284. :param args:
  285. A command string or a command sequence of arguments for the program.
  286. :param catch_stderr: A boolean whether to capture and return stderr.
  287. :param with_retcode: A boolean whether to return the exit code.
  288. :param timeout: A float of how long to wait for the process to
  289. complete before it is killed.
  290. :param raw: A boolean whether to return buffer strings for stdout and
  291. stderr or sequences of output lines.
  292. :param env: A dictionary of environment key/value settings for the
  293. command.
  294. :param verbatim_args: A boolean whether to automatically add inferred arguments.
  295. :param verbatim_env: A boolean whether to automatically add inferred
  296. environment values.
  297. :return list: (stdout [,stderr] [,retcode])
  298. '''
  299. # unused for now
  300. _ = verbatim_args
  301. self.setup()
  302. if args is None:
  303. args = []
  304. if env is None:
  305. env = {}
  306. env_delta = {}
  307. env_delta.update(self.env)
  308. env_delta.update(env)
  309. if not verbatim_env:
  310. env_pypath = env_delta.get('PYTHONPATH', os.environ.get('PYTHONPATH'))
  311. if not env_pypath:
  312. env_pypath = sys.path
  313. else:
  314. env_pypath = env_pypath.split(':')
  315. for path in sys.path:
  316. if path not in env_pypath:
  317. env_pypath.append(path)
  318. # Always ensure that the test tree is searched first for python modules
  319. if RUNTIME_VARS.CODE_DIR != env_pypath[0]:
  320. env_pypath.insert(0, RUNTIME_VARS.CODE_DIR)
  321. if salt.utils.platform.is_windows():
  322. env_delta['PYTHONPATH'] = ';'.join(env_pypath)
  323. else:
  324. env_delta['PYTHONPATH'] = ':'.join(env_pypath)
  325. cmd_env = dict(os.environ)
  326. cmd_env.update(env_delta)
  327. if salt.utils.platform.is_windows() and six.PY2:
  328. for k, v in cmd_env.items():
  329. if isinstance(k, six.text_type) or isinstance(v, six.text_type):
  330. cmd_env[k.encode('ascii')] = v.encode('ascii')
  331. popen_kwargs = {
  332. 'shell': self.shell,
  333. 'stdout': subprocess.PIPE,
  334. 'env': cmd_env,
  335. }
  336. if catch_stderr is True:
  337. popen_kwargs['stderr'] = subprocess.PIPE
  338. if not sys.platform.lower().startswith('win'):
  339. popen_kwargs['close_fds'] = True
  340. def detach_from_parent_group():
  341. '''
  342. A utility function that prevents child process from getting parent signals.
  343. '''
  344. os.setpgrp()
  345. popen_kwargs['preexec_fn'] = detach_from_parent_group
  346. if salt.utils.platform.is_windows():
  347. self.argv = ['python.exe', self.program]
  348. else:
  349. self.argv = [self.program]
  350. self.argv.extend(args)
  351. log.debug('TestProgram.run: %s Environment %s', self.argv, env_delta)
  352. process = subprocess.Popen(self.argv, **popen_kwargs)
  353. self.process = process
  354. if timeout is not None:
  355. stop_at = datetime.now() + timedelta(seconds=timeout)
  356. term_sent = False
  357. while True:
  358. process.poll()
  359. if datetime.now() > stop_at:
  360. try:
  361. terminate_process(pid=process.pid, kill_children=True)
  362. process.wait()
  363. except OSError as exc:
  364. if exc.errno != errno.ESRCH:
  365. raise
  366. out = process.stdout.read().splitlines()
  367. out.extend([
  368. 'Process took more than {0} seconds to complete. '
  369. 'Process Killed!'.format(timeout)
  370. ])
  371. if catch_stderr:
  372. err = process.stderr.read().splitlines()
  373. if with_retcode:
  374. return out, err, process.returncode
  375. else:
  376. return out, err
  377. if with_retcode:
  378. return out, process.returncode
  379. else:
  380. return out
  381. if process.returncode is not None:
  382. break
  383. if catch_stderr:
  384. if sys.version_info < (2, 7):
  385. # On python 2.6, the subprocess'es communicate() method uses
  386. # select which, is limited by the OS to 1024 file descriptors
  387. # We need more available descriptors to run the tests which
  388. # need the stderr output.
  389. # So instead of .communicate() we wait for the process to
  390. # finish, but, as the python docs state "This will deadlock
  391. # when using stdout=PIPE and/or stderr=PIPE and the child
  392. # process generates enough output to a pipe such that it
  393. # blocks waiting for the OS pipe buffer to accept more data.
  394. # Use communicate() to avoid that." <- a catch, catch situation
  395. #
  396. # Use this work around were it's needed only, python 2.6
  397. process.wait()
  398. out = process.stdout.read()
  399. err = process.stderr.read()
  400. else:
  401. out, err = process.communicate()
  402. # Force closing stderr/stdout to release file descriptors
  403. if process.stdout is not None:
  404. process.stdout.close()
  405. if process.stderr is not None:
  406. process.stderr.close()
  407. # pylint: disable=maybe-no-member
  408. try:
  409. if with_retcode:
  410. if out is not None and err is not None:
  411. if not raw:
  412. return out.splitlines(), err.splitlines(), process.returncode
  413. else:
  414. return out, err, process.returncode
  415. return out.splitlines(), [], process.returncode
  416. else:
  417. if out is not None and err is not None:
  418. if not raw:
  419. return out.splitlines(), err.splitlines()
  420. else:
  421. return out, err
  422. if not raw:
  423. return out.splitlines(), []
  424. else:
  425. return out, []
  426. finally:
  427. try:
  428. process.terminate()
  429. except OSError as err:
  430. # process already terminated
  431. pass
  432. # pylint: enable=maybe-no-member
  433. data = process.communicate()
  434. process.stdout.close()
  435. try:
  436. if with_retcode:
  437. if not raw:
  438. return data[0].splitlines(), process.returncode
  439. else:
  440. return data[0], process.returncode
  441. else:
  442. if not raw:
  443. return data[0].splitlines()
  444. else:
  445. return data[0]
  446. finally:
  447. try:
  448. process.terminate()
  449. except OSError as err:
  450. # process already terminated
  451. pass
  452. class TestSaltProgramMeta(TestProgramMeta):
  453. '''
  454. A Meta-class to set self.script from the class name when it is
  455. not specifically set by a "script" argument.
  456. '''
  457. def __new__(mcs, name, bases, attrs):
  458. if attrs.get('script') is None:
  459. if 'Salt' in name:
  460. script = 'salt-{0}'.format(name.rsplit('Salt', 1)[-1].lower())
  461. if script is None:
  462. raise AttributeError(
  463. 'Class {0}: Unable to set "script" attribute: class name'
  464. ' must include "Salt" or "script" must be explicitly set.'.format(name)
  465. )
  466. attrs['script'] = script
  467. config_base = {}
  468. configs = {}
  469. for base in bases:
  470. if 'Salt' not in base.__name__:
  471. continue
  472. config_base.update(getattr(base, 'config_base', {}))
  473. configs.update(getattr(base, 'configs', {}))
  474. config_base.update(attrs.get('config_base', {}))
  475. attrs['config_base'] = config_base
  476. configs.update(attrs.get('configs', {}))
  477. attrs['configs'] = configs
  478. return super(TestSaltProgramMeta, mcs).__new__(mcs, name, bases, attrs)
  479. class TestSaltProgram(six.with_metaclass(TestSaltProgramMeta, TestProgram, ScriptPathMixin)):
  480. '''
  481. This is like TestProgram but with some functions to run a salt-specific
  482. auxiliary program.
  483. '''
  484. config_types = (dict,)
  485. config_attrs = set([
  486. 'log_dir',
  487. 'script_dir',
  488. ])
  489. pub_port = 4505
  490. ret_port = 4506
  491. for port in [pub_port, ret_port]:
  492. sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  493. try:
  494. connect = sock.bind(('localhost', port))
  495. except (socket.error, OSError):
  496. # these ports are already in use, use different ones
  497. pub_port = 4606
  498. ret_port = 4607
  499. break
  500. sock.close()
  501. config_base = {
  502. 'root_dir': '{test_dir}',
  503. 'publish_port': pub_port,
  504. 'ret_port': ret_port,
  505. }
  506. configs = {}
  507. config_dir = os.path.join('etc', 'salt')
  508. log_dir = os.path.join('var', 'log', 'salt')
  509. dirtree = [
  510. '&log_dir',
  511. '&script_dir',
  512. ]
  513. script = ''
  514. script_dir = 'bin'
  515. @staticmethod
  516. def config_caster(cfg):
  517. return salt.utils.yaml.safe_load(cfg)
  518. def __init__(self, *args, **kwargs):
  519. if len(args) < 2 and 'program' not in kwargs:
  520. # This is effectively a place-holder - it gets set correctly after super()
  521. kwargs['program'] = self.script
  522. super(TestSaltProgram, self).__init__(*args, **kwargs)
  523. self.program = self.get_script_path(self.script)
  524. def config_merge(self, base, overrides):
  525. _base = self.config_cast(copy.deepcopy(base))
  526. _overrides = self.config_cast(overrides)
  527. # NOTE: this simple update will not work for deep dictionaries
  528. _base.update(copy.deepcopy(_overrides))
  529. return _base
  530. def config_get(self, config):
  531. cfg_base = {}
  532. for key, val in self.config_base.items():
  533. _val = val
  534. if val and isinstance(val, six.string_types) and val[0] == '&':
  535. _val = getattr(self, val[1:], None)
  536. if _val is None:
  537. continue
  538. cfg_base[key] = _val
  539. if config in self.configs:
  540. cfg = {}
  541. for key, val in self.configs.get(config, {}).get('map', {}).items():
  542. _val = val
  543. if val and isinstance(val, six.string_types) and val[0] == '&':
  544. _val = getattr(self, val[1:], None)
  545. if _val is None:
  546. continue
  547. cfg[key] = _val
  548. cfg = self.config_merge(cfg_base, cfg)
  549. log.debug('Generated config => {0}'.format(cfg))
  550. return cfg
  551. def config_stringify(self, config):
  552. '''Transform the configuration data into a string (suitable to write to a file)'''
  553. subs = self.config_subs()
  554. cfg = {}
  555. for key, val in self.config_get(config).items():
  556. if isinstance(val, six.string_types):
  557. cfg[key] = val.format(**subs)
  558. else:
  559. cfg[key] = val
  560. return salt.utils.yaml.safe_dump(cfg, default_flow_style=False)
  561. def run(self, **kwargs): # pylint: disable=arguments-differ
  562. if not kwargs.get('verbatim_args'):
  563. args = kwargs.setdefault('args', [])
  564. if '-c' not in args and '--config-dir' not in args:
  565. args.extend(['--config-dir', self.abs_path(self.config_dir)])
  566. return super(TestSaltProgram, self).run(**kwargs)
  567. class TestProgramSalt(TestSaltProgram):
  568. '''Class to manage salt'''
  569. configs = {'master': {}}
  570. script = 'salt'
  571. class TestProgramSaltCall(TestSaltProgram):
  572. '''Class to manage salt-call'''
  573. configs = {'minion': {'map': {'id': '{name}'}}}
  574. class TestProgramSaltRun(TestSaltProgram):
  575. '''Class to manage salt-run'''
  576. configs = {'master': {}}
  577. def __init__(self, *args, **kwargs):
  578. cfgb = kwargs.setdefault('config_base', {})
  579. _ = cfgb.setdefault('user', getpass.getuser())
  580. super(TestProgramSaltRun, self).__init__(*args, **kwargs)
  581. class TestDaemon(TestProgram):
  582. '''
  583. Run one of the standard daemons
  584. '''
  585. script = None
  586. pid_file = None
  587. pid_dir = os.path.join('var', 'run')
  588. dirtree = [
  589. '&pid_dir',
  590. ]
  591. def __init__(self, *args, **kwargs):
  592. self.script = kwargs.pop('script', self.script)
  593. self.pid_file = kwargs.pop('pid_file', self.pid_file if self.pid_file else '{0}.pid'.format(self.script))
  594. self.pid_dir = kwargs.pop('pid_dir', self.pid_dir)
  595. self._shutdown = False
  596. if not args and 'program' not in kwargs:
  597. # This is effectively a place-holder - it gets set correctly after super()
  598. kwargs['program'] = self.script
  599. super(TestDaemon, self).__init__(*args, **kwargs)
  600. @property
  601. def pid_path(self):
  602. '''Path to the pid file created by the daemon'''
  603. return os.path.join(self.pid_dir, self.pid_file) if os.path.sep not in self.pid_file else self.pid_file
  604. @property
  605. def daemon_pid(self):
  606. '''Return the daemon PID'''
  607. daemon_pid = None
  608. pid_path = self.abs_path(self.pid_path)
  609. if salt.utils.process.check_pidfile(pid_path):
  610. daemon_pid = salt.utils.process.get_pidfile(pid_path)
  611. return daemon_pid
  612. def wait_for_daemon_pid(self, timeout=10):
  613. '''Wait up to timeout seconds for the PID file to appear and return the PID'''
  614. endtime = time.time() + timeout
  615. while True:
  616. pid = self.daemon_pid
  617. if pid:
  618. return pid
  619. if endtime < time.time():
  620. raise TimeoutError('Timeout waiting for "{0}" pid in "{1}"'.format(
  621. self.name, self.abs_path(self.pid_path)
  622. ))
  623. time.sleep(0.2)
  624. def is_running(self):
  625. '''Is the daemon running?'''
  626. ret = False
  627. if not self._shutdown:
  628. try:
  629. pid = self.wait_for_daemon_pid()
  630. ret = psutils.pid_exists(pid)
  631. except TimeoutError:
  632. pass
  633. return ret
  634. def find_orphans(self, cmdline):
  635. '''Find orphaned processes matching the specified cmdline'''
  636. ret = []
  637. if six.PY3:
  638. cmdline = ' '.join(cmdline)
  639. for proc in psutils.process_iter():
  640. try:
  641. for item in proc.cmdline():
  642. if cmdline in item:
  643. ret.append(proc)
  644. except psutils.NoSuchProcess:
  645. # Process exited between when process_iter was invoked and
  646. # when we tried to invoke this instance's cmdline() func.
  647. continue
  648. except psutils.AccessDenied:
  649. # We might get access denied if not running as root
  650. if not salt.utils.platform.is_windows():
  651. pinfo = proc.as_dict(attrs=['pid', 'name', 'username'])
  652. log.error('Unable to access process %s, '
  653. 'running command %s as user %s',
  654. pinfo['pid'], pinfo['name'], pinfo['username'])
  655. continue
  656. else:
  657. cmd_len = len(cmdline)
  658. for proc in psutils.process_iter():
  659. try:
  660. proc_cmdline = proc.cmdline()
  661. except psutils.NoSuchProcess:
  662. # Process exited between when process_iter was invoked and
  663. # when we tried to invoke this instance's cmdline() func.
  664. continue
  665. except psutils.AccessDenied:
  666. # We might get access denied if not running as root
  667. if not salt.utils.platform.is_windows():
  668. pinfo = proc.as_dict(attrs=['pid', 'name', 'username'])
  669. log.error('Unable to access process %s, '
  670. 'running command %s as user %s',
  671. pinfo['pid'], pinfo['name'], pinfo['username'])
  672. continue
  673. if any((cmdline == proc_cmdline[n:n + cmd_len])
  674. for n in range(len(proc_cmdline) - cmd_len + 1)):
  675. ret.append(proc)
  676. return ret
  677. def shutdown(self, signum=signal.SIGTERM, timeout=10, wait_for_orphans=0):
  678. '''Shutdown a running daemon'''
  679. if not self._shutdown:
  680. try:
  681. pid = self.wait_for_daemon_pid(timeout)
  682. terminate_process(pid=pid, kill_children=True)
  683. except TimeoutError:
  684. pass
  685. if self.process:
  686. terminate_process(pid=self.process.pid, kill_children=True)
  687. self.process.wait()
  688. if wait_for_orphans:
  689. # NOTE: The process for finding orphans is greedy, it just
  690. # looks for processes with the same cmdline which are owned by
  691. # PID 1.
  692. orphans = self.find_orphans(self.argv)
  693. last = time.time()
  694. while True:
  695. if orphans:
  696. log.debug(
  697. 'Terminating orphaned child processes: %s',
  698. orphans
  699. )
  700. terminate_process_list(orphans)
  701. last = time.time()
  702. if (time.time() - last) >= wait_for_orphans:
  703. break
  704. time.sleep(0.25)
  705. orphans = self.find_orphans(self.argv)
  706. self.process = None
  707. self._shutdown = True
  708. def cleanup(self, *args, **kwargs):
  709. '''Remove left-over scaffolding - antithesis of setup()'''
  710. # Shutdown if not alreadt shutdown
  711. self.shutdown()
  712. super(TestDaemon, self).cleanup(*args, **kwargs)
  713. class TestSaltDaemon(six.with_metaclass(TestSaltProgramMeta, TestDaemon, TestSaltProgram)):
  714. '''
  715. A class to run arbitrary salt daemons (master, minion, syndic, etc.)
  716. '''
  717. class TestDaemonSaltMaster(TestSaltDaemon):
  718. '''
  719. Manager for salt-master daemon.
  720. '''
  721. configs = {'master': {}}
  722. def __init__(self, *args, **kwargs):
  723. cfgb = kwargs.setdefault('config_base', {})
  724. _ = cfgb.setdefault('user', getpass.getuser())
  725. super(TestDaemonSaltMaster, self).__init__(*args, **kwargs)
  726. class TestDaemonSaltMinion(TestSaltDaemon):
  727. '''
  728. Manager for salt-minion daemon.
  729. '''
  730. configs = {'minion': {'map': {'id': '{name}'}}}
  731. def __init__(self, *args, **kwargs):
  732. cfgb = kwargs.setdefault('config_base', {})
  733. _ = cfgb.setdefault('user', getpass.getuser())
  734. super(TestDaemonSaltMinion, self).__init__(*args, **kwargs)
  735. class TestDaemonSaltApi(TestSaltDaemon):
  736. '''
  737. Manager for salt-api daemon.
  738. '''
  739. class TestDaemonSaltSyndic(TestSaltDaemon):
  740. '''
  741. Manager for salt-syndic daemon.
  742. '''
  743. configs = {
  744. 'master': {'map': {'syndic_master': 'localhost'}},
  745. 'minion': {'map': {'id': '{name}'}},
  746. }
  747. def __init__(self, *args, **kwargs):
  748. cfgb = kwargs.setdefault('config_base', {})
  749. _ = cfgb.setdefault('user', getpass.getuser())
  750. super(TestDaemonSaltSyndic, self).__init__(*args, **kwargs)
  751. class TestDaemonSaltProxy(TestSaltDaemon):
  752. '''
  753. Manager for salt-proxy daemon.
  754. '''
  755. pid_file = 'salt-minion.pid'
  756. configs = {'proxy': {}}
  757. def __init__(self, *args, **kwargs):
  758. cfgb = kwargs.setdefault('config_base', {})
  759. _ = cfgb.setdefault('user', getpass.getuser())
  760. super(TestDaemonSaltProxy, self).__init__(*args, **kwargs)
  761. def run(self, **kwargs):
  762. if not kwargs.get('verbatim_args'):
  763. args = kwargs.setdefault('args', [])
  764. if '--proxyid' not in args:
  765. args.extend(['--proxyid', self.name])
  766. return super(TestDaemonSaltProxy, self).run(**kwargs)
  767. class TestProgramCase(TestCase):
  768. '''
  769. Utilities for unit tests that use TestProgram()
  770. '''
  771. def setUp(self):
  772. # Setup for scripts
  773. if not getattr(self, '_test_dir', None):
  774. self._test_dir = tempfile.mkdtemp(prefix='salt-testdaemon-')
  775. super(TestProgramCase, self).setUp()
  776. def tearDown(self):
  777. # shutdown for scripts
  778. if self._test_dir and os.path.sep == self._test_dir[0]:
  779. shutil.rmtree(self._test_dir)
  780. self._test_dir = None
  781. super(TestProgramCase, self).tearDown()
  782. def assert_exit_status(self, status, ex_status, message=None, stdout=None, stderr=None):
  783. '''
  784. Helper function to verify exit status and emit failure information.
  785. '''
  786. ex_val = getattr(exitcodes, ex_status)
  787. _message = '' if not message else ' ({0})'.format(message)
  788. _stdout = '' if not stdout else '\nstdout: {0}'.format(stdout)
  789. _stderr = '' if not stderr else '\nstderr: {0}'.format(stderr)
  790. self.assertEqual(
  791. status,
  792. ex_val,
  793. 'Exit status was {0}, must be {1} (salt.default.exitcodes.{2}){3}{4}{5}'.format(
  794. status,
  795. ex_val,
  796. ex_status,
  797. _message,
  798. _stdout,
  799. _stderr,
  800. )
  801. )