1
0

helpers.py 57 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658
  1. # -*- coding: utf-8 -*-
  2. '''
  3. :copyright: Copyright 2013-2017 by the SaltStack Team, see AUTHORS for more details.
  4. :license: Apache 2.0, see LICENSE for more details.
  5. tests.support.helpers
  6. ~~~~~~~~~~~~~~~~~~~~~
  7. Test support helpers
  8. '''
  9. # pylint: disable=repr-flag-used-in-string,wrong-import-order
  10. # Import Python libs
  11. from __future__ import absolute_import, print_function, unicode_literals
  12. import base64
  13. import errno
  14. import functools
  15. import inspect
  16. import logging
  17. import os
  18. import random
  19. import shutil
  20. import signal
  21. import socket
  22. import string
  23. import subprocess
  24. import sys
  25. import tempfile
  26. import textwrap
  27. import threading
  28. import time
  29. import tornado.ioloop
  30. import tornado.web
  31. import types
  32. # Import 3rd-party libs
  33. import psutil # pylint: disable=3rd-party-module-not-gated
  34. from salt.ext import six
  35. from salt.ext.six.moves import range, builtins # pylint: disable=import-error,redefined-builtin
  36. try:
  37. from pytestsalt.utils import get_unused_localhost_port # pylint: disable=unused-import
  38. except ImportError:
  39. from tests.integration import get_unused_localhost_port
  40. # Import Salt Tests Support libs
  41. from tests.support.unit import skip, _id
  42. from tests.support.mock import patch
  43. from tests.support.paths import FILES, TMP
  44. # Import Salt libs
  45. import salt.utils.files
  46. import salt.utils.platform
  47. import salt.utils.stringutils
  48. if salt.utils.platform.is_windows():
  49. import salt.utils.win_functions
  50. else:
  51. import pwd
  52. log = logging.getLogger(__name__)
  53. HAS_SYMLINKS = None
  54. def no_symlinks():
  55. '''
  56. Check if git is installed and has symlinks enabled in the configuration.
  57. '''
  58. global HAS_SYMLINKS
  59. if HAS_SYMLINKS is not None:
  60. return not HAS_SYMLINKS
  61. output = ''
  62. try:
  63. output = subprocess.Popen(
  64. ['git', 'config', '--get', 'core.symlinks'],
  65. cwd=TMP,
  66. stdout=subprocess.PIPE).communicate()[0]
  67. except OSError as exc:
  68. if exc.errno != errno.ENOENT:
  69. raise
  70. except subprocess.CalledProcessError:
  71. # git returned non-zero status
  72. pass
  73. HAS_SYMLINKS = False
  74. if output.strip() == 'true':
  75. HAS_SYMLINKS = True
  76. return not HAS_SYMLINKS
  77. def destructiveTest(caller):
  78. '''
  79. Mark a test case as a destructive test for example adding or removing users
  80. from your system.
  81. .. code-block:: python
  82. class MyTestCase(TestCase):
  83. @destructiveTest
  84. def test_create_user(self):
  85. pass
  86. '''
  87. if inspect.isclass(caller):
  88. # We're decorating a class
  89. old_setup = getattr(caller, 'setUp', None)
  90. def setUp(self, *args, **kwargs):
  91. if os.environ.get('DESTRUCTIVE_TESTS', 'False').lower() == 'false':
  92. self.skipTest('Destructive tests are disabled')
  93. if old_setup is not None:
  94. old_setup(self, *args, **kwargs)
  95. caller.setUp = setUp
  96. return caller
  97. # We're simply decorating functions
  98. @functools.wraps(caller)
  99. def wrap(cls):
  100. if os.environ.get('DESTRUCTIVE_TESTS', 'False').lower() == 'false':
  101. cls.skipTest('Destructive tests are disabled')
  102. return caller(cls)
  103. return wrap
  104. def expensiveTest(caller):
  105. '''
  106. Mark a test case as an expensive test, for example, a test which can cost
  107. money(Salt's cloud provider tests).
  108. .. code-block:: python
  109. class MyTestCase(TestCase):
  110. @expensiveTest
  111. def test_create_user(self):
  112. pass
  113. '''
  114. if inspect.isclass(caller):
  115. # We're decorating a class
  116. old_setup = getattr(caller, 'setUp', None)
  117. def setUp(self, *args, **kwargs):
  118. if os.environ.get('EXPENSIVE_TESTS', 'False').lower() == 'false':
  119. self.skipTest('Expensive tests are disabled')
  120. if old_setup is not None:
  121. old_setup(self, *args, **kwargs)
  122. caller.setUp = setUp
  123. return caller
  124. # We're simply decorating functions
  125. @functools.wraps(caller)
  126. def wrap(cls):
  127. if os.environ.get('EXPENSIVE_TESTS', 'False').lower() == 'false':
  128. cls.skipTest('Expensive tests are disabled')
  129. return caller(cls)
  130. return wrap
  131. def flaky(caller=None, condition=True, attempts=4):
  132. '''
  133. Mark a test as flaky. The test will attempt to run five times,
  134. looking for a successful run. After an immediate second try,
  135. it will use an exponential backoff starting with one second.
  136. .. code-block:: python
  137. class MyTestCase(TestCase):
  138. @flaky
  139. def test_sometimes_works(self):
  140. pass
  141. '''
  142. if caller is None:
  143. return functools.partial(flaky, condition=condition, attempts=attempts)
  144. if isinstance(condition, bool) and condition is False:
  145. # Don't even decorate
  146. return caller
  147. elif callable(condition):
  148. if condition() is False:
  149. # Don't even decorate
  150. return caller
  151. if inspect.isclass(caller):
  152. attrs = [n for n in dir(caller) if n.startswith('test_')]
  153. for attrname in attrs:
  154. try:
  155. function = getattr(caller, attrname)
  156. if not inspect.isfunction(function) and not inspect.ismethod(function):
  157. continue
  158. setattr(caller, attrname, flaky(caller=function, condition=condition, attempts=attempts))
  159. except Exception as exc:
  160. log.exception(exc)
  161. continue
  162. return caller
  163. @functools.wraps(caller)
  164. def wrap(cls):
  165. for attempt in range(0, attempts):
  166. try:
  167. return caller(cls)
  168. except Exception as exc:
  169. if attempt >= attempts -1:
  170. raise exc
  171. backoff_time = attempt ** 2
  172. log.info(
  173. 'Found Exception. Waiting %s seconds to retry.',
  174. backoff_time
  175. )
  176. time.sleep(backoff_time)
  177. return cls
  178. return wrap
  179. def requires_sshd_server(caller):
  180. '''
  181. Mark a test as requiring the tests SSH daemon running.
  182. .. code-block:: python
  183. class MyTestCase(TestCase):
  184. @requiresSshdServer
  185. def test_create_user(self):
  186. pass
  187. '''
  188. if inspect.isclass(caller):
  189. # We're decorating a class
  190. old_setup = getattr(caller, 'setUp', None)
  191. def setUp(self, *args, **kwargs):
  192. if os.environ.get('SSH_DAEMON_RUNNING', 'False').lower() == 'false':
  193. self.skipTest('SSH tests are disabled')
  194. if old_setup is not None:
  195. old_setup(self, *args, **kwargs)
  196. caller.setUp = setUp
  197. return caller
  198. # We're simply decorating functions
  199. @functools.wraps(caller)
  200. def wrap(cls):
  201. if os.environ.get('SSH_DAEMON_RUNNING', 'False').lower() == 'false':
  202. cls.skipTest('SSH tests are disabled')
  203. return caller(cls)
  204. return wrap
  205. class RedirectStdStreams(object):
  206. '''
  207. Temporarily redirect system output to file like objects.
  208. Default is to redirect to `os.devnull`, which just mutes output, `stdout`
  209. and `stderr`.
  210. '''
  211. def __init__(self, stdout=None, stderr=None):
  212. # Late import
  213. import salt.utils.files
  214. if stdout is None:
  215. stdout = salt.utils.files.fopen(os.devnull, 'w') # pylint: disable=resource-leakage
  216. if stderr is None:
  217. stderr = salt.utils.files.fopen(os.devnull, 'w') # pylint: disable=resource-leakage
  218. self.__stdout = stdout
  219. self.__stderr = stderr
  220. self.__redirected = False
  221. self.patcher = patch.multiple(sys, stderr=self.__stderr, stdout=self.__stdout)
  222. def __enter__(self):
  223. self.redirect()
  224. return self
  225. def __exit__(self, exc_type, exc_value, traceback):
  226. self.unredirect()
  227. def redirect(self):
  228. self.old_stdout = sys.stdout
  229. self.old_stdout.flush()
  230. self.old_stderr = sys.stderr
  231. self.old_stderr.flush()
  232. self.patcher.start()
  233. self.__redirected = True
  234. def unredirect(self):
  235. if not self.__redirected:
  236. return
  237. try:
  238. self.__stdout.flush()
  239. self.__stdout.close()
  240. except ValueError:
  241. # already closed?
  242. pass
  243. try:
  244. self.__stderr.flush()
  245. self.__stderr.close()
  246. except ValueError:
  247. # already closed?
  248. pass
  249. self.patcher.stop()
  250. def flush(self):
  251. if self.__redirected:
  252. try:
  253. self.__stdout.flush()
  254. except Exception:
  255. pass
  256. try:
  257. self.__stderr.flush()
  258. except Exception:
  259. pass
  260. class TestsLoggingHandler(object):
  261. '''
  262. Simple logging handler which can be used to test if certain logging
  263. messages get emitted or not:
  264. .. code-block:: python
  265. with TestsLoggingHandler() as handler:
  266. # (...) Do what ever you wish here
  267. handler.messages # here are the emitted log messages
  268. '''
  269. def __init__(self, level=0, format='%(levelname)s:%(message)s'):
  270. self.level = level
  271. self.format = format
  272. self.activated = False
  273. self.prev_logging_level = None
  274. def activate(self):
  275. class Handler(logging.Handler):
  276. def __init__(self, level):
  277. logging.Handler.__init__(self, level)
  278. self.messages = []
  279. def emit(self, record):
  280. self.messages.append(self.format(record))
  281. self.handler = Handler(self.level)
  282. formatter = logging.Formatter(self.format)
  283. self.handler.setFormatter(formatter)
  284. logging.root.addHandler(self.handler)
  285. self.activated = True
  286. # Make sure we're running with the lowest logging level with our
  287. # tests logging handler
  288. current_logging_level = logging.root.getEffectiveLevel()
  289. if current_logging_level > logging.DEBUG:
  290. self.prev_logging_level = current_logging_level
  291. logging.root.setLevel(0)
  292. def deactivate(self):
  293. if not self.activated:
  294. return
  295. logging.root.removeHandler(self.handler)
  296. # Restore previous logging level if changed
  297. if self.prev_logging_level is not None:
  298. logging.root.setLevel(self.prev_logging_level)
  299. @property
  300. def messages(self):
  301. if not self.activated:
  302. return []
  303. return self.handler.messages
  304. def clear(self):
  305. self.handler.messages = []
  306. def __enter__(self):
  307. self.activate()
  308. return self
  309. def __exit__(self, type, value, traceback):
  310. self.deactivate()
  311. self.activated = False
  312. # Mimic some handler attributes and methods
  313. @property
  314. def lock(self):
  315. if self.activated:
  316. return self.handler.lock
  317. def createLock(self):
  318. if self.activated:
  319. return self.handler.createLock()
  320. def acquire(self):
  321. if self.activated:
  322. return self.handler.acquire()
  323. def release(self):
  324. if self.activated:
  325. return self.handler.release()
  326. def relative_import(import_name, relative_from='../'):
  327. '''
  328. Update sys.path to include `relative_from` before importing `import_name`
  329. '''
  330. try:
  331. return __import__(import_name)
  332. except ImportError:
  333. previous_frame = inspect.getframeinfo(inspect.currentframe().f_back)
  334. sys.path.insert(
  335. 0, os.path.realpath(
  336. os.path.join(
  337. os.path.abspath(
  338. os.path.dirname(previous_frame.filename)
  339. ),
  340. relative_from
  341. )
  342. )
  343. )
  344. return __import__(import_name)
  345. class ForceImportErrorOn(object):
  346. '''
  347. This class is meant to be used in mock'ed test cases which require an
  348. ``ImportError`` to be raised.
  349. >>> import os.path
  350. >>> with ForceImportErrorOn('os.path'):
  351. ... import os.path
  352. ...
  353. Traceback (most recent call last):
  354. File "<stdin>", line 2, in <module>
  355. File "salttesting/helpers.py", line 263, in __import__
  356. 'Forced ImportError raised for {0!r}'.format(name)
  357. ImportError: Forced ImportError raised for 'os.path'
  358. >>>
  359. >>> with ForceImportErrorOn(('os', 'path')):
  360. ... import os.path
  361. ... sys.modules.pop('os', None)
  362. ... from os import path
  363. ...
  364. <module 'os' from '/usr/lib/python2.7/os.pyc'>
  365. Traceback (most recent call last):
  366. File "<stdin>", line 4, in <module>
  367. File "salttesting/helpers.py", line 288, in __fake_import__
  368. name, ', '.join(fromlist)
  369. ImportError: Forced ImportError raised for 'from os import path'
  370. >>>
  371. >>> with ForceImportErrorOn(('os', 'path'), 'os.path'):
  372. ... import os.path
  373. ... sys.modules.pop('os', None)
  374. ... from os import path
  375. ...
  376. Traceback (most recent call last):
  377. File "<stdin>", line 2, in <module>
  378. File "salttesting/helpers.py", line 281, in __fake_import__
  379. 'Forced ImportError raised for {0!r}'.format(name)
  380. ImportError: Forced ImportError raised for 'os.path'
  381. >>>
  382. '''
  383. def __init__(self, *module_names):
  384. self.__module_names = {}
  385. for entry in module_names:
  386. if isinstance(entry, (list, tuple)):
  387. modname = entry[0]
  388. self.__module_names[modname] = set(entry[1:])
  389. else:
  390. self.__module_names[entry] = None
  391. self.__original_import = builtins.__import__
  392. self.patcher = patch.object(builtins, '__import__', self.__fake_import__)
  393. def patch_import_function(self):
  394. self.patcher.start()
  395. def restore_import_funtion(self):
  396. self.patcher.stop()
  397. def __fake_import__(self,
  398. name,
  399. globals_={} if six.PY2 else None,
  400. locals_={} if six.PY2 else None,
  401. fromlist=[] if six.PY2 else (),
  402. level=-1 if six.PY2 else 0):
  403. if name in self.__module_names:
  404. importerror_fromlist = self.__module_names.get(name)
  405. if importerror_fromlist is None:
  406. raise ImportError(
  407. 'Forced ImportError raised for {0!r}'.format(name)
  408. )
  409. if importerror_fromlist.intersection(set(fromlist)):
  410. raise ImportError(
  411. 'Forced ImportError raised for {0!r}'.format(
  412. 'from {0} import {1}'.format(
  413. name, ', '.join(fromlist)
  414. )
  415. )
  416. )
  417. return self.__original_import(name, globals_, locals_, fromlist, level)
  418. def __enter__(self):
  419. self.patch_import_function()
  420. return self
  421. def __exit__(self, exc_type, exc_value, traceback):
  422. self.restore_import_funtion()
  423. class MockWraps(object):
  424. '''
  425. Helper class to be used with the mock library.
  426. To be used in the ``wraps`` keyword of ``Mock`` or ``MagicMock`` where you
  427. want to trigger a side effect for X times, and afterwards, call the
  428. original and un-mocked method.
  429. As an example:
  430. >>> def original():
  431. ... print 'original'
  432. ...
  433. >>> def side_effect():
  434. ... print 'side effect'
  435. ...
  436. >>> mw = MockWraps(original, 2, side_effect)
  437. >>> mw()
  438. side effect
  439. >>> mw()
  440. side effect
  441. >>> mw()
  442. original
  443. >>>
  444. '''
  445. def __init__(self, original, expected_failures, side_effect):
  446. self.__original = original
  447. self.__expected_failures = expected_failures
  448. self.__side_effect = side_effect
  449. self.__call_counter = 0
  450. def __call__(self, *args, **kwargs):
  451. try:
  452. if self.__call_counter < self.__expected_failures:
  453. if isinstance(self.__side_effect, types.FunctionType):
  454. return self.__side_effect()
  455. raise self.__side_effect
  456. return self.__original(*args, **kwargs)
  457. finally:
  458. self.__call_counter += 1
  459. def requires_network(only_local_network=False):
  460. '''
  461. Simple decorator which is supposed to skip a test case in case there's no
  462. network connection to the internet.
  463. '''
  464. def decorator(func):
  465. @functools.wraps(func)
  466. def wrapper(cls):
  467. has_local_network = False
  468. # First lets try if we have a local network. Inspired in
  469. # verify_socket
  470. try:
  471. pubsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  472. retsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  473. pubsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
  474. pubsock.bind(('', 18000))
  475. pubsock.close()
  476. retsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
  477. retsock.bind(('', 18001))
  478. retsock.close()
  479. has_local_network = True
  480. except socket.error:
  481. # I wonder if we just have IPV6 support?
  482. try:
  483. pubsock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
  484. retsock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
  485. pubsock.setsockopt(
  486. socket.SOL_SOCKET, socket.SO_REUSEADDR, 1
  487. )
  488. pubsock.bind(('', 18000))
  489. pubsock.close()
  490. retsock.setsockopt(
  491. socket.SOL_SOCKET, socket.SO_REUSEADDR, 1
  492. )
  493. retsock.bind(('', 18001))
  494. retsock.close()
  495. has_local_network = True
  496. except socket.error:
  497. # Let's continue
  498. pass
  499. if only_local_network is True:
  500. if has_local_network is False:
  501. # Since we're only supposed to check local network, and no
  502. # local network was detected, skip the test
  503. cls.skipTest('No local network was detected')
  504. return func(cls)
  505. # We are using the google.com DNS records as numerical IPs to avoid
  506. # DNS lookups which could greatly slow down this check
  507. for addr in ('173.194.41.198', '173.194.41.199', '173.194.41.200',
  508. '173.194.41.201', '173.194.41.206', '173.194.41.192',
  509. '173.194.41.193', '173.194.41.194', '173.194.41.195',
  510. '173.194.41.196', '173.194.41.197'):
  511. sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  512. try:
  513. sock.settimeout(0.25)
  514. sock.connect((addr, 80))
  515. # We connected? Stop the loop
  516. break
  517. except socket.error:
  518. # Let's check the next IP
  519. continue
  520. else:
  521. cls.skipTest('No internet network connection was detected')
  522. finally:
  523. sock.close()
  524. return func(cls)
  525. return wrapper
  526. return decorator
  527. def with_system_user(username, on_existing='delete', delete=True, password=None, groups=None):
  528. '''
  529. Create and optionally destroy a system user to be used within a test
  530. case. The system user is created using the ``user`` salt module.
  531. The decorated testcase function must accept 'username' as an argument.
  532. :param username: The desired username for the system user.
  533. :param on_existing: What to do when the desired username is taken. The
  534. available options are:
  535. * nothing: Do nothing, act as if the user was created.
  536. * delete: delete and re-create the existing user
  537. * skip: skip the test case
  538. '''
  539. if on_existing not in ('nothing', 'delete', 'skip'):
  540. raise RuntimeError(
  541. 'The value of \'on_existing\' can only be one of, '
  542. '\'nothing\', \'delete\' and \'skip\''
  543. )
  544. if not isinstance(delete, bool):
  545. raise RuntimeError(
  546. 'The value of \'delete\' can only be \'True\' or \'False\''
  547. )
  548. def decorator(func):
  549. @functools.wraps(func)
  550. def wrap(cls):
  551. # Let's add the user to the system.
  552. log.debug('Creating system user {0!r}'.format(username))
  553. kwargs = {'timeout': 60, 'groups': groups}
  554. if salt.utils.platform.is_windows():
  555. kwargs.update({'password': password})
  556. create_user = cls.run_function('user.add', [username], **kwargs)
  557. if not create_user:
  558. log.debug('Failed to create system user')
  559. # The user was not created
  560. if on_existing == 'skip':
  561. cls.skipTest(
  562. 'Failed to create system user {0!r}'.format(
  563. username
  564. )
  565. )
  566. if on_existing == 'delete':
  567. log.debug(
  568. 'Deleting the system user {0!r}'.format(
  569. username
  570. )
  571. )
  572. delete_user = cls.run_function(
  573. 'user.delete', [username, True, True]
  574. )
  575. if not delete_user:
  576. cls.skipTest(
  577. 'A user named {0!r} already existed on the '
  578. 'system and re-creating it was not possible'
  579. .format(username)
  580. )
  581. log.debug(
  582. 'Second time creating system user {0!r}'.format(
  583. username
  584. )
  585. )
  586. create_user = cls.run_function('user.add', [username], **kwargs)
  587. if not create_user:
  588. cls.skipTest(
  589. 'A user named {0!r} already existed, was deleted '
  590. 'as requested, but re-creating it was not possible'
  591. .format(username)
  592. )
  593. failure = None
  594. try:
  595. try:
  596. return func(cls, username)
  597. except Exception as exc: # pylint: disable=W0703
  598. log.error(
  599. 'Running {0!r} raised an exception: {1}'.format(
  600. func, exc
  601. ),
  602. exc_info=True
  603. )
  604. # Store the original exception details which will be raised
  605. # a little further down the code
  606. failure = sys.exc_info()
  607. finally:
  608. if delete:
  609. delete_user = cls.run_function(
  610. 'user.delete', [username, True, True], timeout=60
  611. )
  612. if not delete_user:
  613. if failure is None:
  614. log.warning(
  615. 'Although the actual test-case did not fail, '
  616. 'deleting the created system user {0!r} '
  617. 'afterwards did.'.format(username)
  618. )
  619. else:
  620. log.warning(
  621. 'The test-case failed and also did the removal'
  622. ' of the system user {0!r}'.format(username)
  623. )
  624. if failure is not None:
  625. # If an exception was thrown, raise it
  626. six.reraise(failure[0], failure[1], failure[2])
  627. return wrap
  628. return decorator
  629. def with_system_group(group, on_existing='delete', delete=True):
  630. '''
  631. Create and optionally destroy a system group to be used within a test
  632. case. The system user is crated using the ``group`` salt module.
  633. The decorated testcase function must accept 'group' as an argument.
  634. :param group: The desired group name for the system user.
  635. :param on_existing: What to do when the desired username is taken. The
  636. available options are:
  637. * nothing: Do nothing, act as if the group was created
  638. * delete: delete and re-create the existing user
  639. * skip: skip the test case
  640. '''
  641. if on_existing not in ('nothing', 'delete', 'skip'):
  642. raise RuntimeError(
  643. 'The value of \'on_existing\' can only be one of, '
  644. '\'nothing\', \'delete\' and \'skip\''
  645. )
  646. if not isinstance(delete, bool):
  647. raise RuntimeError(
  648. 'The value of \'delete\' can only be \'True\' or \'False\''
  649. )
  650. def decorator(func):
  651. @functools.wraps(func)
  652. def wrap(cls):
  653. # Let's add the user to the system.
  654. log.debug('Creating system group {0!r}'.format(group))
  655. create_group = cls.run_function('group.add', [group])
  656. if not create_group:
  657. log.debug('Failed to create system group')
  658. # The group was not created
  659. if on_existing == 'skip':
  660. cls.skipTest(
  661. 'Failed to create system group {0!r}'.format(group)
  662. )
  663. if on_existing == 'delete':
  664. log.debug(
  665. 'Deleting the system group {0!r}'.format(group)
  666. )
  667. delete_group = cls.run_function('group.delete', [group])
  668. if not delete_group:
  669. cls.skipTest(
  670. 'A group named {0!r} already existed on the '
  671. 'system and re-creating it was not possible'
  672. .format(group)
  673. )
  674. log.debug(
  675. 'Second time creating system group {0!r}'.format(
  676. group
  677. )
  678. )
  679. create_group = cls.run_function('group.add', [group])
  680. if not create_group:
  681. cls.skipTest(
  682. 'A group named {0!r} already existed, was deleted '
  683. 'as requested, but re-creating it was not possible'
  684. .format(group)
  685. )
  686. failure = None
  687. try:
  688. try:
  689. return func(cls, group)
  690. except Exception as exc: # pylint: disable=W0703
  691. log.error(
  692. 'Running {0!r} raised an exception: {1}'.format(
  693. func, exc
  694. ),
  695. exc_info=True
  696. )
  697. # Store the original exception details which will be raised
  698. # a little further down the code
  699. failure = sys.exc_info()
  700. finally:
  701. if delete:
  702. delete_group = cls.run_function('group.delete', [group])
  703. if not delete_group:
  704. if failure is None:
  705. log.warning(
  706. 'Although the actual test-case did not fail, '
  707. 'deleting the created system group {0!r} '
  708. 'afterwards did.'.format(group)
  709. )
  710. else:
  711. log.warning(
  712. 'The test-case failed and also did the removal'
  713. ' of the system group {0!r}'.format(group)
  714. )
  715. if failure is not None:
  716. # If an exception was thrown, raise it
  717. six.reraise(failure[0], failure[1], failure[2])
  718. return wrap
  719. return decorator
  720. def with_system_user_and_group(username, group,
  721. on_existing='delete', delete=True):
  722. '''
  723. Create and optionally destroy a system user and group to be used within a
  724. test case. The system user is crated using the ``user`` salt module, and
  725. the system group is created with the ``group`` salt module.
  726. The decorated testcase function must accept both the 'username' and 'group'
  727. arguments.
  728. :param username: The desired username for the system user.
  729. :param group: The desired name for the system group.
  730. :param on_existing: What to do when the desired username is taken. The
  731. available options are:
  732. * nothing: Do nothing, act as if the user was created.
  733. * delete: delete and re-create the existing user
  734. * skip: skip the test case
  735. '''
  736. if on_existing not in ('nothing', 'delete', 'skip'):
  737. raise RuntimeError(
  738. 'The value of \'on_existing\' can only be one of, '
  739. '\'nothing\', \'delete\' and \'skip\''
  740. )
  741. if not isinstance(delete, bool):
  742. raise RuntimeError(
  743. 'The value of \'delete\' can only be \'True\' or \'False\''
  744. )
  745. def decorator(func):
  746. @functools.wraps(func)
  747. def wrap(cls):
  748. # Let's add the user to the system.
  749. log.debug('Creating system user {0!r}'.format(username))
  750. create_user = cls.run_function('user.add', [username])
  751. log.debug('Creating system group {0!r}'.format(group))
  752. create_group = cls.run_function('group.add', [group])
  753. if not create_user:
  754. log.debug('Failed to create system user')
  755. # The user was not created
  756. if on_existing == 'skip':
  757. cls.skipTest(
  758. 'Failed to create system user {0!r}'.format(
  759. username
  760. )
  761. )
  762. if on_existing == 'delete':
  763. log.debug(
  764. 'Deleting the system user {0!r}'.format(
  765. username
  766. )
  767. )
  768. delete_user = cls.run_function(
  769. 'user.delete', [username, True, True]
  770. )
  771. if not delete_user:
  772. cls.skipTest(
  773. 'A user named {0!r} already existed on the '
  774. 'system and re-creating it was not possible'
  775. .format(username)
  776. )
  777. log.debug(
  778. 'Second time creating system user {0!r}'.format(
  779. username
  780. )
  781. )
  782. create_user = cls.run_function('user.add', [username])
  783. if not create_user:
  784. cls.skipTest(
  785. 'A user named {0!r} already existed, was deleted '
  786. 'as requested, but re-creating it was not possible'
  787. .format(username)
  788. )
  789. if not create_group:
  790. log.debug('Failed to create system group')
  791. # The group was not created
  792. if on_existing == 'skip':
  793. cls.skipTest(
  794. 'Failed to create system group {0!r}'.format(group)
  795. )
  796. if on_existing == 'delete':
  797. log.debug(
  798. 'Deleting the system group {0!r}'.format(group)
  799. )
  800. delete_group = cls.run_function('group.delete', [group])
  801. if not delete_group:
  802. cls.skipTest(
  803. 'A group named {0!r} already existed on the '
  804. 'system and re-creating it was not possible'
  805. .format(group)
  806. )
  807. log.debug(
  808. 'Second time creating system group {0!r}'.format(
  809. group
  810. )
  811. )
  812. create_group = cls.run_function('group.add', [group])
  813. if not create_group:
  814. cls.skipTest(
  815. 'A group named {0!r} already existed, was deleted '
  816. 'as requested, but re-creating it was not possible'
  817. .format(group)
  818. )
  819. failure = None
  820. try:
  821. try:
  822. return func(cls, username, group)
  823. except Exception as exc: # pylint: disable=W0703
  824. log.error(
  825. 'Running {0!r} raised an exception: {1}'.format(
  826. func, exc
  827. ),
  828. exc_info=True
  829. )
  830. # Store the original exception details which will be raised
  831. # a little further down the code
  832. failure = sys.exc_info()
  833. finally:
  834. if delete:
  835. delete_user = cls.run_function(
  836. 'user.delete', [username, True, True]
  837. )
  838. delete_group = cls.run_function('group.delete', [group])
  839. if not delete_user:
  840. if failure is None:
  841. log.warning(
  842. 'Although the actual test-case did not fail, '
  843. 'deleting the created system user {0!r} '
  844. 'afterwards did.'.format(username)
  845. )
  846. else:
  847. log.warning(
  848. 'The test-case failed and also did the removal'
  849. ' of the system user {0!r}'.format(username)
  850. )
  851. if not delete_group:
  852. if failure is None:
  853. log.warning(
  854. 'Although the actual test-case did not fail, '
  855. 'deleting the created system group {0!r} '
  856. 'afterwards did.'.format(group)
  857. )
  858. else:
  859. log.warning(
  860. 'The test-case failed and also did the removal'
  861. ' of the system group {0!r}'.format(group)
  862. )
  863. if failure is not None:
  864. # If an exception was thrown, raise it
  865. six.reraise(failure[0], failure[1], failure[2])
  866. return wrap
  867. return decorator
  868. class WithTempfile(object):
  869. def __init__(self, **kwargs):
  870. self.create = kwargs.pop('create', True)
  871. if 'dir' not in kwargs:
  872. kwargs['dir'] = TMP
  873. if 'prefix' not in kwargs:
  874. kwargs['prefix'] = '__salt.test.'
  875. self.kwargs = kwargs
  876. def __call__(self, func):
  877. self.func = func
  878. return functools.wraps(func)(
  879. lambda testcase, *args, **kwargs: self.wrap(testcase, *args, **kwargs) # pylint: disable=W0108
  880. )
  881. def wrap(self, testcase, *args, **kwargs):
  882. name = salt.utils.files.mkstemp(**self.kwargs)
  883. if not self.create:
  884. os.remove(name)
  885. try:
  886. return self.func(testcase, name, *args, **kwargs)
  887. finally:
  888. try:
  889. os.remove(name)
  890. except OSError:
  891. pass
  892. with_tempfile = WithTempfile
  893. class WithTempdir(object):
  894. def __init__(self, **kwargs):
  895. self.create = kwargs.pop('create', True)
  896. if 'dir' not in kwargs:
  897. kwargs['dir'] = TMP
  898. self.kwargs = kwargs
  899. def __call__(self, func):
  900. self.func = func
  901. return functools.wraps(func)(
  902. lambda testcase, *args, **kwargs: self.wrap(testcase, *args, **kwargs) # pylint: disable=W0108
  903. )
  904. def wrap(self, testcase, *args, **kwargs):
  905. tempdir = tempfile.mkdtemp(**self.kwargs)
  906. if not self.create:
  907. os.rmdir(tempdir)
  908. try:
  909. return self.func(testcase, tempdir, *args, **kwargs)
  910. finally:
  911. shutil.rmtree(tempdir, ignore_errors=True)
  912. with_tempdir = WithTempdir
  913. def requires_system_grains(func):
  914. '''
  915. Function decorator which loads and passes the system's grains to the test
  916. case.
  917. '''
  918. @functools.wraps(func)
  919. def decorator(*args, **kwargs):
  920. if not hasattr(requires_system_grains, '__grains__'):
  921. import salt.config
  922. root_dir = tempfile.mkdtemp(dir=TMP)
  923. defaults = salt.config.DEFAULT_MINION_OPTS.copy()
  924. defaults.pop('conf_file')
  925. defaults.update({
  926. 'root_dir': root_dir,
  927. 'cachedir': 'cachedir',
  928. 'sock_dir': 'sock',
  929. 'pki_dir': 'pki',
  930. 'log_file': 'logs/minion',
  931. 'pidfile': 'pids/minion.pid'
  932. })
  933. opts = salt.config.minion_config(None, defaults=defaults)
  934. requires_system_grains.__grains__ = salt.loader.grains(opts)
  935. shutil.rmtree(root_dir, ignore_errors=True)
  936. kwargs['grains'] = requires_system_grains.__grains__
  937. return func(*args, **kwargs)
  938. return decorator
  939. def requires_salt_modules(*names):
  940. '''
  941. Makes sure the passed salt module is available. Skips the test if not
  942. .. versionadded:: 0.5.2
  943. '''
  944. def decorator(caller):
  945. if inspect.isclass(caller):
  946. # We're decorating a class
  947. old_setup = getattr(caller, 'setUp', None)
  948. def setUp(self, *args, **kwargs):
  949. if old_setup is not None:
  950. old_setup(self, *args, **kwargs)
  951. if not hasattr(self, 'run_function'):
  952. raise RuntimeError(
  953. '{0} does not have the \'run_function\' method which '
  954. 'is necessary to collect the loaded modules'.format(
  955. self.__class__.__name__
  956. )
  957. )
  958. if not hasattr(requires_salt_modules, '__available_modules__'):
  959. requires_salt_modules.__available_modules__ = set()
  960. _names = []
  961. for name in names:
  962. if name not in requires_salt_modules.__available_modules__:
  963. _names.append(name)
  964. if _names:
  965. not_found_modules = self.run_function('runtests_helpers.modules_available', _names)
  966. for name in _names:
  967. if name not in not_found_modules:
  968. requires_salt_modules.__available_modules__.add(name)
  969. if not_found_modules:
  970. if len(not_found_modules) == 1:
  971. self.skipTest('Salt module {0!r} is not available'.format(not_found_modules[0]))
  972. self.skipTest('Salt modules not available: {0!r}'.format(not_found_modules))
  973. caller.setUp = setUp
  974. return caller
  975. # We're simply decorating functions
  976. @functools.wraps(caller)
  977. def wrapper(cls):
  978. if not hasattr(cls, 'run_function'):
  979. raise RuntimeError(
  980. '{0} does not have the \'run_function\' method which is '
  981. 'necessary to collect the loaded modules'.format(
  982. cls.__class__.__name__
  983. )
  984. )
  985. if not hasattr(requires_salt_modules, '__available_modules__'):
  986. requires_salt_modules.__available_modules__ = set()
  987. _names = []
  988. for name in names:
  989. if name not in requires_salt_modules.__available_modules__:
  990. _names.append(name)
  991. if _names:
  992. not_found_modules = cls.run_function('runtests_helpers.modules_available', _names)
  993. for name in _names:
  994. if name not in not_found_modules:
  995. requires_salt_modules.__available_modules__.add(name)
  996. if not_found_modules:
  997. if len(not_found_modules) == 1:
  998. cls.skipTest('Salt module {0!r} is not available'.format(not_found_modules[0]))
  999. cls.skipTest('Salt modules not available: {0!r}'.format(not_found_modules))
  1000. return caller(cls)
  1001. return wrapper
  1002. return decorator
  1003. def skip_if_binaries_missing(*binaries, **kwargs):
  1004. import salt.utils.path
  1005. if len(binaries) == 1:
  1006. if isinstance(binaries[0], (list, tuple, set, frozenset)):
  1007. binaries = binaries[0]
  1008. check_all = kwargs.pop('check_all', False)
  1009. message = kwargs.pop('message', None)
  1010. if kwargs:
  1011. raise RuntimeError(
  1012. 'The only supported keyword argument is \'check_all\' and '
  1013. '\'message\'. Invalid keyword arguments: {0}'.format(
  1014. ', '.join(kwargs.keys())
  1015. )
  1016. )
  1017. if check_all:
  1018. for binary in binaries:
  1019. if salt.utils.path.which(binary) is None:
  1020. return skip(
  1021. '{0}The {1!r} binary was not found'.format(
  1022. message and '{0}. '.format(message) or '',
  1023. binary
  1024. )
  1025. )
  1026. elif salt.utils.path.which_bin(binaries) is None:
  1027. return skip(
  1028. '{0}None of the following binaries was found: {1}'.format(
  1029. message and '{0}. '.format(message) or '',
  1030. ', '.join(binaries)
  1031. )
  1032. )
  1033. return _id
  1034. def skip_if_not_root(func):
  1035. if not sys.platform.startswith('win'):
  1036. if os.getuid() != 0:
  1037. func.__unittest_skip__ = True
  1038. func.__unittest_skip_why__ = 'You must be logged in as root to run this test'
  1039. else:
  1040. current_user = salt.utils.win_functions.get_current_user()
  1041. if current_user != 'SYSTEM':
  1042. if not salt.utils.win_functions.is_admin(current_user):
  1043. func.__unittest_skip__ = True
  1044. func.__unittest_skip_why__ = 'You must be logged in as an Administrator to run this test'
  1045. return func
  1046. if sys.platform.startswith('win'):
  1047. SIGTERM = signal.CTRL_BREAK_EVENT # pylint: disable=no-member
  1048. else:
  1049. SIGTERM = signal.SIGTERM
  1050. def collect_child_processes(pid):
  1051. '''
  1052. Try to collect any started child processes of the provided pid
  1053. '''
  1054. # Let's get the child processes of the started subprocess
  1055. try:
  1056. parent = psutil.Process(pid)
  1057. if hasattr(parent, 'children'):
  1058. children = parent.children(recursive=True)
  1059. else:
  1060. children = []
  1061. except psutil.NoSuchProcess:
  1062. children = []
  1063. return children[::-1] # return a reversed list of the children
  1064. def _terminate_process_list(process_list, kill=False, slow_stop=False):
  1065. for process in process_list[:][::-1]: # Iterate over a reversed copy of the list
  1066. if not psutil.pid_exists(process.pid):
  1067. process_list.remove(process)
  1068. continue
  1069. try:
  1070. if not kill and process.status() == psutil.STATUS_ZOMBIE:
  1071. # Zombie processes will exit once child processes also exit
  1072. continue
  1073. try:
  1074. cmdline = process.cmdline()
  1075. except psutil.AccessDenied:
  1076. # OSX is more restrictive about the above information
  1077. cmdline = None
  1078. except OSError:
  1079. cmdline = None
  1080. if not cmdline:
  1081. try:
  1082. cmdline = process.as_dict()
  1083. except Exception:
  1084. cmdline = 'UNKNOWN PROCESS'
  1085. if kill:
  1086. log.info('Killing process(%s): %s', process.pid, cmdline)
  1087. process.kill()
  1088. else:
  1089. log.info('Terminating process(%s): %s', process.pid, cmdline)
  1090. try:
  1091. if slow_stop:
  1092. # Allow coverage data to be written down to disk
  1093. process.send_signal(SIGTERM)
  1094. try:
  1095. process.wait(2)
  1096. except psutil.TimeoutExpired:
  1097. if psutil.pid_exists(process.pid):
  1098. continue
  1099. else:
  1100. process.terminate()
  1101. except OSError as exc:
  1102. if exc.errno not in (errno.ESRCH, errno.EACCES):
  1103. raise
  1104. if not psutil.pid_exists(process.pid):
  1105. process_list.remove(process)
  1106. except psutil.NoSuchProcess:
  1107. process_list.remove(process)
  1108. def terminate_process_list(process_list, kill=False, slow_stop=False):
  1109. def on_process_terminated(proc):
  1110. log.info('Process %s terminated with exit code: %s', getattr(proc, '_cmdline', proc), proc.returncode)
  1111. # Try to terminate processes with the provided kill and slow_stop parameters
  1112. log.info('Terminating process list. 1st step. kill: %s, slow stop: %s', kill, slow_stop)
  1113. # Cache the cmdline since that will be inaccessible once the process is terminated
  1114. for proc in process_list:
  1115. try:
  1116. cmdline = proc.cmdline()
  1117. except (psutil.NoSuchProcess, psutil.AccessDenied):
  1118. # OSX is more restrictive about the above information
  1119. cmdline = None
  1120. if not cmdline:
  1121. try:
  1122. cmdline = proc
  1123. except (psutil.NoSuchProcess, psutil.AccessDenied):
  1124. cmdline = '<could not be retrived; dead process: {0}>'.format(proc)
  1125. proc._cmdline = cmdline
  1126. _terminate_process_list(process_list, kill=kill, slow_stop=slow_stop)
  1127. psutil.wait_procs(process_list, timeout=15, callback=on_process_terminated)
  1128. if process_list:
  1129. # If there's still processes to be terminated, retry and kill them if slow_stop is False
  1130. log.info('Terminating process list. 2nd step. kill: %s, slow stop: %s', slow_stop is False, slow_stop)
  1131. _terminate_process_list(process_list, kill=slow_stop is False, slow_stop=slow_stop)
  1132. psutil.wait_procs(process_list, timeout=10, callback=on_process_terminated)
  1133. if process_list:
  1134. # If there's still processes to be terminated, just kill them, no slow stopping now
  1135. log.info('Terminating process list. 3rd step. kill: True, slow stop: False')
  1136. _terminate_process_list(process_list, kill=True, slow_stop=False)
  1137. psutil.wait_procs(process_list, timeout=5, callback=on_process_terminated)
  1138. if process_list:
  1139. # In there's still processes to be terminated, log a warning about it
  1140. log.warning('Some processes failed to properly terminate: %s', process_list)
  1141. def terminate_process(pid=None, process=None, children=None, kill_children=False, slow_stop=False):
  1142. '''
  1143. Try to terminate/kill the started processe
  1144. '''
  1145. children = children or []
  1146. process_list = []
  1147. def on_process_terminated(proc):
  1148. if proc.returncode:
  1149. log.info('Process %s terminated with exit code: %s', getattr(proc, '_cmdline', proc), proc.returncode)
  1150. else:
  1151. log.info('Process %s terminated', getattr(proc, '_cmdline', proc))
  1152. if pid and not process:
  1153. try:
  1154. process = psutil.Process(pid)
  1155. process_list.append(process)
  1156. except psutil.NoSuchProcess:
  1157. # Process is already gone
  1158. process = None
  1159. if kill_children:
  1160. if process:
  1161. if not children:
  1162. children = collect_child_processes(process.pid)
  1163. else:
  1164. # Let's collect children again since there might be new ones
  1165. children.extend(collect_child_processes(pid))
  1166. if children:
  1167. process_list.extend(children)
  1168. if process_list:
  1169. if process:
  1170. log.info('Stopping process %s and respective children: %s', process, children)
  1171. else:
  1172. log.info('Terminating process list: %s', process_list)
  1173. terminate_process_list(process_list, kill=slow_stop is False, slow_stop=slow_stop)
  1174. if process and psutil.pid_exists(process.pid):
  1175. log.warning('Process left behind which we were unable to kill: %s', process)
  1176. def terminate_process_pid(pid, only_children=False):
  1177. children = []
  1178. process = None
  1179. # Let's begin the shutdown routines
  1180. try:
  1181. process = psutil.Process(pid)
  1182. children = collect_child_processes(pid)
  1183. except psutil.NoSuchProcess:
  1184. log.info('No process with the PID %s was found running', pid)
  1185. if only_children:
  1186. return terminate_process(children=children, kill_children=True, slow_stop=True)
  1187. return terminate_process(pid=pid, process=process, children=children, kill_children=True, slow_stop=True)
  1188. def repeat(caller=None, condition=True, times=5):
  1189. '''
  1190. Repeat a test X amount of times until the first failure.
  1191. .. code-block:: python
  1192. class MyTestCase(TestCase):
  1193. @repeat
  1194. def test_sometimes_works(self):
  1195. pass
  1196. '''
  1197. if caller is None:
  1198. return functools.partial(repeat, condition=condition, times=times)
  1199. if isinstance(condition, bool) and condition is False:
  1200. # Don't even decorate
  1201. return caller
  1202. elif callable(condition):
  1203. if condition() is False:
  1204. # Don't even decorate
  1205. return caller
  1206. if inspect.isclass(caller):
  1207. attrs = [n for n in dir(caller) if n.startswith('test_')]
  1208. for attrname in attrs:
  1209. try:
  1210. function = getattr(caller, attrname)
  1211. if not inspect.isfunction(function) and not inspect.ismethod(function):
  1212. continue
  1213. setattr(caller, attrname, repeat(caller=function, condition=condition, times=times))
  1214. except Exception as exc:
  1215. log.exception(exc)
  1216. continue
  1217. return caller
  1218. @functools.wraps(caller)
  1219. def wrap(cls):
  1220. result = None
  1221. for attempt in range(1, times+1):
  1222. log.info('%s test run %d of %s times', cls, attempt, times)
  1223. caller(cls)
  1224. return cls
  1225. return wrap
  1226. def http_basic_auth(login_cb=lambda username, password: False):
  1227. '''
  1228. A crude decorator to force a handler to request HTTP Basic Authentication
  1229. Example usage:
  1230. .. code-block:: python
  1231. @http_basic_auth(lambda u, p: u == 'foo' and p == 'bar')
  1232. class AuthenticatedHandler(tornado.web.RequestHandler):
  1233. pass
  1234. '''
  1235. def wrapper(handler_class):
  1236. def wrap_execute(handler_execute):
  1237. def check_auth(handler, kwargs):
  1238. auth = handler.request.headers.get('Authorization')
  1239. if auth is None or not auth.startswith('Basic '):
  1240. # No username/password entered yet, we need to return a 401
  1241. # and set the WWW-Authenticate header to request login.
  1242. handler.set_status(401)
  1243. handler.set_header(
  1244. 'WWW-Authenticate', 'Basic realm=Restricted')
  1245. else:
  1246. # Strip the 'Basic ' from the beginning of the auth header
  1247. # leaving the base64-encoded secret
  1248. username, password = \
  1249. base64.b64decode(auth[6:]).split(':', 1)
  1250. if login_cb(username, password):
  1251. # Authentication successful
  1252. return
  1253. else:
  1254. # Authentication failed
  1255. handler.set_status(403)
  1256. handler._transforms = []
  1257. handler.finish()
  1258. def _execute(self, transforms, *args, **kwargs):
  1259. check_auth(self, kwargs)
  1260. return handler_execute(self, transforms, *args, **kwargs)
  1261. return _execute
  1262. handler_class._execute = wrap_execute(handler_class._execute)
  1263. return handler_class
  1264. return wrapper
  1265. def generate_random_name(prefix, size=6):
  1266. '''
  1267. Generates a random name by combining the provided prefix with a randomly generated
  1268. ascii string.
  1269. .. versionadded:: 2018.3.0
  1270. prefix
  1271. The string to prefix onto the randomly generated ascii string.
  1272. size
  1273. The number of characters to generate. Default: 6.
  1274. '''
  1275. return prefix + ''.join(
  1276. random.choice(string.ascii_uppercase + string.digits)
  1277. for x in range(size)
  1278. )
  1279. class Webserver(object):
  1280. '''
  1281. Starts a tornado webserver on 127.0.0.1 on a random available port
  1282. USAGE:
  1283. .. code-block:: python
  1284. from tests.support.helpers import Webserver
  1285. webserver = Webserver('/path/to/web/root')
  1286. webserver.start()
  1287. webserver.stop()
  1288. '''
  1289. def __init__(self,
  1290. root=None,
  1291. port=None,
  1292. wait=5,
  1293. handler=None):
  1294. '''
  1295. root
  1296. Root directory of webserver. If not passed, it will default to the
  1297. location of the base environment of the integration suite's file
  1298. roots (tests/integration/files/file/base/)
  1299. port
  1300. Port on which to listen. If not passed, a random one will be chosen
  1301. at the time the start() function is invoked.
  1302. wait : 5
  1303. Number of seconds to wait for the socket to be open before raising
  1304. an exception
  1305. handler
  1306. Can be used to use a subclass of tornado.web.StaticFileHandler,
  1307. such as when enforcing authentication with the http_basic_auth
  1308. decorator.
  1309. '''
  1310. if port is not None and not isinstance(port, six.integer_types):
  1311. raise ValueError('port must be an integer')
  1312. if root is None:
  1313. root = os.path.join(FILES, 'file', 'base')
  1314. try:
  1315. self.root = os.path.realpath(root)
  1316. except AttributeError:
  1317. raise ValueError('root must be a string')
  1318. self.port = port
  1319. self.wait = wait
  1320. self.handler = handler \
  1321. if handler is not None \
  1322. else tornado.web.StaticFileHandler
  1323. self.web_root = None
  1324. def target(self):
  1325. '''
  1326. Threading target which stands up the tornado application
  1327. '''
  1328. self.ioloop = tornado.ioloop.IOLoop()
  1329. self.ioloop.make_current()
  1330. self.application = tornado.web.Application(
  1331. [(r'/(.*)', self.handler, {'path': self.root})])
  1332. self.application.listen(self.port)
  1333. self.ioloop.start()
  1334. @property
  1335. def listening(self):
  1336. if self.port is None:
  1337. return False
  1338. sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  1339. return sock.connect_ex(('127.0.0.1', self.port)) == 0
  1340. def url(self, path):
  1341. '''
  1342. Convenience function which, given a file path, will return a URL that
  1343. points to that path. If the path is relative, it will just be appended
  1344. to self.web_root.
  1345. '''
  1346. if self.web_root is None:
  1347. raise RuntimeError('Webserver instance has not been started')
  1348. err_msg = 'invalid path, must be either a relative path or a path ' \
  1349. 'within {0}'.format(self.root)
  1350. try:
  1351. relpath = path \
  1352. if not os.path.isabs(path) \
  1353. else os.path.relpath(path, self.root)
  1354. if relpath.startswith('..' + os.sep):
  1355. raise ValueError(err_msg)
  1356. return '/'.join((self.web_root, relpath))
  1357. except AttributeError:
  1358. raise ValueError(err_msg)
  1359. def start(self):
  1360. '''
  1361. Starts the webserver
  1362. '''
  1363. if self.port is None:
  1364. self.port = get_unused_localhost_port()
  1365. self.web_root = 'http://127.0.0.1:{0}'.format(self.port)
  1366. self.server_thread = threading.Thread(target=self.target)
  1367. self.server_thread.daemon = True
  1368. self.server_thread.start()
  1369. for idx in range(self.wait + 1):
  1370. if self.listening:
  1371. break
  1372. if idx != self.wait:
  1373. time.sleep(1)
  1374. else:
  1375. raise Exception(
  1376. 'Failed to start tornado webserver on 127.0.0.1:{0} within '
  1377. '{1} seconds'.format(self.port, self.wait)
  1378. )
  1379. def stop(self):
  1380. '''
  1381. Stops the webserver
  1382. '''
  1383. self.ioloop.add_callback(self.ioloop.stop)
  1384. self.server_thread.join()
  1385. def win32_kill_process_tree(pid, sig=signal.SIGTERM, include_parent=True,
  1386. timeout=None, on_terminate=None):
  1387. '''
  1388. Kill a process tree (including grandchildren) with signal "sig" and return
  1389. a (gone, still_alive) tuple. "on_terminate", if specified, is a callabck
  1390. function which is called as soon as a child terminates.
  1391. '''
  1392. if pid == os.getpid():
  1393. raise RuntimeError("I refuse to kill myself")
  1394. try:
  1395. parent = psutil.Process(pid)
  1396. except psutil.NoSuchProcess:
  1397. log.debug("PID not found alive: %d", pid)
  1398. return ([], [])
  1399. children = parent.children(recursive=True)
  1400. if include_parent:
  1401. children.append(parent)
  1402. for p in children:
  1403. p.send_signal(sig)
  1404. gone, alive = psutil.wait_procs(children, timeout=timeout,
  1405. callback=on_terminate)
  1406. return (gone, alive)
  1407. def this_user():
  1408. '''
  1409. Get the user associated with the current process.
  1410. '''
  1411. if salt.utils.platform.is_windows():
  1412. return salt.utils.win_functions.get_current_user(with_domain=False)
  1413. return pwd.getpwuid(os.getuid())[0]
  1414. def dedent(text, linesep=os.linesep):
  1415. '''
  1416. A wrapper around textwrap.dedent that also sets line endings.
  1417. '''
  1418. linesep = salt.utils.stringutils.to_unicode(linesep)
  1419. unicode_text = textwrap.dedent(salt.utils.stringutils.to_unicode(text))
  1420. clean_text = linesep.join(unicode_text.splitlines())
  1421. if unicode_text.endswith(u'\n'):
  1422. clean_text += linesep
  1423. if not isinstance(text, six.text_type):
  1424. return salt.utils.stringutils.to_bytes(clean_text)
  1425. return clean_text