You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

dictconfig.py 23 KiB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565
  1. # This is a copy of the Python logging.config.dictconfig module,
  2. # reproduced with permission. It is provided here for backwards
  3. # compatibility for Python versions prior to 2.7.
  4. #
  5. # Copyright 2009-2010 by Vinay Sajip. All Rights Reserved.
  6. #
  7. # Permission to use, copy, modify, and distribute this software and its
  8. # documentation for any purpose and without fee is hereby granted,
  9. # provided that the above copyright notice appear in all copies and that
  10. # both that copyright notice and this permission notice appear in
  11. # supporting documentation, and that the name of Vinay Sajip
  12. # not be used in advertising or publicity pertaining to distribution
  13. # of the software without specific, written prior permission.
  14. # VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
  15. # ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
  16. # VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR
  17. # ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER
  18. # IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
  19. # OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  20. from __future__ import absolute_import
  21. import logging.handlers
  22. import re
  23. import sys
  24. import types
  25. from pip._vendor import six
  26. # flake8: noqa
  27. IDENTIFIER = re.compile('^[a-z_][a-z0-9_]*$', re.I)
  28. def valid_ident(s):
  29. m = IDENTIFIER.match(s)
  30. if not m:
  31. raise ValueError('Not a valid Python identifier: %r' % s)
  32. return True
  33. #
  34. # This function is defined in logging only in recent versions of Python
  35. #
  36. try:
  37. from logging import _checkLevel
  38. except ImportError:
  39. def _checkLevel(level):
  40. if isinstance(level, int):
  41. rv = level
  42. elif str(level) == level:
  43. if level not in logging._levelNames:
  44. raise ValueError('Unknown level: %r' % level)
  45. rv = logging._levelNames[level]
  46. else:
  47. raise TypeError('Level not an integer or a '
  48. 'valid string: %r' % level)
  49. return rv
  50. # The ConvertingXXX classes are wrappers around standard Python containers,
  51. # and they serve to convert any suitable values in the container. The
  52. # conversion converts base dicts, lists and tuples to their wrapped
  53. # equivalents, whereas strings which match a conversion format are converted
  54. # appropriately.
  55. #
  56. # Each wrapper should have a configurator attribute holding the actual
  57. # configurator to use for conversion.
  58. class ConvertingDict(dict):
  59. """A converting dictionary wrapper."""
  60. def __getitem__(self, key):
  61. value = dict.__getitem__(self, key)
  62. result = self.configurator.convert(value)
  63. # If the converted value is different, save for next time
  64. if value is not result:
  65. self[key] = result
  66. if type(result) in (ConvertingDict, ConvertingList,
  67. ConvertingTuple):
  68. result.parent = self
  69. result.key = key
  70. return result
  71. def get(self, key, default=None):
  72. value = dict.get(self, key, default)
  73. result = self.configurator.convert(value)
  74. # If the converted value is different, save for next time
  75. if value is not result:
  76. self[key] = result
  77. if type(result) in (ConvertingDict, ConvertingList,
  78. ConvertingTuple):
  79. result.parent = self
  80. result.key = key
  81. return result
  82. def pop(self, key, default=None):
  83. value = dict.pop(self, key, default)
  84. result = self.configurator.convert(value)
  85. if value is not result:
  86. if type(result) in (ConvertingDict, ConvertingList,
  87. ConvertingTuple):
  88. result.parent = self
  89. result.key = key
  90. return result
  91. class ConvertingList(list):
  92. """A converting list wrapper."""
  93. def __getitem__(self, key):
  94. value = list.__getitem__(self, key)
  95. result = self.configurator.convert(value)
  96. # If the converted value is different, save for next time
  97. if value is not result:
  98. self[key] = result
  99. if type(result) in (ConvertingDict, ConvertingList,
  100. ConvertingTuple):
  101. result.parent = self
  102. result.key = key
  103. return result
  104. def pop(self, idx=-1):
  105. value = list.pop(self, idx)
  106. result = self.configurator.convert(value)
  107. if value is not result:
  108. if type(result) in (ConvertingDict, ConvertingList,
  109. ConvertingTuple):
  110. result.parent = self
  111. return result
  112. class ConvertingTuple(tuple):
  113. """A converting tuple wrapper."""
  114. def __getitem__(self, key):
  115. value = tuple.__getitem__(self, key)
  116. result = self.configurator.convert(value)
  117. if value is not result:
  118. if type(result) in (ConvertingDict, ConvertingList,
  119. ConvertingTuple):
  120. result.parent = self
  121. result.key = key
  122. return result
  123. class BaseConfigurator(object):
  124. """
  125. The configurator base class which defines some useful defaults.
  126. """
  127. CONVERT_PATTERN = re.compile(r'^(?P<prefix>[a-z]+)://(?P<suffix>.*)$')
  128. WORD_PATTERN = re.compile(r'^\s*(\w+)\s*')
  129. DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*')
  130. INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*')
  131. DIGIT_PATTERN = re.compile(r'^\d+$')
  132. value_converters = {
  133. 'ext' : 'ext_convert',
  134. 'cfg' : 'cfg_convert',
  135. }
  136. # We might want to use a different one, e.g. importlib
  137. importer = __import__
  138. def __init__(self, config):
  139. self.config = ConvertingDict(config)
  140. self.config.configurator = self
  141. def resolve(self, s):
  142. """
  143. Resolve strings to objects using standard import and attribute
  144. syntax.
  145. """
  146. name = s.split('.')
  147. used = name.pop(0)
  148. try:
  149. found = self.importer(used)
  150. for frag in name:
  151. used += '.' + frag
  152. try:
  153. found = getattr(found, frag)
  154. except AttributeError:
  155. self.importer(used)
  156. found = getattr(found, frag)
  157. return found
  158. except ImportError:
  159. e, tb = sys.exc_info()[1:]
  160. v = ValueError('Cannot resolve %r: %s' % (s, e))
  161. v.__cause__, v.__traceback__ = e, tb
  162. raise v
  163. def ext_convert(self, value):
  164. """Default converter for the ext:// protocol."""
  165. return self.resolve(value)
  166. def cfg_convert(self, value):
  167. """Default converter for the cfg:// protocol."""
  168. rest = value
  169. m = self.WORD_PATTERN.match(rest)
  170. if m is None:
  171. raise ValueError("Unable to convert %r" % value)
  172. else:
  173. rest = rest[m.end():]
  174. d = self.config[m.groups()[0]]
  175. # print d, rest
  176. while rest:
  177. m = self.DOT_PATTERN.match(rest)
  178. if m:
  179. d = d[m.groups()[0]]
  180. else:
  181. m = self.INDEX_PATTERN.match(rest)
  182. if m:
  183. idx = m.groups()[0]
  184. if not self.DIGIT_PATTERN.match(idx):
  185. d = d[idx]
  186. else:
  187. try:
  188. n = int(idx) # try as number first (most likely)
  189. d = d[n]
  190. except TypeError:
  191. d = d[idx]
  192. if m:
  193. rest = rest[m.end():]
  194. else:
  195. raise ValueError('Unable to convert '
  196. '%r at %r' % (value, rest))
  197. # rest should be empty
  198. return d
  199. def convert(self, value):
  200. """
  201. Convert values to an appropriate type. dicts, lists and tuples are
  202. replaced by their converting alternatives. Strings are checked to
  203. see if they have a conversion format and are converted if they do.
  204. """
  205. if not isinstance(value, ConvertingDict) and isinstance(value, dict):
  206. value = ConvertingDict(value)
  207. value.configurator = self
  208. elif not isinstance(value, ConvertingList) and isinstance(value, list):
  209. value = ConvertingList(value)
  210. value.configurator = self
  211. elif not isinstance(value, ConvertingTuple) and\
  212. isinstance(value, tuple):
  213. value = ConvertingTuple(value)
  214. value.configurator = self
  215. elif isinstance(value, six.string_types): # str for py3k
  216. m = self.CONVERT_PATTERN.match(value)
  217. if m:
  218. d = m.groupdict()
  219. prefix = d['prefix']
  220. converter = self.value_converters.get(prefix, None)
  221. if converter:
  222. suffix = d['suffix']
  223. converter = getattr(self, converter)
  224. value = converter(suffix)
  225. return value
  226. def configure_custom(self, config):
  227. """Configure an object with a user-supplied factory."""
  228. c = config.pop('()')
  229. if not hasattr(c, '__call__') and hasattr(types, 'ClassType') and type(c) != types.ClassType:
  230. c = self.resolve(c)
  231. props = config.pop('.', None)
  232. # Check for valid identifiers
  233. kwargs = dict((k, config[k]) for k in config if valid_ident(k))
  234. result = c(**kwargs)
  235. if props:
  236. for name, value in props.items():
  237. setattr(result, name, value)
  238. return result
  239. def as_tuple(self, value):
  240. """Utility function which converts lists to tuples."""
  241. if isinstance(value, list):
  242. value = tuple(value)
  243. return value
  244. class DictConfigurator(BaseConfigurator):
  245. """
  246. Configure logging using a dictionary-like object to describe the
  247. configuration.
  248. """
  249. def configure(self):
  250. """Do the configuration."""
  251. config = self.config
  252. if 'version' not in config:
  253. raise ValueError("dictionary doesn't specify a version")
  254. if config['version'] != 1:
  255. raise ValueError("Unsupported version: %s" % config['version'])
  256. incremental = config.pop('incremental', False)
  257. EMPTY_DICT = {}
  258. logging._acquireLock()
  259. try:
  260. if incremental:
  261. handlers = config.get('handlers', EMPTY_DICT)
  262. # incremental handler config only if handler name
  263. # ties in to logging._handlers (Python 2.7)
  264. if sys.version_info[:2] == (2, 7):
  265. for name in handlers:
  266. if name not in logging._handlers:
  267. raise ValueError('No handler found with '
  268. 'name %r' % name)
  269. else:
  270. try:
  271. handler = logging._handlers[name]
  272. handler_config = handlers[name]
  273. level = handler_config.get('level', None)
  274. if level:
  275. handler.setLevel(_checkLevel(level))
  276. except StandardError as e:
  277. raise ValueError('Unable to configure handler '
  278. '%r: %s' % (name, e))
  279. loggers = config.get('loggers', EMPTY_DICT)
  280. for name in loggers:
  281. try:
  282. self.configure_logger(name, loggers[name], True)
  283. except StandardError as e:
  284. raise ValueError('Unable to configure logger '
  285. '%r: %s' % (name, e))
  286. root = config.get('root', None)
  287. if root:
  288. try:
  289. self.configure_root(root, True)
  290. except StandardError as e:
  291. raise ValueError('Unable to configure root '
  292. 'logger: %s' % e)
  293. else:
  294. disable_existing = config.pop('disable_existing_loggers', True)
  295. logging._handlers.clear()
  296. del logging._handlerList[:]
  297. # Do formatters first - they don't refer to anything else
  298. formatters = config.get('formatters', EMPTY_DICT)
  299. for name in formatters:
  300. try:
  301. formatters[name] = self.configure_formatter(
  302. formatters[name])
  303. except StandardError as e:
  304. raise ValueError('Unable to configure '
  305. 'formatter %r: %s' % (name, e))
  306. # Next, do filters - they don't refer to anything else, either
  307. filters = config.get('filters', EMPTY_DICT)
  308. for name in filters:
  309. try:
  310. filters[name] = self.configure_filter(filters[name])
  311. except StandardError as e:
  312. raise ValueError('Unable to configure '
  313. 'filter %r: %s' % (name, e))
  314. # Next, do handlers - they refer to formatters and filters
  315. # As handlers can refer to other handlers, sort the keys
  316. # to allow a deterministic order of configuration
  317. handlers = config.get('handlers', EMPTY_DICT)
  318. for name in sorted(handlers):
  319. try:
  320. handler = self.configure_handler(handlers[name])
  321. handler.name = name
  322. handlers[name] = handler
  323. except StandardError as e:
  324. raise ValueError('Unable to configure handler '
  325. '%r: %s' % (name, e))
  326. # Next, do loggers - they refer to handlers and filters
  327. # we don't want to lose the existing loggers,
  328. # since other threads may have pointers to them.
  329. # existing is set to contain all existing loggers,
  330. # and as we go through the new configuration we
  331. # remove any which are configured. At the end,
  332. # what's left in existing is the set of loggers
  333. # which were in the previous configuration but
  334. # which are not in the new configuration.
  335. root = logging.root
  336. existing = list(root.manager.loggerDict)
  337. # The list needs to be sorted so that we can
  338. # avoid disabling child loggers of explicitly
  339. # named loggers. With a sorted list it is easier
  340. # to find the child loggers.
  341. existing.sort()
  342. # We'll keep the list of existing loggers
  343. # which are children of named loggers here...
  344. child_loggers = []
  345. # now set up the new ones...
  346. loggers = config.get('loggers', EMPTY_DICT)
  347. for name in loggers:
  348. if name in existing:
  349. i = existing.index(name)
  350. prefixed = name + "."
  351. pflen = len(prefixed)
  352. num_existing = len(existing)
  353. i = i + 1 # look at the entry after name
  354. while (i < num_existing) and\
  355. (existing[i][:pflen] == prefixed):
  356. child_loggers.append(existing[i])
  357. i = i + 1
  358. existing.remove(name)
  359. try:
  360. self.configure_logger(name, loggers[name])
  361. except StandardError as e:
  362. raise ValueError('Unable to configure logger '
  363. '%r: %s' % (name, e))
  364. # Disable any old loggers. There's no point deleting
  365. # them as other threads may continue to hold references
  366. # and by disabling them, you stop them doing any logging.
  367. # However, don't disable children of named loggers, as that's
  368. # probably not what was intended by the user.
  369. for log in existing:
  370. logger = root.manager.loggerDict[log]
  371. if log in child_loggers:
  372. logger.level = logging.NOTSET
  373. logger.handlers = []
  374. logger.propagate = True
  375. elif disable_existing:
  376. logger.disabled = True
  377. # And finally, do the root logger
  378. root = config.get('root', None)
  379. if root:
  380. try:
  381. self.configure_root(root)
  382. except StandardError as e:
  383. raise ValueError('Unable to configure root '
  384. 'logger: %s' % e)
  385. finally:
  386. logging._releaseLock()
  387. def configure_formatter(self, config):
  388. """Configure a formatter from a dictionary."""
  389. if '()' in config:
  390. factory = config['()'] # for use in exception handler
  391. try:
  392. result = self.configure_custom(config)
  393. except TypeError as te:
  394. if "'format'" not in str(te):
  395. raise
  396. # Name of parameter changed from fmt to format.
  397. # Retry with old name.
  398. # This is so that code can be used with older Python versions
  399. #(e.g. by Django)
  400. config['fmt'] = config.pop('format')
  401. config['()'] = factory
  402. result = self.configure_custom(config)
  403. else:
  404. fmt = config.get('format', None)
  405. dfmt = config.get('datefmt', None)
  406. result = logging.Formatter(fmt, dfmt)
  407. return result
  408. def configure_filter(self, config):
  409. """Configure a filter from a dictionary."""
  410. if '()' in config:
  411. result = self.configure_custom(config)
  412. else:
  413. name = config.get('name', '')
  414. result = logging.Filter(name)
  415. return result
  416. def add_filters(self, filterer, filters):
  417. """Add filters to a filterer from a list of names."""
  418. for f in filters:
  419. try:
  420. filterer.addFilter(self.config['filters'][f])
  421. except StandardError as e:
  422. raise ValueError('Unable to add filter %r: %s' % (f, e))
  423. def configure_handler(self, config):
  424. """Configure a handler from a dictionary."""
  425. formatter = config.pop('formatter', None)
  426. if formatter:
  427. try:
  428. formatter = self.config['formatters'][formatter]
  429. except StandardError as e:
  430. raise ValueError('Unable to set formatter '
  431. '%r: %s' % (formatter, e))
  432. level = config.pop('level', None)
  433. filters = config.pop('filters', None)
  434. if '()' in config:
  435. c = config.pop('()')
  436. if not hasattr(c, '__call__') and hasattr(types, 'ClassType') and type(c) != types.ClassType:
  437. c = self.resolve(c)
  438. factory = c
  439. else:
  440. klass = self.resolve(config.pop('class'))
  441. # Special case for handler which refers to another handler
  442. if issubclass(klass, logging.handlers.MemoryHandler) and\
  443. 'target' in config:
  444. try:
  445. config['target'] = self.config['handlers'][config['target']]
  446. except StandardError as e:
  447. raise ValueError('Unable to set target handler '
  448. '%r: %s' % (config['target'], e))
  449. elif issubclass(klass, logging.handlers.SMTPHandler) and\
  450. 'mailhost' in config:
  451. config['mailhost'] = self.as_tuple(config['mailhost'])
  452. elif issubclass(klass, logging.handlers.SysLogHandler) and\
  453. 'address' in config:
  454. config['address'] = self.as_tuple(config['address'])
  455. factory = klass
  456. kwargs = dict((k, config[k]) for k in config if valid_ident(k))
  457. try:
  458. result = factory(**kwargs)
  459. except TypeError as te:
  460. if "'stream'" not in str(te):
  461. raise
  462. # The argument name changed from strm to stream
  463. # Retry with old name.
  464. # This is so that code can be used with older Python versions
  465. #(e.g. by Django)
  466. kwargs['strm'] = kwargs.pop('stream')
  467. result = factory(**kwargs)
  468. if formatter:
  469. result.setFormatter(formatter)
  470. if level is not None:
  471. result.setLevel(_checkLevel(level))
  472. if filters:
  473. self.add_filters(result, filters)
  474. return result
  475. def add_handlers(self, logger, handlers):
  476. """Add handlers to a logger from a list of names."""
  477. for h in handlers:
  478. try:
  479. logger.addHandler(self.config['handlers'][h])
  480. except StandardError as e:
  481. raise ValueError('Unable to add handler %r: %s' % (h, e))
  482. def common_logger_config(self, logger, config, incremental=False):
  483. """
  484. Perform configuration which is common to root and non-root loggers.
  485. """
  486. level = config.get('level', None)
  487. if level is not None:
  488. logger.setLevel(_checkLevel(level))
  489. if not incremental:
  490. # Remove any existing handlers
  491. for h in logger.handlers[:]:
  492. logger.removeHandler(h)
  493. handlers = config.get('handlers', None)
  494. if handlers:
  495. self.add_handlers(logger, handlers)
  496. filters = config.get('filters', None)
  497. if filters:
  498. self.add_filters(logger, filters)
  499. def configure_logger(self, name, config, incremental=False):
  500. """Configure a non-root logger from a dictionary."""
  501. logger = logging.getLogger(name)
  502. self.common_logger_config(logger, config, incremental)
  503. propagate = config.get('propagate', None)
  504. if propagate is not None:
  505. logger.propagate = propagate
  506. def configure_root(self, config, incremental=False):
  507. """Configure a root logger from a dictionary."""
  508. root = logging.getLogger()
  509. self.common_logger_config(root, config, incremental)
  510. dictConfigClass = DictConfigurator
  511. def dictConfig(config):
  512. """Configure logging using a dictionary."""
  513. dictConfigClass(config).configure()