python logging with yaml

Recently, I was made a service which can provide a simple way to get best model. so, i spent lot of time to read source code of auto-sklearn, auto-sklearn is an automated machine learning toolkit and a drop-in replacement for a scikit-learn estimator.

when I write my logging module I found the logging module of auto-sklearn which used yaml file as config, the config file：

 1 ---
 2 version: 1
 3 disable_existing_loggers: False
 4 formatters:
 5   simple:
 6     format: '[%(asctime)s: %(levelname)s] %(message)s'
 7 
 8 handlers:
 9   console:
10     class: logging.StreamHandler
11     level: WARNING
12     formatter: simple
13     stream: ext://sys.stdout
14 
15   file_handler:
16     class: logging.FileHandler
17     level: DEBUG
18     formatter: simple
19     filename: autosklearn.log
20 
21 root:
22   level: ERROR
23   handlers: [console, file_handler]
24 
25 loggers:
26   server:
27     level: INFO
28     handlers: [file_handler]
29     propagate: no

the caller can define loggers、handlers、formatters here, after that, caller need write setup code and get logger code.

 1 # -*- encoding: utf-8 -*-
 2 import logging
 3 import logging.config
 4 import os
 5 import yaml
 6 
 7 
 8 def setup_logger(output_file=None, logging_config=None):
 9     # logging_config must be a dictionary object specifying the configuration
10     if not os.path.exists(os.path.dirname(output_file)):
11         os.makedirs(os.path.dirname(output_file))
12     if logging_config is not None:
13         if output_file is not None:
14             logging_config['handlers']['file_handler']['filename'] = output_file
15         logging.config.dictConfig(logging_config)
16     else:
17         with open(os.path.join(os.path.dirname(__file__), 'logging.yaml'),
18                   'r') as fh:
19             logging_config = yaml.safe_load(fh)
20         if output_file is not None:
21             logging_config['handlers']['file_handler']['filename'] = output_file
22         logging.config.dictConfig(logging_config)
23 
24 
25 def _create_logger(name):
26     return logging.getLogger(name)
27 
28 
29 def get_logger(name):
30     logger = PickableLoggerAdapter(name)
31     return logger
32 
33 
34 class PickableLoggerAdapter(object):
35 
36     def __init__(self, name):
37         self.name = name
38         self.logger = _create_logger(name)
39 
40     def __getstate__(self):
41         """
42         Method is called when pickle dumps an object.
43 
44         Returns
45         -------
46         Dictionary, representing the object state to be pickled. Ignores
47         the self.logger field and only returns the logger name.
48         """
49         return {'name': self.name}
50 
51     def __setstate__(self, state):
52         """
53         Method is called when pickle loads an object. Retrieves the name and
54         creates a logger.
55 
56         Parameters
57         ----------
58         state - dictionary, containing the logger name.
59 
60         """
61         self.name = state['name']
62         self.logger = _create_logger(self.name)
63 
64     def debug(self, msg, *args, **kwargs):
65         self.logger.debug(msg, *args, **kwargs)
66 
67     def info(self, msg, *args, **kwargs):
68         self.logger.info(msg, *args, **kwargs)
69 
70     def warning(self, msg, *args, **kwargs):
71         self.logger.warning(msg, *args, **kwargs)
72 
73     def error(self, msg, *args, **kwargs):
74         self.logger.error(msg, *args, **kwargs)
75 
76     def exception(self, msg, *args, **kwargs):
77         self.logger.exception(msg, *args, **kwargs)
78 
79     def critical(self, msg, *args, **kwargs):
80         self.logger.critical(msg, *args, **kwargs)
81 
82     def log(self, level, msg, *args, **kwargs):
83         self.logger.log(level, msg, *args, **kwargs)
84 
85     def isEnabledFor(self, level):
86         return self.logger.isEnabledFor(level)

get_logger return a logger object, setup_logger setup logger handler which define output where.

when you use it,like this

1 from .util import get_logger, setup_logger
2 
3 
4 APP_DIR = os.path.dirname(os.path.abspath(__file__))
5 setup_logger(output_file=os.path.join(APP_DIR, 'logs', 'server.log'))
6 LOG = get_logger("server")

Define a global variable LOG for other modules to call.

all done.