1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 """Classes to handle advanced configuration in simple to complex applications.
19
20 Allows to load the configuration from a file or from command line
21 options, to generate a sample configuration file or to display
22 program's usage. Fills the gap between optik/optparse and ConfigParser
23 by adding data types (which are also available as a standalone optik
24 extension in the `optik_ext` module).
25
26
27 Quick start: simplest usage
28 ---------------------------
29
30 .. python ::
31
32 >>> import sys
33 >>> from logilab.common.configuration import Configuration
34 >>> options = [('dothis', {'type':'yn', 'default': True, 'metavar': '<y or n>'}),
35 ... ('value', {'type': 'string', 'metavar': '<string>'}),
36 ... ('multiple', {'type': 'csv', 'default': ('yop',),
37 ... 'metavar': '<comma separated values>',
38 ... 'help': 'you can also document the option'}),
39 ... ('number', {'type': 'int', 'default':2, 'metavar':'<int>'}),
40 ... ]
41 >>> config = Configuration(options=options, name='My config')
42 >>> print config['dothis']
43 True
44 >>> print config['value']
45 None
46 >>> print config['multiple']
47 ('yop',)
48 >>> print config['number']
49 2
50 >>> print config.help()
51 Usage: [options]
52
53 Options:
54 -h, --help show this help message and exit
55 --dothis=<y or n>
56 --value=<string>
57 --multiple=<comma separated values>
58 you can also document the option [current: none]
59 --number=<int>
60
61 >>> f = open('myconfig.ini', 'w')
62 >>> f.write('''[MY CONFIG]
63 ... number = 3
64 ... dothis = no
65 ... multiple = 1,2,3
66 ... ''')
67 >>> f.close()
68 >>> config.load_file_configuration('myconfig.ini')
69 >>> print config['dothis']
70 False
71 >>> print config['value']
72 None
73 >>> print config['multiple']
74 ['1', '2', '3']
75 >>> print config['number']
76 3
77 >>> sys.argv = ['mon prog', '--value', 'bacon', '--multiple', '4,5,6',
78 ... 'nonoptionargument']
79 >>> print config.load_command_line_configuration()
80 ['nonoptionargument']
81 >>> print config['value']
82 bacon
83 >>> config.generate_config()
84 # class for simple configurations which don't need the
85 # manager / providers model and prefer delegation to inheritance
86 #
87 # configuration values are accessible through a dict like interface
88 #
89 [MY CONFIG]
90
91 dothis=no
92
93 value=bacon
94
95 # you can also document the option
96 multiple=4,5,6
97
98 number=3
99 >>>
100
101
102
103
104
105 """
106 __docformat__ = "restructuredtext en"
107
108 __all__ = ('OptionsManagerMixIn', 'OptionsProviderMixIn',
109 'ConfigurationMixIn', 'Configuration',
110 'OptionsManager2ConfigurationAdapter')
111
112 import os
113 import sys
114 import re
115 from os.path import exists, expanduser
116 from copy import copy
117 from ConfigParser import ConfigParser, NoOptionError, NoSectionError, \
118 DuplicateSectionError
119 from warnings import warn
120
121 from logilab.common.compat import set, reversed
122 from logilab.common.textutils import normalize_text, unquote
123 from logilab.common.deprecation import deprecated
124 from logilab.common import optik_ext as optparse
125
126 REQUIRED = []
127
128 check_csv = deprecated('use lgc.optik_ext.check_csv')(optparse.check_csv)
131 """raised by set_option when it doesn't know what to do for an action"""
132
135 encoding = encoding or getattr(stream, 'encoding', None)
136 if not encoding:
137 import locale
138 encoding = locale.getpreferredencoding()
139 return encoding
140
142 if isinstance(string, unicode):
143 return string.encode(encoding)
144 return str(string)
145
150 """validate and return a converted value for option of type 'choice'
151 """
152 if not value in optdict['choices']:
153 msg = "option %s: invalid value: %r, should be in %s"
154 raise optparse.OptionValueError(msg % (name, value, optdict['choices']))
155 return value
156
158 """validate and return a converted value for option of type 'choice'
159 """
160 choices = optdict['choices']
161 values = optparse.check_csv(None, name, value)
162 for value in values:
163 if not value in choices:
164 msg = "option %s: invalid value: %r, should be in %s"
165 raise optparse.OptionValueError(msg % (name, value, choices))
166 return values
167
169 """validate and return a converted value for option of type 'csv'
170 """
171 return optparse.check_csv(None, name, value)
172
174 """validate and return a converted value for option of type 'yn'
175 """
176 return optparse.check_yn(None, name, value)
177
179 """validate and return a converted value for option of type 'named'
180 """
181 return optparse.check_named(None, name, value)
182
184 """validate and return a filepath for option of type 'file'"""
185 return optparse.check_file(None, name, value)
186
188 """validate and return a valid color for option of type 'color'"""
189 return optparse.check_color(None, name, value)
190
192 """validate and return a string for option of type 'password'"""
193 return optparse.check_password(None, name, value)
194
196 """validate and return a mx DateTime object for option of type 'date'"""
197 return optparse.check_date(None, name, value)
198
200 """validate and return a time object for option of type 'time'"""
201 return optparse.check_time(None, name, value)
202
204 """validate and return an integer for option of type 'bytes'"""
205 return optparse.check_bytes(None, name, value)
206
207
208 VALIDATORS = {'string' : unquote,
209 'int' : int,
210 'float': float,
211 'file': file_validator,
212 'font': unquote,
213 'color': color_validator,
214 'regexp': re.compile,
215 'csv': csv_validator,
216 'yn': yn_validator,
217 'bool': yn_validator,
218 'named': named_validator,
219 'password': password_validator,
220 'date': date_validator,
221 'time': time_validator,
222 'bytes': bytes_validator,
223 'choice': choice_validator,
224 'multiple_choice': multiple_choice_validator,
225 }
228 if opttype not in VALIDATORS:
229 raise Exception('Unsupported type "%s"' % opttype)
230 try:
231 return VALIDATORS[opttype](optdict, option, value)
232 except TypeError:
233 try:
234 return VALIDATORS[opttype](value)
235 except optparse.OptionValueError:
236 raise
237 except:
238 raise optparse.OptionValueError('%s value (%r) should be of type %s' %
239 (option, value, opttype))
240
251
255
267 return input_validator
268
269 INPUT_FUNCTIONS = {
270 'string': input_string,
271 'password': input_password,
272 }
273
274 for opttype in VALIDATORS.keys():
275 INPUT_FUNCTIONS.setdefault(opttype, _make_input_function(opttype))
278 """monkey patch OptionParser.expand_default since we have a particular
279 way to handle defaults to avoid overriding values in the configuration
280 file
281 """
282 if self.parser is None or not self.default_tag:
283 return option.help
284 optname = option._long_opts[0][2:]
285 try:
286 provider = self.parser.options_manager._all_options[optname]
287 except KeyError:
288 value = None
289 else:
290 optdict = provider.get_option_def(optname)
291 optname = provider.option_name(optname, optdict)
292 value = getattr(provider.config, optname, optdict)
293 value = format_option_value(optdict, value)
294 if value is optparse.NO_DEFAULT or not value:
295 value = self.NO_DEFAULT_VALUE
296 return option.help.replace(self.default_tag, str(value))
297
298
299 -def convert(value, optdict, name=''):
300 """return a validated value for an option according to its type
301
302 optional argument name is only used for error message formatting
303 """
304 try:
305 _type = optdict['type']
306 except KeyError:
307
308 return value
309 return _call_validator(_type, optdict, name, value)
310
315
332
347
366
387
388 format_section = ini_format_section
409
412 """MixIn to handle a configuration from both a configuration file and
413 command line options
414 """
415
416 - def __init__(self, usage, config_file=None, version=None, quiet=0):
417 self.config_file = config_file
418 self.reset_parsers(usage, version=version)
419
420 self.options_providers = []
421
422 self._all_options = {}
423 self._short_options = {}
424 self._nocallback_options = {}
425 self._mygroups = dict()
426
427 self.quiet = quiet
428 self._maxlevel = 0
429
431
432 self.cfgfile_parser = ConfigParser()
433
434 self.cmdline_parser = optparse.OptionParser(usage=usage, version=version)
435 self.cmdline_parser.options_manager = self
436 self._optik_option_attrs = set(self.cmdline_parser.option_class.ATTRS)
437
439 """register an options provider"""
440 assert provider.priority <= 0, "provider's priority can't be >= 0"
441 for i in range(len(self.options_providers)):
442 if provider.priority > self.options_providers[i].priority:
443 self.options_providers.insert(i, provider)
444 break
445 else:
446 self.options_providers.append(provider)
447 non_group_spec_options = [option for option in provider.options
448 if 'group' not in option[1]]
449 groups = getattr(provider, 'option_groups', ())
450 if own_group and non_group_spec_options:
451 self.add_option_group(provider.name.upper(), provider.__doc__,
452 non_group_spec_options, provider)
453 else:
454 for opt, optdict in non_group_spec_options:
455 self.add_optik_option(provider, self.cmdline_parser, opt, optdict)
456 for gname, gdoc in groups:
457 gname = gname.upper()
458 goptions = [option for option in provider.options
459 if option[1].get('group', '').upper() == gname]
460 self.add_option_group(gname, gdoc, goptions, provider)
461
463 """add an option group including the listed options
464 """
465 assert options
466
467 if group_name in self._mygroups:
468 group = self._mygroups[group_name]
469 else:
470 group = optparse.OptionGroup(self.cmdline_parser,
471 title=group_name.capitalize())
472 self.cmdline_parser.add_option_group(group)
473 group.level = provider.level
474 self._mygroups[group_name] = group
475
476 if group_name != "DEFAULT":
477 self.cfgfile_parser.add_section(group_name)
478
479 for opt, optdict in options:
480 self.add_optik_option(provider, group, opt, optdict)
481
483 if 'inputlevel' in optdict:
484 warn('"inputlevel" in option dictionary for %s is deprecated, use'
485 '"level"' % opt, DeprecationWarning)
486 optdict['level'] = optdict.pop('inputlevel')
487 args, optdict = self.optik_option(provider, opt, optdict)
488 option = optikcontainer.add_option(*args, **optdict)
489 self._all_options[opt] = provider
490 self._maxlevel = max(self._maxlevel, option.level)
491
493 """get our personal option definition and return a suitable form for
494 use with optik/optparse
495 """
496 optdict = copy(optdict)
497 others = {}
498 if 'action' in optdict:
499 self._nocallback_options[provider] = opt
500 else:
501 optdict['action'] = 'callback'
502 optdict['callback'] = self.cb_set_provider_option
503
504
505 if 'default' in optdict:
506 if (optparse.OPTPARSE_FORMAT_DEFAULT and 'help' in optdict and
507 optdict.get('default') is not None and
508 not optdict['action'] in ('store_true', 'store_false')):
509 optdict['help'] += ' [current: %default]'
510 del optdict['default']
511 args = ['--' + opt]
512 if 'short' in optdict:
513 self._short_options[optdict['short']] = opt
514 args.append('-' + optdict['short'])
515 del optdict['short']
516
517 for key in optdict.keys():
518 if not key in self._optik_option_attrs:
519 optdict.pop(key)
520 return args, optdict
521
523 """optik callback for option setting"""
524 if opt.startswith('--'):
525
526 opt = opt[2:]
527 else:
528
529 opt = self._short_options[opt[1:]]
530
531 if value is None:
532 value = 1
533 self.global_set_option(opt, value)
534
536 """set option on the correct option provider"""
537 self._all_options[opt].set_option(opt, value)
538
540 """write a configuration file according to the current configuration
541 into the given stream or stdout
542 """
543 options_by_section = {}
544 sections = []
545 for provider in self.options_providers:
546 for section, options in provider.options_by_section():
547 if section is None:
548 section = provider.name
549 if section in skipsections:
550 continue
551 options = [(n, d, v) for (n, d, v) in options
552 if d.get('type') is not None]
553 if not options:
554 continue
555 if not section in sections:
556 sections.append(section)
557 alloptions = options_by_section.setdefault(section, [])
558 alloptions += options
559 stream = stream or sys.stdout
560 encoding = _get_encoding(encoding, stream)
561 printed = False
562 for section in sections:
563 if printed:
564 print >> stream, '\n'
565 format_section(stream, section.upper(), options_by_section[section],
566 encoding)
567 printed = True
568
569 - def generate_manpage(self, pkginfo, section=1, stream=None):
570 """write a man page for the current configuration into the given
571 stream or stdout
572 """
573 self._monkeypatch_expand_default()
574 try:
575 optparse.generate_manpage(self.cmdline_parser, pkginfo,
576 section, stream=stream or sys.stdout,
577 level=self._maxlevel)
578 finally:
579 self._unmonkeypatch_expand_default()
580
581
582
584 """initialize configuration using default values"""
585 for provider in self.options_providers:
586 provider.load_defaults()
587
592
594 """read the configuration file but do not load it (i.e. dispatching
595 values to each options provider)
596 """
597 helplevel = 1
598 while helplevel <= self._maxlevel:
599 opt = '-'.join(['long'] * helplevel) + '-help'
600 if opt in self._all_options:
601 break
602 def helpfunc(option, opt, val, p, level=helplevel):
603 print self.help(level)
604 sys.exit(0)
605 helpmsg = '%s verbose help.' % ' '.join(['more'] * helplevel)
606 optdict = {'action' : 'callback', 'callback' : helpfunc,
607 'help' : helpmsg}
608 provider = self.options_providers[0]
609 self.add_optik_option(provider, self.cmdline_parser, opt, optdict)
610 provider.options += ( (opt, optdict), )
611 helplevel += 1
612 if config_file is None:
613 config_file = self.config_file
614 if config_file is not None:
615 config_file = expanduser(config_file)
616 if config_file and exists(config_file):
617 parser = self.cfgfile_parser
618 parser.read([config_file])
619
620 for sect, values in parser._sections.items():
621 if not sect.isupper() and values:
622 parser._sections[sect.upper()] = values
623 elif not self.quiet:
624 msg = 'No config file found, using default configuration'
625 print >> sys.stderr, msg
626 return
627
645
647 """dispatch values previously read from a configuration file to each
648 options provider)
649 """
650 parser = self.cfgfile_parser
651 for provider in self.options_providers:
652 for section, option, optdict in provider.all_options():
653 try:
654 value = parser.get(section, option)
655 provider.set_option(option, value, optdict=optdict)
656 except (NoSectionError, NoOptionError), ex:
657 continue
658
660 """override configuration according to given parameters
661 """
662 for opt, opt_value in kwargs.items():
663 opt = opt.replace('_', '-')
664 provider = self._all_options[opt]
665 provider.set_option(opt, opt_value)
666
668 """override configuration according to command line parameters
669
670 return additional arguments
671 """
672 self._monkeypatch_expand_default()
673 try:
674 if args is None:
675 args = sys.argv[1:]
676 else:
677 args = list(args)
678 (options, args) = self.cmdline_parser.parse_args(args=args)
679 for provider in self._nocallback_options.keys():
680 config = provider.config
681 for attr in config.__dict__.keys():
682 value = getattr(options, attr, None)
683 if value is None:
684 continue
685 setattr(config, attr, value)
686 return args
687 finally:
688 self._unmonkeypatch_expand_default()
689
690
691
692
694 """add a dummy option section for help purpose """
695 group = optparse.OptionGroup(self.cmdline_parser,
696 title=title.capitalize(),
697 description=description)
698 group.level = level
699 self._maxlevel = max(self._maxlevel, level)
700 self.cmdline_parser.add_option_group(group)
701
703
704 try:
705 self.__expand_default_backup = optparse.HelpFormatter.expand_default
706 optparse.HelpFormatter.expand_default = expand_default
707 except AttributeError:
708
709 pass
711
712 if hasattr(optparse.HelpFormatter, 'expand_default'):
713
714 optparse.HelpFormatter.expand_default = self.__expand_default_backup
715
716 - def help(self, level=0):
717 """return the usage string for available options """
718 self.cmdline_parser.formatter.output_level = level
719 self._monkeypatch_expand_default()
720 try:
721 return self.cmdline_parser.format_help()
722 finally:
723 self._unmonkeypatch_expand_default()
724
725 @property
727 msg = '"_optik_parser" attribute has been renamed to "cmdline_parser"'
728 warn(msg, DeprecationWarning)
729 return self.cmdline_parser
730
731 @property
733 msg ='"_config_parser" attribute has been renamed to "cfgfile_parser"'
734 warn(msg, DeprecationWarning, stacklevel=2)
735 return self.cfgfile_parser
736
739 """used to ease late binding of default method (so you can define options
740 on the class using default methods on the configuration instance)
741 """
743 self.method = methname
744 self._inst = None
745
746 - def bind(self, instance):
747 """bind the method to its instance"""
748 if self._inst is None:
749 self._inst = instance
750
752 assert self._inst, 'unbound method'
753 return getattr(self._inst, self.method)(*args, **kwargs)
754
757 """Mixin to provide options to an OptionsManager"""
758
759
760 priority = -1
761 name = 'default'
762 options = ()
763 level = 0
764
766 self.config = optparse.Values()
767 for option in self.options:
768 try:
769 option, optdict = option
770 except ValueError:
771 raise Exception('Bad option: %r' % option)
772 if isinstance(optdict.get('default'), Method):
773 optdict['default'].bind(self)
774 elif isinstance(optdict.get('callback'), Method):
775 optdict['callback'].bind(self)
776 self.load_defaults()
777
779 """initialize the provider using default values"""
780 for opt, optdict in self.options:
781 action = optdict.get('action')
782 if action != 'callback':
783
784 default = self.option_default(opt, optdict)
785 if default is REQUIRED:
786 continue
787 self.set_option(opt, default, action, optdict)
788
790 """return the default value for an option"""
791 if optdict is None:
792 optdict = self.get_option_def(opt)
793 default = optdict.get('default')
794 if callable(default):
795 default = default()
796 return default
797
799 """get the config attribute corresponding to opt
800 """
801 if optdict is None:
802 optdict = self.get_option_def(opt)
803 return optdict.get('dest', opt.replace('-', '_'))
804
806 """get the current value for the given option"""
807 return getattr(self.config, self.option_name(opt), None)
808
809 - def set_option(self, opt, value, action=None, optdict=None):
810 """method called to set an option (registered in the options list)
811 """
812
813 if optdict is None:
814 optdict = self.get_option_def(opt)
815 if value is not None:
816 value = convert(value, optdict, opt)
817 if action is None:
818 action = optdict.get('action', 'store')
819 if optdict.get('type') == 'named':
820 optname = self.option_name(opt, optdict)
821 currentvalue = getattr(self.config, optname, None)
822 if currentvalue:
823 currentvalue.update(value)
824 value = currentvalue
825 if action == 'store':
826 setattr(self.config, self.option_name(opt, optdict), value)
827 elif action in ('store_true', 'count'):
828 setattr(self.config, self.option_name(opt, optdict), 0)
829 elif action == 'store_false':
830 setattr(self.config, self.option_name(opt, optdict), 1)
831 elif action == 'append':
832 opt = self.option_name(opt, optdict)
833 _list = getattr(self.config, opt, None)
834 if _list is None:
835 if type(value) in (type(()), type([])):
836 _list = value
837 elif value is not None:
838 _list = []
839 _list.append(value)
840 setattr(self.config, opt, _list)
841 elif type(_list) is type(()):
842 setattr(self.config, opt, _list + (value,))
843 else:
844 _list.append(value)
845 elif action == 'callback':
846 optdict['callback'](None, opt, value, None)
847 else:
848 raise UnsupportedAction(action)
849
871
873 """return the dictionary defining an option given it's name"""
874 assert self.options
875 for option in self.options:
876 if option[0] == opt:
877 return option[1]
878 raise optparse.OptionError('no such option in section %r' % self.name, opt)
879
880
882 """return an iterator on available options for this provider
883 option are actually described by a 3-uple:
884 (section, option name, option dictionary)
885 """
886 for section, options in self.options_by_section():
887 if section is None:
888 if self.name is None:
889 continue
890 section = self.name.upper()
891 for option, optiondict, value in options:
892 yield section, option, optiondict
893
895 """return an iterator on options grouped by section
896
897 (section, [list of (optname, optdict, optvalue)])
898 """
899 sections = {}
900 for optname, optdict in self.options:
901 sections.setdefault(optdict.get('group'), []).append(
902 (optname, optdict, self.option_value(optname)))
903 if None in sections:
904 yield None, sections.pop(None)
905 for section, options in sections.items():
906 yield section.upper(), options
907
913
916 """basic mixin for simple configurations which don't need the
917 manager / providers model
918 """
935
944
947
949 return iter(self.config.__dict__.iteritems())
950
952 try:
953 return getattr(self.config, self.option_name(key))
954 except (optparse.OptionValueError, AttributeError):
955 raise KeyError(key)
956
959
960 - def get(self, key, default=None):
961 try:
962 return getattr(self.config, self.option_name(key))
963 except (optparse.OptionError, AttributeError):
964 return default
965
968 """class for simple configurations which don't need the
969 manager / providers model and prefer delegation to inheritance
970
971 configuration values are accessible through a dict like interface
972 """
973
974 - def __init__(self, config_file=None, options=None, name=None,
975 usage=None, doc=None, version=None):
983
986 """Adapt an option manager to behave like a
987 `logilab.common.configuration.Configuration` instance
988 """
990 self.config = provider
991
993 return getattr(self.config, key)
994
996 provider = self.config._all_options[key]
997 try:
998 return getattr(provider.config, provider.option_name(key))
999 except AttributeError:
1000 raise KeyError(key)
1001
1004
1005 - def get(self, key, default=None):
1006 provider = self.config._all_options[key]
1007 try:
1008 return getattr(provider.config, provider.option_name(key))
1009 except AttributeError:
1010 return default
1011
1014 """initialize newconfig from a deprecated configuration file
1015
1016 possible changes:
1017 * ('renamed', oldname, newname)
1018 * ('moved', option, oldgroup, newgroup)
1019 * ('typechanged', option, oldtype, newvalue)
1020 """
1021
1022 changesindex = {}
1023 for action in changes:
1024 if action[0] == 'moved':
1025 option, oldgroup, newgroup = action[1:]
1026 changesindex.setdefault(option, []).append((action[0], oldgroup, newgroup))
1027 continue
1028 if action[0] == 'renamed':
1029 oldname, newname = action[1:]
1030 changesindex.setdefault(newname, []).append((action[0], oldname))
1031 continue
1032 if action[0] == 'typechanged':
1033 option, oldtype, newvalue = action[1:]
1034 changesindex.setdefault(option, []).append((action[0], oldtype, newvalue))
1035 continue
1036 if action[1] in ('added', 'removed'):
1037 continue
1038 raise Exception('unknown change %s' % action[0])
1039
1040 options = []
1041 for optname, optdef in newconfig.options:
1042 for action in changesindex.pop(optname, ()):
1043 if action[0] == 'moved':
1044 oldgroup, newgroup = action[1:]
1045 optdef = optdef.copy()
1046 optdef['group'] = oldgroup
1047 elif action[0] == 'renamed':
1048 optname = action[1]
1049 elif action[0] == 'typechanged':
1050 oldtype = action[1]
1051 optdef = optdef.copy()
1052 optdef['type'] = oldtype
1053 options.append((optname, optdef))
1054 if changesindex:
1055 raise Exception('unapplied changes: %s' % changesindex)
1056 oldconfig = Configuration(options=options, name=newconfig.name)
1057
1058 oldconfig.load_file_configuration(configfile)
1059
1060 changes.reverse()
1061 done = set()
1062 for action in changes:
1063 if action[0] == 'renamed':
1064 oldname, newname = action[1:]
1065 newconfig[newname] = oldconfig[oldname]
1066 done.add(newname)
1067 elif action[0] == 'typechanged':
1068 optname, oldtype, newvalue = action[1:]
1069 newconfig[optname] = newvalue
1070 done.add(optname)
1071 for optname, optdef in newconfig.options:
1072 if optdef.get('type') and not optname in done:
1073 newconfig.set_option(optname, oldconfig[optname], optdict=optdef)
1074
1077 """preprocess options to remove duplicate"""
1078 alloptions = {}
1079 options = list(options)
1080 for i in range(len(options)-1, -1, -1):
1081 optname, optdict = options[i]
1082 if optname in alloptions:
1083 options.pop(i)
1084 alloptions[optname].update(optdict)
1085 else:
1086 alloptions[optname] = optdict
1087 return tuple(options)
1088