You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
701 lines
20 KiB
701 lines
20 KiB
# Copyright (c) 2018 gevent. See LICENSE for details.
|
|
"""
|
|
gevent tunables.
|
|
|
|
This should be used as ``from gevent import config``. That variable
|
|
is an object of :class:`Config`.
|
|
|
|
.. versionadded:: 1.3a2
|
|
"""
|
|
|
|
from __future__ import print_function, absolute_import, division
|
|
|
|
import importlib
|
|
import os
|
|
import textwrap
|
|
|
|
from gevent._compat import string_types
|
|
from gevent._compat import WIN
|
|
|
|
__all__ = [
|
|
'config',
|
|
]
|
|
|
|
ALL_SETTINGS = []
|
|
|
|
class SettingType(type):
|
|
# pylint:disable=bad-mcs-classmethod-argument
|
|
|
|
def __new__(cls, name, bases, cls_dict):
|
|
if name == 'Setting':
|
|
return type.__new__(cls, name, bases, cls_dict)
|
|
|
|
cls_dict["order"] = len(ALL_SETTINGS)
|
|
if 'name' not in cls_dict:
|
|
cls_dict['name'] = name.lower()
|
|
|
|
if 'environment_key' not in cls_dict:
|
|
cls_dict['environment_key'] = 'GEVENT_' + cls_dict['name'].upper()
|
|
|
|
|
|
new_class = type.__new__(cls, name, bases, cls_dict)
|
|
new_class.fmt_desc(cls_dict.get("desc", ""))
|
|
new_class.__doc__ = new_class.desc
|
|
ALL_SETTINGS.append(new_class)
|
|
|
|
if new_class.document:
|
|
setting_name = cls_dict['name']
|
|
|
|
def getter(self):
|
|
return self.settings[setting_name].get()
|
|
|
|
def setter(self, value): # pragma: no cover
|
|
# The setter should never be hit, Config has a
|
|
# __setattr__ that would override. But for the sake
|
|
# of consistency we provide one.
|
|
self.settings[setting_name].set(value)
|
|
|
|
prop = property(getter, setter, doc=new_class.__doc__)
|
|
|
|
setattr(Config, cls_dict['name'], prop)
|
|
return new_class
|
|
|
|
def fmt_desc(cls, desc):
|
|
desc = textwrap.dedent(desc).strip()
|
|
if hasattr(cls, 'shortname_map'):
|
|
desc += (
|
|
"\n\nThis is an importable value. It can be "
|
|
"given as a string naming an importable object, "
|
|
"or a list of strings in preference order and the first "
|
|
"successfully importable object will be used. (Separate values "
|
|
"in the environment variable with commas.) "
|
|
"It can also be given as the callable object itself (in code). "
|
|
)
|
|
if cls.shortname_map:
|
|
desc += "Shorthand names for default objects are %r" % (list(cls.shortname_map),)
|
|
if getattr(cls.validate, '__doc__'):
|
|
desc += '\n\n' + textwrap.dedent(cls.validate.__doc__).strip()
|
|
if isinstance(cls.default, str) and hasattr(cls, 'shortname_map'):
|
|
default = "`%s`" % (cls.default,)
|
|
else:
|
|
default = "`%r`" % (cls.default,)
|
|
desc += "\n\nThe default value is %s" % (default,)
|
|
desc += ("\n\nThe environment variable ``%s`` "
|
|
"can be used to control this." % (cls.environment_key,))
|
|
setattr(cls, "desc", desc)
|
|
return desc
|
|
|
|
def validate_invalid(value):
|
|
raise ValueError("Not a valid value: %r" % (value,))
|
|
|
|
def validate_bool(value):
|
|
"""
|
|
This is a boolean value.
|
|
|
|
In the environment variable, it may be given as ``1``, ``true``,
|
|
``on`` or ``yes`` for `True`, or ``0``, ``false``, ``off``, or
|
|
``no`` for `False`.
|
|
"""
|
|
if isinstance(value, string_types):
|
|
value = value.lower().strip()
|
|
if value in ('1', 'true', 'on', 'yes'):
|
|
value = True
|
|
elif value in ('0', 'false', 'off', 'no') or not value:
|
|
value = False
|
|
else:
|
|
raise ValueError("Invalid boolean string: %r" % (value,))
|
|
return bool(value)
|
|
|
|
def validate_anything(value):
|
|
return value
|
|
|
|
convert_str_value_as_is = validate_anything
|
|
|
|
class Setting(object):
|
|
name = None
|
|
value = None
|
|
validate = staticmethod(validate_invalid)
|
|
default = None
|
|
environment_key = None
|
|
document = True
|
|
|
|
desc = """\
|
|
|
|
A long ReST description.
|
|
|
|
The first line should be a single sentence.
|
|
|
|
"""
|
|
|
|
def _convert(self, value):
|
|
if isinstance(value, string_types):
|
|
return value.split(',')
|
|
return value
|
|
|
|
def _default(self):
|
|
result = os.environ.get(self.environment_key, self.default)
|
|
result = self._convert(result)
|
|
return result
|
|
|
|
def get(self):
|
|
# If we've been specifically set, return it
|
|
if 'value' in self.__dict__:
|
|
return self.value
|
|
# Otherwise, read from the environment and reify
|
|
# so we return consistent results.
|
|
self.value = self.validate(self._default())
|
|
return self.value
|
|
|
|
def set(self, val):
|
|
self.value = self.validate(self._convert(val))
|
|
|
|
|
|
Setting = SettingType('Setting', (Setting,), dict(Setting.__dict__))
|
|
|
|
def make_settings():
|
|
"""
|
|
Return fresh instances of all classes defined in `ALL_SETTINGS`.
|
|
"""
|
|
settings = {}
|
|
for setting_kind in ALL_SETTINGS:
|
|
setting = setting_kind()
|
|
assert setting.name not in settings
|
|
settings[setting.name] = setting
|
|
return settings
|
|
|
|
|
|
class Config(object):
|
|
"""
|
|
Global configuration for gevent.
|
|
|
|
There is one instance of this object at ``gevent.config``. If you
|
|
are going to make changes in code, instead of using the documented
|
|
environment variables, you need to make the changes before using
|
|
any parts of gevent that might need those settings. For example::
|
|
|
|
>>> from gevent import config
|
|
>>> config.fileobject = 'thread'
|
|
|
|
>>> from gevent import fileobject
|
|
>>> fileobject.FileObject.__name__
|
|
'FileObjectThread'
|
|
|
|
.. versionadded:: 1.3a2
|
|
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.settings = make_settings()
|
|
|
|
def __getattr__(self, name):
|
|
if name not in self.settings:
|
|
raise AttributeError("No configuration setting for: %r" % name)
|
|
return self.settings[name].get()
|
|
|
|
def __setattr__(self, name, value):
|
|
if name != "settings" and name in self.settings:
|
|
self.set(name, value)
|
|
else:
|
|
super(Config, self).__setattr__(name, value)
|
|
|
|
def set(self, name, value):
|
|
if name not in self.settings:
|
|
raise AttributeError("No configuration setting for: %r" % name)
|
|
self.settings[name].set(value)
|
|
|
|
def __dir__(self):
|
|
return list(self.settings)
|
|
|
|
|
|
class ImportableSetting(object):
|
|
|
|
def _import_one_of(self, candidates):
|
|
assert isinstance(candidates, list)
|
|
if not candidates:
|
|
raise ImportError('Cannot import from empty list')
|
|
|
|
for item in candidates[:-1]:
|
|
try:
|
|
return self._import_one(item)
|
|
except ImportError:
|
|
pass
|
|
|
|
return self._import_one(candidates[-1])
|
|
|
|
def _import_one(self, path, _MISSING=object()):
|
|
if not isinstance(path, string_types):
|
|
return path
|
|
|
|
if '.' not in path or '/' in path:
|
|
raise ImportError("Cannot import %r. "
|
|
"Required format: [package.]module.class. "
|
|
"Or choose from %r"
|
|
% (path, list(self.shortname_map)))
|
|
|
|
|
|
module, item = path.rsplit('.', 1)
|
|
module = importlib.import_module(module)
|
|
x = getattr(module, item, _MISSING)
|
|
if x is _MISSING:
|
|
raise ImportError('Cannot import %r from %r' % (item, module))
|
|
return x
|
|
|
|
shortname_map = {}
|
|
|
|
def validate(self, value):
|
|
if isinstance(value, type):
|
|
return value
|
|
return self._import_one_of([self.shortname_map.get(x, x) for x in value])
|
|
|
|
def get_options(self):
|
|
result = {}
|
|
for name, val in self.shortname_map.items():
|
|
try:
|
|
result[name] = self._import_one(val)
|
|
except ImportError as e:
|
|
result[name] = e
|
|
return result
|
|
|
|
|
|
class BoolSettingMixin(object):
|
|
validate = staticmethod(validate_bool)
|
|
# Don't do string-to-list conversion.
|
|
_convert = staticmethod(convert_str_value_as_is)
|
|
|
|
|
|
class IntSettingMixin(object):
|
|
# Don't do string-to-list conversion.
|
|
def _convert(self, value):
|
|
if value:
|
|
return int(value)
|
|
|
|
validate = staticmethod(validate_anything)
|
|
|
|
|
|
class _PositiveValueMixin(object):
|
|
|
|
def validate(self, value):
|
|
if value is not None and value <= 0:
|
|
raise ValueError("Must be positive")
|
|
return value
|
|
|
|
|
|
class FloatSettingMixin(_PositiveValueMixin):
|
|
def _convert(self, value):
|
|
if value:
|
|
return float(value)
|
|
|
|
|
|
class ByteCountSettingMixin(_PositiveValueMixin):
|
|
|
|
_MULTIPLES = {
|
|
# All keys must be the same size.
|
|
'kb': 1024,
|
|
'mb': 1024 * 1024,
|
|
'gb': 1024 * 1024 * 1024,
|
|
}
|
|
|
|
_SUFFIX_SIZE = 2
|
|
|
|
def _convert(self, value):
|
|
if not value or not isinstance(value, str):
|
|
return value
|
|
value = value.lower()
|
|
for s, m in self._MULTIPLES.items():
|
|
if value[-self._SUFFIX_SIZE:] == s:
|
|
return int(value[:-self._SUFFIX_SIZE]) * m
|
|
return int(value)
|
|
|
|
|
|
class Resolver(ImportableSetting, Setting):
|
|
|
|
desc = """\
|
|
The callable that will be used to create
|
|
:attr:`gevent.hub.Hub.resolver`.
|
|
|
|
See :doc:`dns` for more information.
|
|
"""
|
|
|
|
default = [
|
|
'thread',
|
|
'dnspython',
|
|
'ares',
|
|
'block',
|
|
]
|
|
|
|
shortname_map = {
|
|
'ares': 'gevent.resolver.ares.Resolver',
|
|
'thread': 'gevent.resolver.thread.Resolver',
|
|
'block': 'gevent.resolver.blocking.Resolver',
|
|
'dnspython': 'gevent.resolver.dnspython.Resolver',
|
|
}
|
|
|
|
|
|
|
|
class Threadpool(ImportableSetting, Setting):
|
|
|
|
desc = """\
|
|
The kind of threadpool we use.
|
|
"""
|
|
|
|
default = 'gevent.threadpool.ThreadPool'
|
|
|
|
|
|
class Loop(ImportableSetting, Setting):
|
|
|
|
desc = """\
|
|
The kind of the loop we use.
|
|
|
|
On Windows, this defaults to libuv, while on
|
|
other platforms it defaults to libev.
|
|
|
|
"""
|
|
|
|
default = [
|
|
'libev-cext',
|
|
'libev-cffi',
|
|
'libuv-cffi',
|
|
] if not WIN else [
|
|
'libuv-cffi',
|
|
'libev-cext',
|
|
'libev-cffi',
|
|
]
|
|
|
|
shortname_map = {
|
|
'libev-cext': 'gevent.libev.corecext.loop',
|
|
'libev-cffi': 'gevent.libev.corecffi.loop',
|
|
'libuv-cffi': 'gevent.libuv.loop.loop',
|
|
}
|
|
|
|
shortname_map['libuv'] = shortname_map['libuv-cffi']
|
|
|
|
|
|
class FormatContext(ImportableSetting, Setting):
|
|
name = 'format_context'
|
|
|
|
# using pprint.pformat can override custom __repr__ methods on dict/list
|
|
# subclasses, which can be a security concern
|
|
default = 'pprint.saferepr'
|
|
|
|
|
|
class LibevBackend(Setting):
|
|
name = 'libev_backend'
|
|
environment_key = 'GEVENT_BACKEND'
|
|
|
|
desc = """\
|
|
The backend for libev, such as 'select'
|
|
"""
|
|
|
|
default = None
|
|
|
|
validate = staticmethod(validate_anything)
|
|
|
|
|
|
class FileObject(ImportableSetting, Setting):
|
|
desc = """\
|
|
The kind of ``FileObject`` we will use.
|
|
|
|
See :mod:`gevent.fileobject` for a detailed description.
|
|
|
|
"""
|
|
environment_key = 'GEVENT_FILE'
|
|
|
|
default = [
|
|
'posix',
|
|
'thread',
|
|
]
|
|
|
|
shortname_map = {
|
|
'thread': 'gevent._fileobjectcommon.FileObjectThread',
|
|
'posix': 'gevent._fileobjectposix.FileObjectPosix',
|
|
'block': 'gevent._fileobjectcommon.FileObjectBlock'
|
|
}
|
|
|
|
|
|
class WatchChildren(BoolSettingMixin, Setting):
|
|
desc = """\
|
|
Should we *not* watch children with the event loop watchers?
|
|
|
|
This is an advanced setting.
|
|
|
|
See :mod:`gevent.os` for a detailed description.
|
|
"""
|
|
name = 'disable_watch_children'
|
|
environment_key = 'GEVENT_NOWAITPID'
|
|
default = False
|
|
|
|
|
|
class TraceMalloc(IntSettingMixin, Setting):
|
|
name = 'trace_malloc'
|
|
environment_key = 'PYTHONTRACEMALLOC'
|
|
default = False
|
|
|
|
desc = """\
|
|
Should FFI objects track their allocation?
|
|
|
|
This is only useful for low-level debugging.
|
|
|
|
On Python 3, this environment variable is built in to the
|
|
interpreter, and it may also be set with the ``-X
|
|
tracemalloc`` command line argument.
|
|
|
|
On Python 2, gevent interprets this argument and adds extra
|
|
tracking information for FFI objects.
|
|
"""
|
|
|
|
|
|
class TrackGreenletTree(BoolSettingMixin, Setting):
|
|
name = 'track_greenlet_tree'
|
|
environment_key = 'GEVENT_TRACK_GREENLET_TREE'
|
|
default = True
|
|
|
|
desc = """\
|
|
Should `Greenlet` objects track their spawning tree?
|
|
|
|
Setting this to a false value will make spawning `Greenlet`
|
|
objects and using `spawn_raw` faster, but the
|
|
``spawning_greenlet``, ``spawn_tree_locals`` and ``spawning_stack``
|
|
will not be captured.
|
|
|
|
.. versionadded:: 1.3b1
|
|
"""
|
|
|
|
|
|
## Monitoring settings
|
|
# All env keys should begin with GEVENT_MONITOR
|
|
|
|
class MonitorThread(BoolSettingMixin, Setting):
|
|
name = 'monitor_thread'
|
|
environment_key = 'GEVENT_MONITOR_THREAD_ENABLE'
|
|
default = False
|
|
|
|
desc = """\
|
|
Should each hub start a native OS thread to monitor
|
|
for problems?
|
|
|
|
Such a thread will periodically check to see if the event loop
|
|
is blocked for longer than `max_blocking_time`, producing output on
|
|
the hub's exception stream (stderr by default) if it detects this condition.
|
|
|
|
If this setting is true, then this thread will be created
|
|
the first time the hub is switched to,
|
|
or you can call :meth:`gevent.hub.Hub.start_periodic_monitoring_thread` at any
|
|
time to create it (from the same thread that will run the hub). That function
|
|
will return an instance of :class:`gevent.events.IPeriodicMonitorThread`
|
|
to which you can add your own monitoring functions. That function
|
|
also emits an event of :class:`gevent.events.PeriodicMonitorThreadStartedEvent`.
|
|
|
|
.. seealso:: `max_blocking_time`
|
|
|
|
.. versionadded:: 1.3b1
|
|
"""
|
|
|
|
class MaxBlockingTime(FloatSettingMixin, Setting):
|
|
name = 'max_blocking_time'
|
|
# This environment key doesn't follow the convention because it's
|
|
# meant to match a key used by existing projects
|
|
environment_key = 'GEVENT_MAX_BLOCKING_TIME'
|
|
default = 0.1
|
|
|
|
desc = """\
|
|
If the `monitor_thread` is enabled, this is
|
|
approximately how long (in seconds)
|
|
the event loop will be allowed to block before a warning is issued.
|
|
|
|
This function depends on using `greenlet.settrace`, so installing
|
|
your own trace function after starting the monitoring thread will
|
|
cause this feature to misbehave unless you call the function
|
|
returned by `greenlet.settrace`. If you install a tracing function *before*
|
|
the monitoring thread is started, it will still be called.
|
|
|
|
.. note:: In the unlikely event of creating and using multiple different
|
|
gevent hubs in the same native thread in a short period of time,
|
|
especially without destroying the hubs, false positives may be reported.
|
|
|
|
.. versionadded:: 1.3b1
|
|
"""
|
|
|
|
class MonitorMemoryPeriod(FloatSettingMixin, Setting):
|
|
name = 'memory_monitor_period'
|
|
|
|
environment_key = 'GEVENT_MONITOR_MEMORY_PERIOD'
|
|
default = 5
|
|
|
|
desc = """\
|
|
If `monitor_thread` is enabled, this is approximately how long
|
|
(in seconds) we will go between checking the processes memory usage.
|
|
|
|
Checking the memory usage is relatively expensive on some operating
|
|
systems, so this should not be too low. gevent will place a floor
|
|
value on it.
|
|
"""
|
|
|
|
class MonitorMemoryMaxUsage(ByteCountSettingMixin, Setting):
|
|
name = 'max_memory_usage'
|
|
|
|
environment_key = 'GEVENT_MONITOR_MEMORY_MAX'
|
|
default = None
|
|
|
|
desc = """\
|
|
If `monitor_thread` is enabled,
|
|
then if memory usage exceeds this amount (in bytes), events will
|
|
be emitted. See `gevent.events`. In the environment variable, you can use
|
|
a suffix of 'kb', 'mb' or 'gb' to specify the value in kilobytes, megabytes
|
|
or gigibytes.
|
|
|
|
There is no default value for this setting. If you wish to
|
|
cap memory usage, you must choose a value.
|
|
"""
|
|
|
|
# The ares settings are all interpreted by
|
|
# gevent/resolver/ares.pyx, so we don't do
|
|
# any validation here.
|
|
|
|
class AresSettingMixin(object):
|
|
|
|
document = False
|
|
|
|
@property
|
|
def kwarg_name(self):
|
|
return self.name[5:]
|
|
|
|
validate = staticmethod(validate_anything)
|
|
|
|
_convert = staticmethod(convert_str_value_as_is)
|
|
|
|
class AresFlags(AresSettingMixin, Setting):
|
|
name = 'ares_flags'
|
|
default = None
|
|
environment_key = 'GEVENTARES_FLAGS'
|
|
|
|
class AresTimeout(AresSettingMixin, Setting):
|
|
document = True
|
|
name = 'ares_timeout'
|
|
default = None
|
|
environment_key = 'GEVENTARES_TIMEOUT'
|
|
desc = """\
|
|
|
|
.. deprecated:: 1.3a2
|
|
Prefer the :attr:`resolver_timeout` setting. If both are set,
|
|
the results are not defined.
|
|
"""
|
|
|
|
class AresTries(AresSettingMixin, Setting):
|
|
name = 'ares_tries'
|
|
default = None
|
|
environment_key = 'GEVENTARES_TRIES'
|
|
|
|
class AresNdots(AresSettingMixin, Setting):
|
|
name = 'ares_ndots'
|
|
default = None
|
|
environment_key = 'GEVENTARES_NDOTS'
|
|
|
|
class AresUDPPort(AresSettingMixin, Setting):
|
|
name = 'ares_udp_port'
|
|
default = None
|
|
environment_key = 'GEVENTARES_UDP_PORT'
|
|
|
|
class AresTCPPort(AresSettingMixin, Setting):
|
|
name = 'ares_tcp_port'
|
|
default = None
|
|
environment_key = 'GEVENTARES_TCP_PORT'
|
|
|
|
class AresServers(AresSettingMixin, Setting):
|
|
document = True
|
|
name = 'ares_servers'
|
|
default = None
|
|
environment_key = 'GEVENTARES_SERVERS'
|
|
desc = """\
|
|
A list of strings giving the IP addresses of nameservers for the ares resolver.
|
|
|
|
In the environment variable, these strings are separated by commas.
|
|
|
|
.. deprecated:: 1.3a2
|
|
Prefer the :attr:`resolver_nameservers` setting. If both are set,
|
|
the results are not defined.
|
|
"""
|
|
|
|
# Generic nameservers, works for dnspython and ares.
|
|
class ResolverNameservers(AresSettingMixin, Setting):
|
|
document = True
|
|
name = 'resolver_nameservers'
|
|
default = None
|
|
environment_key = 'GEVENT_RESOLVER_NAMESERVERS'
|
|
desc = """\
|
|
A list of strings giving the IP addresses of nameservers for the (non-system) resolver.
|
|
|
|
In the environment variable, these strings are separated by commas.
|
|
|
|
.. rubric:: Resolver Behaviour
|
|
|
|
* blocking
|
|
|
|
Ignored
|
|
|
|
* Threaded
|
|
|
|
Ignored
|
|
|
|
* dnspython
|
|
|
|
If this setting is not given, the dnspython resolver will
|
|
load nameservers to use from ``/etc/resolv.conf``
|
|
or the Windows registry. This setting replaces any nameservers read
|
|
from those means. Note that the file and registry are still read
|
|
for other settings.
|
|
|
|
.. caution:: dnspython does not validate the members of the list.
|
|
An improper address (such as a hostname instead of IP) has
|
|
undefined results, including hanging the process.
|
|
|
|
* ares
|
|
|
|
Similar to dnspython, but with more platform and compile-time
|
|
options. ares validates that the members of the list are valid
|
|
addresses.
|
|
"""
|
|
|
|
# Normal string-to-list rules. But still validate_anything.
|
|
_convert = Setting._convert
|
|
|
|
# TODO: In the future, support reading a resolv.conf file
|
|
# *other* than /etc/resolv.conf, and do that both on Windows
|
|
# and other platforms. Also offer the option to disable the system
|
|
# configuration entirely.
|
|
|
|
@property
|
|
def kwarg_name(self):
|
|
return 'servers'
|
|
|
|
# Generic timeout, works for dnspython and ares
|
|
class ResolverTimeout(FloatSettingMixin, AresSettingMixin, Setting):
|
|
document = True
|
|
name = 'resolver_timeout'
|
|
environment_key = 'GEVENT_RESOLVER_TIMEOUT'
|
|
desc = """\
|
|
The total amount of time that the DNS resolver will spend making queries.
|
|
|
|
Only the ares and dnspython resolvers support this.
|
|
|
|
.. versionadded:: 1.3a2
|
|
"""
|
|
|
|
@property
|
|
def kwarg_name(self):
|
|
return 'timeout'
|
|
|
|
config = Config()
|
|
|
|
# Go ahead and attempt to import the loop when this class is
|
|
# instantiated. The hub won't work if the loop can't be found. This
|
|
# can solve problems with the class being imported from multiple
|
|
# threads at once, leading to one of the imports failing.
|
|
# factories are themselves handled lazily. See #687.
|
|
|
|
# Don't cache it though, in case the user re-configures through the
|
|
# API.
|
|
|
|
try:
|
|
Loop().get()
|
|
except ImportError: # pragma: no cover
|
|
pass
|
|
|