diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/s3ql.egg-info/PKG-INFO | 31 | ||||
-rw-r--r-- | src/s3ql.egg-info/SOURCES.txt | 1 | ||||
-rw-r--r-- | src/s3ql/__init__.py | 2 | ||||
-rw-r--r-- | src/s3ql/adm.py | 1 | ||||
-rw-r--r-- | src/s3ql/cp.py | 1 | ||||
-rw-r--r-- | src/s3ql/ctrl.py | 1 | ||||
-rw-r--r-- | src/s3ql/fsck.py | 1 | ||||
-rw-r--r-- | src/s3ql/lock.py | 1 | ||||
-rw-r--r-- | src/s3ql/logging.py | 109 | ||||
-rw-r--r-- | src/s3ql/mkfs.py | 12 | ||||
-rw-r--r-- | src/s3ql/mount.py | 7 | ||||
-rw-r--r-- | src/s3ql/parse_args.py | 7 | ||||
-rw-r--r-- | src/s3ql/remove.py | 1 | ||||
-rw-r--r-- | src/s3ql/statfs.py | 1 | ||||
-rw-r--r-- | src/s3ql/umount.py | 1 |
15 files changed, 41 insertions, 136 deletions
diff --git a/src/s3ql.egg-info/PKG-INFO b/src/s3ql.egg-info/PKG-INFO index ee234f3..7cf1735 100644 --- a/src/s3ql.egg-info/PKG-INFO +++ b/src/s3ql.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: s3ql -Version: 2.17.1 +Version: 2.17.1-hg2 Summary: a full-featured file system for online data storage Home-page: https://bitbucket.org/nikratio/s3ql/ Author: Nikolaus Rath @@ -92,13 +92,12 @@ Description: .. Development Status ================== - S3QL is considered stable and suitable for production use. However, - upgrades from one minor version to the next (e.g. *2.x* to *2.x+1*) - may change the public interface (e.g. different command line options), - or require the file system structure to be upgraded (so that the file - system can no longer be accessed by older releases). Therefore, it is - strongly recommended to read the changelog (`Changes.txt` in the S3QL - tarball) before upgrading. + S3QL is considered stable and suitable for production use. Starting + with version 2.17.1, S3QL uses semantic versioning. This means that + backwards-incompatible versions (e.g., versions that require an + upgrade of the file system revision) will be reflected in an increase + of the major version number. + Supported Platforms =================== @@ -111,22 +110,6 @@ Description: .. try to fix them. - Which Version Should I Download? - ================================ - - Short answer: if your system supports Python 3.3 or newer, download - the most recent *2.x* version. - - Long answer: there are two supported branches of S3QL. Both branches - are suitable for production use. The *maint-1.x* branch (version - numbers *1.x*) is no longer actively developed and receives only - selected high-impact bugfixes. It is provided for systems without - Python 3 support. For systems with Python 3.3 or newer, it is - recommended run the *default* S3QL branch (with version numbers - *2.x*). This branch is actively developed and has a number of new - features that are not available in the *1.x* versions. - - Typical Usage ============= diff --git a/src/s3ql.egg-info/SOURCES.txt b/src/s3ql.egg-info/SOURCES.txt index dc1fed6..654bb97 100644 --- a/src/s3ql.egg-info/SOURCES.txt +++ b/src/s3ql.egg-info/SOURCES.txt @@ -228,6 +228,7 @@ src/s3ql/backends/swiftks.py tests/common.py tests/conftest.py tests/mock_server.py +tests/mytest.py tests/pytest.ini tests/t1_backends.py tests/t1_dump.py diff --git a/src/s3ql/__init__.py b/src/s3ql/__init__.py index 0843967..4259f03 100644 --- a/src/s3ql/__init__.py +++ b/src/s3ql/__init__.py @@ -22,7 +22,7 @@ __all__ = [ 'adm', 'backends', 'block_cache', 'common', 'calc_mro', 'REV_VER_MAP', 'RELEASE', 'BUFSIZE', 'CTRL_NAME', 'CTRL_INODE' ] -VERSION = '2.17.1' +VERSION = '2.17.1+hg2' RELEASE = '%s' % VERSION # TODO: On next revision bump, consider removing support for TIME diff --git a/src/s3ql/adm.py b/src/s3ql/adm.py index c8ca0db..d0e6757 100644 --- a/src/s3ql/adm.py +++ b/src/s3ql/adm.py @@ -64,7 +64,6 @@ def parse_args(args): parser.add_backend_options() parser.add_cachedir() parser.add_version() - parser.add_fatal_warnings() options = parser.parse_args(args) diff --git a/src/s3ql/cp.py b/src/s3ql/cp.py index 159b1b5..a6e31a4 100644 --- a/src/s3ql/cp.py +++ b/src/s3ql/cp.py @@ -35,7 +35,6 @@ def parse_args(args): parser.add_debug() parser.add_quiet() parser.add_version() - parser.add_fatal_warnings() parser.add_argument('source', help='source directory', type=(lambda x: x.rstrip('/'))) diff --git a/src/s3ql/ctrl.py b/src/s3ql/ctrl.py index ad5f4ae..ab89cc8 100644 --- a/src/s3ql/ctrl.py +++ b/src/s3ql/ctrl.py @@ -34,7 +34,6 @@ def parse_args(args): parser.add_debug() parser.add_quiet() parser.add_version() - parser.add_fatal_warnings() subparsers = parser.add_subparsers(metavar='<action>', dest='action', help='may be either of') diff --git a/src/s3ql/fsck.py b/src/s3ql/fsck.py index 1235da0..3b03414 100644 --- a/src/s3ql/fsck.py +++ b/src/s3ql/fsck.py @@ -1130,7 +1130,6 @@ def parse_args(args): parser.add_backend_options() parser.add_version() parser.add_storage_url() - parser.add_fatal_warnings() parser.add_argument("--batch", action="store_true", default=False, help="If user input is required, exit without prompting.") diff --git a/src/s3ql/lock.py b/src/s3ql/lock.py index 79dd491..645e321 100644 --- a/src/s3ql/lock.py +++ b/src/s3ql/lock.py @@ -29,7 +29,6 @@ def parse_args(args): parser.add_debug() parser.add_quiet() parser.add_version() - parser.add_fatal_warnings() parser.add_argument('path', metavar='<path>', nargs='+', help='Directories to make immutable.', diff --git a/src/s3ql/logging.py b/src/s3ql/logging.py index f61c733..b66482e 100644 --- a/src/s3ql/logging.py +++ b/src/s3ql/logging.py @@ -12,27 +12,6 @@ import warnings import sys import os.path -# Logging messages with severities larger or equal -# than this value will raise exceptions. -EXCEPTION_SEVERITY = logging.CRITICAL+1 - - -class LoggingError(Exception): - ''' - Raised when a `Logger` instance is used to log a message with - a severity larger than its `exception_severity`. - ''' - - formatter = logging.Formatter('%(message)s') - - def __init__(self, record): - super().__init__() - self.record = record - - def __str__(self): - return 'Unexpected log message: ' + self.formatter.format(self.record) - - class QuietError(Exception): ''' QuietError is the base class for exceptions that should not result @@ -53,6 +32,14 @@ class QuietError(Exception): def __str__(self): return self.msg +class MyFormatter(logging.Formatter): + '''Prepend severity to log message if it exceeds threshold''' + + def format(self, record): + s = super().format(record) + if record.levelno > logging.INFO: + s = '%s: %s' % (record.levelname, s) + return s def create_handler(target): '''Create logging handler for given target''' @@ -75,7 +62,7 @@ def create_handler(target): try: handler = logging.handlers.RotatingFileHandler(fullpath, - maxBytes=1024 ** 2, backupCount=5) + maxBytes=10 * 1024**2, backupCount=5) except PermissionError: raise QuietError('No permission to write log file %s' % fullpath, exitcode=10) @@ -88,6 +75,12 @@ def create_handler(target): return handler def setup_logging(options): + + # We want to be able to detect warnings and higher severities + # in the captured test output. 'critical' has too many potential + # false positives, so we rename this level to "FATAL". + logging.addLevelName(logging.CRITICAL, 'FATAL') + root_logger = logging.getLogger() if root_logger.handlers: root_logger.debug("Logging already initialized.") @@ -99,8 +92,8 @@ def setup_logging(options): elif options.debug and (not hasattr(options, 'log') or not options.log): # When we have debugging enabled but no separate log target, # make stdout logging more detailed. - formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(process)s %(threadName)s ' - '%(name)s.%(funcName)s: %(message)s', + formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(process)s %(levelname)-8s ' + '%(threadName)s %(name)s.%(funcName)s: %(message)s', datefmt="%Y-%m-%d %H:%M:%S") stdout_handler.setFormatter(formatter) stdout_handler.setLevel(logging.NOTSET) @@ -119,24 +112,7 @@ def setup_logging(options): else: root_logger.setLevel(logging.INFO) logging.disable(logging.DEBUG) - - - if hasattr(options, 'fatal_warnings') and options.fatal_warnings: - global EXCEPTION_SEVERITY - EXCEPTION_SEVERITY = logging.WARNING - - # When ResourceWarnings are emitted, any exceptions thrown - # by warning.showwarning() are lost (not even printed to stdout) - # so we cannot rely on our logging mechanism to turn these - # warnings into (visible) exceptions. Instead, we assume that - # --fatal-warnings implies that ResourceWarnings should be - # enabled and tell the warnings module to raise exceptions - # immediately (so that they're at least printed to stderr). - warnings.simplefilter('error', ResourceWarning) - - else: - logging.captureWarnings(capture=True) - + logging.captureWarnings(capture=True) return stdout_handler @@ -150,35 +126,24 @@ def setup_excepthook(): def excepthook(type_, val, tb): root_logger = logging.getLogger() if isinstance(val, QuietError): - # force_log attribute ensures that logging handler will - # not raise exception (if EXCEPTION_SEVERITY is set) - root_logger.error(val.msg, extra={ 'force_log': True }) + root_logger.error(val.msg) sys.exit(val.exitcode) else: - # force_log attribute ensures that logging handler will - # not raise exception (if EXCEPTION_SEVERITY is set) root_logger.error('Uncaught top-level exception:', - exc_info=(type_, val, tb), - extra={ 'force_log': True}) + exc_info=(type_, val, tb)) sys.exit(1) sys.excepthook = excepthook - def add_stdout_logging(quiet=False): - '''Add stdout logging handler to root logger - - If *quiet* is True, logging is sent to stderr rather than stdout, and only - messages with severity WARNING are printed. - ''' + '''Add stdout logging handler to root logger''' root_logger = logging.getLogger() - formatter = logging.Formatter('%(message)s') + formatter = MyFormatter('%(message)s') + handler = logging.StreamHandler(sys.stderr) if quiet: - handler = logging.StreamHandler(sys.stderr) handler.setLevel(logging.WARNING) else: - handler = logging.StreamHandler(sys.stdout) handler.setLevel(logging.INFO) handler.setFormatter(formatter) root_logger.addHandler(handler) @@ -188,9 +153,6 @@ class Logger(logging.getLoggerClass()): ''' This class has the following features in addition to `logging.Logger`: - * Loggers can automatically raise exceptions when a log message exceeds - a specified severity. This is useful when running unit tests. - * Log messages that are emitted with an *log_once* attribute in the *extra* parameter are only emitted once per logger. ''' @@ -200,10 +162,6 @@ class Logger(logging.getLoggerClass()): self.log_cache = set() def handle(self, record): - if (record.levelno >= EXCEPTION_SEVERITY - and not hasattr(record, 'force_log')): - raise LoggingError(record) - if hasattr(record, 'log_once') and record.log_once: id_ = hash((record.name, record.levelno, record.msg, record.args, record.exc_info)) @@ -211,28 +169,9 @@ class Logger(logging.getLoggerClass()): return self.log_cache.add(id_) - # Do not call superclass method directly so that we can - # re-use this method when monkeypatching the root logger. - return self._handle_real(record) - - def _handle_real(self, record): return super().handle(record) - +logging.setLoggerClass(Logger) # Convenience object for use in logging calls, e.g. # log.warning('This will be printed only once', extra=LOG_ONCE) LOG_ONCE = { 'log_once': True } - -# Ensure that no handlers have been created yet -loggers = logging.Logger.manager.loggerDict -if len(loggers) != 0: - raise ImportError('%s must be imported before loggers are created! ' - 'Existing loggers: %s' % (__name__, loggers.keys())) - -# Monkeypatch the root logger -#pylint: disable=W0212 -root_logger_class = type(logging.getLogger()) -root_logger_class._handle_real = root_logger_class.handle -root_logger_class.handle = Logger.handle - -logging.setLoggerClass(Logger) diff --git a/src/s3ql/mkfs.py b/src/s3ql/mkfs.py index b6da7ce..310a78a 100644 --- a/src/s3ql/mkfs.py +++ b/src/s3ql/mkfs.py @@ -38,7 +38,6 @@ def parse_args(args): parser.add_backend_options() parser.add_version() parser.add_storage_url() - parser.add_fatal_warnings() parser.add_argument("-L", default='', help="Filesystem label", dest="label", metavar='<name>',) @@ -90,8 +89,8 @@ def main(args=None): if options.max_obj_size < 1024: # This warning should never be converrted to an exception - log.warning('Warning: maximum object sizes less than 1 MiB will seriously degrade ' - 'performance.', extra={ 'force_log': True }) + log.warning('Maximum object sizes less than 1 MiB will degrade ' + 'performance.', extra={ 'force_log': True }) plain_backend = get_backend(options, raw=True) atexit.register(plain_backend.close) @@ -100,10 +99,9 @@ def main(args=None): "the 'Important Rules to Avoid Loosing Data' section.") if isinstance(plain_backend, s3.Backend) and '.' in plain_backend.bucket_name: - log.warning('***Warning*** S3 Buckets with names containing dots cannot be ' - 'accessed using SSL!') - log.warning('(cf. https://forums.aws.amazon.com/thread.jspa?threadID=130560)') - + log.warning('S3 Buckets with names containing dots cannot be ' + 'accessed using SSL!' + '(cf. https://forums.aws.amazon.com/thread.jspa?threadID=130560)') if 's3ql_metadata' in plain_backend: if not options.force: diff --git a/src/s3ql/mount.py b/src/s3ql/mount.py index ffb9b2f..f2fef6f 100644 --- a/src/s3ql/mount.py +++ b/src/s3ql/mount.py @@ -139,7 +139,7 @@ def main(args=None): options.cachesize = min(rec_cachesize, 0.8 * avail_cache) log.info('Setting cache size to %d MB', options.cachesize / 1024) elif options.cachesize > avail_cache: - log.warning('Warning! Requested cache size %d MB, but only %d MB available', + log.warning('Requested cache size %d MB, but only %d MB available', options.cachesize / 1024, avail_cache / 1024) if options.nfs: @@ -342,8 +342,8 @@ def determine_threads(options): if threads > 0: log.info('Using %d upload threads (memory limited).', threads) else: - log.warning('Warning: compression will require %d MiB memory ' - '(%d%% of total system memory', mem_per_thread / 1024 ** 2, + log.warning('Compression will require %d MiB memory ' + '(%d%% of total system memory', mem_per_thread / 1024 ** 2, mem_per_thread * 100 / memory) threads = 1 return threads @@ -504,7 +504,6 @@ def parse_args(args): parser.add_backend_options() parser.add_version() parser.add_storage_url() - parser.add_fatal_warnings() parser.add_argument("mountpoint", metavar='<mountpoint>', type=os.path.abspath, help='Where to mount the file system') diff --git a/src/s3ql/parse_args.py b/src/s3ql/parse_args.py index f7361d1..163b997 100644 --- a/src/s3ql/parse_args.py +++ b/src/s3ql/parse_args.py @@ -177,13 +177,6 @@ class ArgumentParser(argparse.ArgumentParser): 'will be rotated when they reach 1 MiB, and at most 5 old log ' 'files will be kept. Default: ``%(default)s``') - def add_fatal_warnings(self): - # Make all log messages of severity warning or higher raise - # exceptions. This option is not listed in the --help output and used by - # the unit tests. - self.add_argument("--fatal-warnings", action='store_true', default=False, - help=argparse.SUPPRESS) - def add_storage_url(self): self.add_argument("storage_url", metavar='<storage-url>', type=storage_url_type, diff --git a/src/s3ql/remove.py b/src/s3ql/remove.py index c14d125..b0f1395 100644 --- a/src/s3ql/remove.py +++ b/src/s3ql/remove.py @@ -28,7 +28,6 @@ def parse_args(args): parser.add_debug() parser.add_quiet() parser.add_version() - parser.add_fatal_warnings() parser.add_argument('path', metavar='<path>', nargs='+', help='Directories to remove', diff --git a/src/s3ql/statfs.py b/src/s3ql/statfs.py index 533c68a..57c3408 100644 --- a/src/s3ql/statfs.py +++ b/src/s3ql/statfs.py @@ -24,7 +24,6 @@ def parse_args(args): parser.add_debug() parser.add_quiet() parser.add_version() - parser.add_fatal_warnings() parser.add_argument("mountpoint", metavar='<mountpoint>', type=(lambda x: x.rstrip('/')), diff --git a/src/s3ql/umount.py b/src/s3ql/umount.py index 7ec711c..cd55146 100644 --- a/src/s3ql/umount.py +++ b/src/s3ql/umount.py @@ -35,7 +35,6 @@ def parse_args(args): parser.add_debug() parser.add_quiet() parser.add_version() - parser.add_fatal_warnings() parser.add_argument("mountpoint", metavar='<mountpoint>', type=(lambda x: x.rstrip('/')), |