diff options
| author | Nikolaus Rath <Nikolaus@rath.org> | 2018-07-21 12:09:30 +0100 |
|---|---|---|
| committer | Nikolaus Rath <Nikolaus@rath.org> | 2018-07-21 12:09:30 +0100 |
| commit | 10d09ff205e4f527d4ac40cb22f9f98495257458 (patch) | |
| tree | d8d321f73543626e0b8394572f0fa468f48e3487 | |
| parent | cc6c38d4009452ff5c464c71cbea337c29d97875 (diff) | |
Import s3ql_2.29+dfsg.orig.tar.bz2
47 files changed, 432 insertions, 320 deletions
diff --git a/Changes.txt b/Changes.txt index 31717bb..0b1997f 100644 --- a/Changes.txt +++ b/Changes.txt @@ -1,3 +1,13 @@ +2018-07-21, S3QL 2.29 + + * Removal of files that have not yet been uploaded to the backend is + now faster, with less data being uploaded only to be removed + shortly after. + + * Clarified NFS support in documentation. + + * contrib/clone_fs.py is now working again. + 2018-06-14, S3QL 2.28 * Fixed upgrade process for unencrypted filesystems (was broken in @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: s3ql -Version: 2.28 +Version: 2.29 Summary: a full-featured file system for online data storage Home-page: https://bitbucket.org/nikratio/s3ql/ Author: Nikolaus Rath @@ -144,7 +144,7 @@ Description: .. `s3ql+subscribe@googlegroups.com <mailto:s3ql+subscribe@googlegroups.com>`_. - Please report any bugs you may encounter in the `Bitbucket Issue Tracker`_. + Please report any bugs you may encounter in the `GitHub Issue Tracker`_. Contributing ============ @@ -161,7 +161,7 @@ Description: .. .. _`Installation Instructions`: https://bitbucket.org/nikratio/s3ql/wiki/Installation .. _`S3QL FAQ`: https://bitbucket.org/nikratio/s3ql/wiki/FAQ .. _`S3QL Mailing List`: http://groups.google.com/group/s3ql - .. _`Bitbucket Issue Tracker`: https://bitbucket.org/nikratio/s3ql/issues + .. _`GitHub Issue Tracker`: https://github.com/s3ql/s3ql/issues .. _BitBucket: https://bitbucket.org/nikratio/s3ql/ .. _GitHub: https://github.com/s3ql/main .. _`Rath Consulting`: http://www.rath-consulting.biz/ @@ -135,7 +135,7 @@ The following resources are available: `s3ql+subscribe@googlegroups.com <mailto:s3ql+subscribe@googlegroups.com>`_. -Please report any bugs you may encounter in the `Bitbucket Issue Tracker`_. +Please report any bugs you may encounter in the `GitHub Issue Tracker`_. Contributing ============ @@ -152,7 +152,7 @@ Professional support is offered via `Rath Consulting`_. .. _`Installation Instructions`: https://bitbucket.org/nikratio/s3ql/wiki/Installation .. _`S3QL FAQ`: https://bitbucket.org/nikratio/s3ql/wiki/FAQ .. _`S3QL Mailing List`: http://groups.google.com/group/s3ql -.. _`Bitbucket Issue Tracker`: https://bitbucket.org/nikratio/s3ql/issues +.. _`GitHub Issue Tracker`: https://github.com/s3ql/s3ql/issues .. _BitBucket: https://bitbucket.org/nikratio/s3ql/ .. _GitHub: https://github.com/s3ql/main .. _`Rath Consulting`: http://www.rath-consulting.biz/ diff --git a/contrib/benchmark.py b/contrib/benchmark.py index 7ac9c2b..2af39a9 100755 --- a/contrib/benchmark.py +++ b/contrib/benchmark.py @@ -49,7 +49,6 @@ def parse_args(args): description='Measure S3QL write performance, uplink bandwidth and ' 'compression speed and determine limiting factor.') - parser.add_authfile() parser.add_quiet() parser.add_debug() parser.add_backend_options() @@ -160,7 +159,7 @@ def main(args=None): out_speed = dict() for alg in ALGS: log.info('compressing with %s-6...', alg) - backend = ComprencBackend(b'pass', (alg, 6), Backend('local://' + backend_dir, None, None)) + backend = ComprencBackend(b'pass', (alg, 6),Backend(argparse.Namespace(storage_url='local://' + backend_dir))) def do_write(dst): #pylint: disable=E0102 src.seek(0) stamp = time.time() diff --git a/contrib/clone_fs.py b/contrib/clone_fs.py index 470bcdc..389630c 100755 --- a/contrib/clone_fs.py +++ b/contrib/clone_fs.py @@ -14,6 +14,7 @@ import os import sys import tempfile import time +import argparse from queue import Queue, Full as QueueFull # We are running from the S3QL source directory, make sure @@ -24,8 +25,8 @@ if (os.path.exists(os.path.join(basedir, 'setup.py')) and sys.path = [os.path.join(basedir, 'src')] + sys.path from s3ql.logging import logging, setup_logging, QuietError -from s3ql.common import get_backend_factory, AsyncFn, handle_on_return -from s3ql.backends.common import DanglingStorageURLError +from s3ql.common import AsyncFn, handle_on_return +from s3ql.backends.common import DanglingStorageURLError, NoSuchObject from s3ql import BUFSIZE from s3ql.parse_args import ArgumentParser, storage_url_type @@ -37,25 +38,50 @@ def parse_args(args): parser = ArgumentParser( description='Clone an S3QL file system.') - parser.add_authfile() parser.add_quiet() parser.add_debug() parser.add_backend_options() parser.add_version() + parser.add_argument("--threads", type=int, default=3, + help='Number of threads to use') + + # Can't use parser.add_storage_url(), because we need both a source + # and destination. + parser.add_argument("--authfile", type=str, metavar='<path>', + default=os.path.expanduser("~/.s3ql/authinfo2"), + help='Read authentication credentials from this file ' + '(default: `~/.s3ql/authinfo2)`') parser.add_argument("src_storage_url", metavar='<source-storage-url>', type=storage_url_type, help='Storage URL of the source backend that contains the file system') - parser.add_argument("dst_storage_url", metavar='<destination-storage-url>', type=storage_url_type, help='Storage URL of the destination backend') - parser.add_argument("--threads", type=int, default=3, - help='Number of threads to use') - return parser.parse_args(args) + options = parser.parse_args(args) + setup_logging(options) + # Print message so that the user has some idea what credentials are + # wanted (if not specified in authfile). + log.info('Connecting to source backend...') + options.storage_url = options.src_storage_url + parser._init_backend_factory(options) + src_options = argparse.Namespace() + src_options.__dict__.update(options.__dict__) + options.src_backend_factory = lambda: src_options.backend_class(src_options) + + log.info('Connecting to destination backend...') + options.storage_url = options.dst_storage_url + parser._init_backend_factory(options) + dst_options = argparse.Namespace() + dst_options.__dict__.update(options.__dict__) + options.dst_backend_factory = lambda: dst_options.backend_class(dst_options) + del options.storage_url + del options.backend_class + + return options @handle_on_return def copy_loop(queue, src_backend_factory, dst_backend_factory, on_return): @@ -96,18 +122,17 @@ def copy_loop(queue, src_backend_factory, dst_backend_factory, on_return): def main(args=None): options = parse_args(args) - setup_logging(options) + src_backend_factory = options.src_backend_factory + dst_backend_factory = options.dst_backend_factory + + # Try to access both backends before starting threads try: - options.storage_url = options.src_storage_url - src_backend_factory = get_backend_factory(options.src_storage_url, - options.backend_options, - options.authfile, raw=True) - - options.storage_url = options.dst_storage_url - dst_backend_factory = get_backend_factory(options.dst_storage_url, - options.backend_options, - options.authfile, raw=True) + src_backend_factory().lookup('s3ql_metadata') + try: + dst_backend_factory().lookup('some random object') + except NoSuchObject: + pass except DanglingStorageURLError as exc: raise QuietError(str(exc)) from None diff --git a/contrib/expire_backups.1 b/contrib/expire_backups.1 index 5a16eec..6bf67f2 100644 --- a/contrib/expire_backups.1 +++ b/contrib/expire_backups.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "EXPIRE_BACKUPS" "1" "Jun 14, 2018" "2.28" "S3QL" +.TH "EXPIRE_BACKUPS" "1" "Jul 21, 2018" "2.29" "S3QL" .SH NAME expire_backups \- Intelligently expire old backups . diff --git a/contrib/pcp.1 b/contrib/pcp.1 index 89ead9c..7ba3081 100644 --- a/contrib/pcp.1 +++ b/contrib/pcp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "PCP" "1" "Jun 14, 2018" "2.28" "S3QL" +.TH "PCP" "1" "Jul 21, 2018" "2.29" "S3QL" .SH NAME pcp \- Recursive, parallel copy of directory trees . diff --git a/contrib/remove_objects.py b/contrib/remove_objects.py index 167fefd..339dc36 100755 --- a/contrib/remove_objects.py +++ b/contrib/remove_objects.py @@ -30,15 +30,12 @@ def parse_args(args): parser = ArgumentParser(description='Batch remove objects from an S3QL backend') - parser.add_authfile() + parser.add_storage_url() parser.add_quiet() parser.add_debug() parser.add_backend_options() parser.add_version() - parser.add_argument("storage_url", type=storage_url_type, - help='Storage URL of the backend to delete from') - parser.add_argument("file", type=argparse.FileType(mode='r', encoding='utf-8'), help='File with newline separated object keys to delete') diff --git a/doc/latex/manual.aux b/doc/latex/manual.aux index 20229de..c6931be 100644 --- a/doc/latex/manual.aux +++ b/doc/latex/manual.aux @@ -42,13 +42,13 @@ \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{installation::doc}{{2}{3}{Installation}{chapter.2}{}} -\newlabel{installation:installation}{{2}{3}{Installation}{chapter.2}{}} \newlabel{installation:github}{{2}{3}{Installation}{chapter.2}{}} +\newlabel{installation:installation}{{2}{3}{Installation}{chapter.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.1}Dependencies}{3}{section.2.1}} \newlabel{installation:dependencies}{{2.1}{3}{Dependencies}{section.2.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.2}Installing S3QL}{4}{section.2.2}} -\newlabel{installation:installing-s3ql}{{2.2}{4}{Installing S3QL}{section.2.2}{}} \newlabel{installation:inst-s3ql}{{2.2}{4}{Installing S3QL}{section.2.2}{}} +\newlabel{installation:installing-s3ql}{{2.2}{4}{Installing S3QL}{section.2.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.3}Development Version}{4}{section.2.3}} \newlabel{installation:development-version}{{2.3}{4}{Development Version}{section.2.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.4}Running tests requiring remote servers}{4}{section.2.4}} @@ -58,17 +58,17 @@ \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{backends::doc}{{3}{7}{Storage Backends}{chapter.3}{}} -\newlabel{backends:id1}{{3}{7}{Storage Backends}{chapter.3}{}} \newlabel{backends:storage-backends}{{3}{7}{Storage Backends}{chapter.3}{}} \newlabel{backends:sphinx}{{3}{7}{Storage Backends}{chapter.3}{}} +\newlabel{backends:id1}{{3}{7}{Storage Backends}{chapter.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.1}Google Storage}{7}{section.3.1}} \newlabel{backends:google-storage}{{3.1}{7}{Google Storage}{section.3.1}{}} \newlabel{backends:cmdoption-gs_backend-arg-no-ssl}{{3.1}{7}{Google Storage}{section*.3}{}} \newlabel{backends:cmdoption-gs_backend-arg-ssl-ca-path}{{3.1}{8}{Google Storage}{section*.4}{}} \newlabel{backends:cmdoption-gs_backend-arg-tcp-timeout}{{3.1}{8}{Google Storage}{section*.5}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.2}Amazon S3}{8}{section.3.2}} -\newlabel{backends:amazon-s3}{{3.2}{8}{Amazon S3}{section.3.2}{}} \newlabel{backends:google-storage-manager}{{3.2}{8}{Amazon S3}{section.3.2}{}} +\newlabel{backends:amazon-s3}{{3.2}{8}{Amazon S3}{section.3.2}{}} \newlabel{backends:cmdoption-s3_backend-arg-no-ssl}{{3.2}{8}{Amazon S3}{section*.6}{}} \newlabel{backends:cmdoption-s3_backend-arg-ssl-ca-path}{{3.2}{8}{Amazon S3}{section*.7}{}} \newlabel{backends:cmdoption-s3_backend-arg-tcp-timeout}{{3.2}{8}{Amazon S3}{section*.8}{}} @@ -76,8 +76,8 @@ \newlabel{backends:cmdoption-s3_backend-arg-ia}{{3.2}{8}{Amazon S3}{section*.10}{}} \newlabel{backends:cmdoption-s3_backend-arg-rrs}{{3.2}{8}{Amazon S3}{section*.11}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.3}OpenStack/Swift}{9}{section.3.3}} -\newlabel{backends:openstack-swift}{{3.3}{9}{OpenStack/Swift}{section.3.3}{}} \newlabel{backends:openstack-backend}{{3.3}{9}{OpenStack/Swift}{section.3.3}{}} +\newlabel{backends:openstack-swift}{{3.3}{9}{OpenStack/Swift}{section.3.3}{}} \newlabel{backends:cmdoption-swift_backend-arg-no-ssl}{{3.3}{9}{OpenStack/Swift}{section*.12}{}} \newlabel{backends:cmdoption-swift_backend-arg-ssl-ca-path}{{3.3}{9}{OpenStack/Swift}{section*.13}{}} \newlabel{backends:cmdoption-swift_backend-arg-tcp-timeout}{{3.3}{9}{OpenStack/Swift}{section*.14}{}} @@ -100,10 +100,10 @@ \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{durability:sshfs}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} -\newlabel{durability:durability}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} \newlabel{durability::doc}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} \newlabel{durability:important-rules-to-avoid-losing-data}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} +\newlabel{durability:sshfs}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} +\newlabel{durability:durability}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.1}Rules in a Nutshell}{13}{section.4.1}} \newlabel{durability:rules-in-a-nutshell}{{4.1}{13}{Rules in a Nutshell}{section.4.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.2}Consistency Window List}{14}{section.4.2}} @@ -111,8 +111,8 @@ \@writefile{toc}{\contentsline {section}{\numberline {4.3}Data Consistency}{14}{section.4.3}} \newlabel{durability:data-consistency}{{4.3}{14}{Data Consistency}{section.4.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.4}Data Durability}{15}{section.4.4}} -\newlabel{durability:backend-reliability}{{4.4}{15}{Data Durability}{section.4.4}{}} \newlabel{durability:data-durability}{{4.4}{15}{Data Durability}{section.4.4}{}} +\newlabel{durability:backend-reliability}{{4.4}{15}{Data Durability}{section.4.4}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {5}File System Creation}{17}{chapter.5}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} @@ -149,11 +149,13 @@ \newlabel{mount:maximum-number-of-cache-entries}{{7.3.1}{23}{Maximum Number of Cache Entries}{subsection.7.3.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {7.3.2}Cache Flushing and Expiration}{23}{subsection.7.3.2}} \newlabel{mount:cache-flushing-and-expiration}{{7.3.2}{23}{Cache Flushing and Expiration}{subsection.7.3.2}{}} -\@writefile{toc}{\contentsline {section}{\numberline {7.4}Failure Modes}{23}{section.7.4}} -\newlabel{mount:failure-modes}{{7.4}{23}{Failure Modes}{section.7.4}{}} -\@writefile{toc}{\contentsline {section}{\numberline {7.5}Automatic Mounting}{24}{section.7.5}} -\newlabel{mount:automatic-mounting}{{7.5}{24}{Automatic Mounting}{section.7.5}{}} -\newlabel{mount:logcheck}{{7.5}{24}{Automatic Mounting}{section.7.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {7.4}NFS Support}{23}{section.7.4}} +\newlabel{mount:nfs-support}{{7.4}{23}{NFS Support}{section.7.4}{}} +\@writefile{toc}{\contentsline {section}{\numberline {7.5}Failure Modes}{23}{section.7.5}} +\newlabel{mount:failure-modes}{{7.5}{23}{Failure Modes}{section.7.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {7.6}Automatic Mounting}{24}{section.7.6}} +\newlabel{mount:automatic-mounting}{{7.6}{24}{Automatic Mounting}{section.7.6}{}} +\newlabel{mount:logcheck}{{7.6}{24}{Automatic Mounting}{section.7.6}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {8}Advanced S3QL Features}{25}{chapter.8}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} @@ -169,11 +171,11 @@ \newlabel{special:s3qlstat}{{8.2}{26}{Getting Statistics}{section.8.2}{}} \newlabel{special:getting-statistics}{{8.2}{26}{Getting Statistics}{section.8.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.3}Immutable Trees}{26}{section.8.3}} -\newlabel{special:immutable-trees}{{8.3}{26}{Immutable Trees}{section.8.3}{}} \newlabel{special:s3qllock}{{8.3}{26}{Immutable Trees}{section.8.3}{}} +\newlabel{special:immutable-trees}{{8.3}{26}{Immutable Trees}{section.8.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.4}Fast Recursive Removal}{27}{section.8.4}} -\newlabel{special:s3qlrm}{{8.4}{27}{Fast Recursive Removal}{section.8.4}{}} \newlabel{special:fast-recursive-removal}{{8.4}{27}{Fast Recursive Removal}{section.8.4}{}} +\newlabel{special:s3qlrm}{{8.4}{27}{Fast Recursive Removal}{section.8.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.5}Runtime Configuration}{27}{section.8.5}} \newlabel{special:runtime-configuration}{{8.5}{27}{Runtime Configuration}{section.8.5}{}} \newlabel{special:s3qlctrl}{{8.5}{27}{Runtime Configuration}{section.8.5}{}} @@ -219,8 +221,8 @@ \@writefile{toc}{\contentsline {section}{\numberline {12.5}expire\_backups.py}{38}{section.12.5}} \newlabel{contrib:expire-backups-py}{{12.5}{38}{expire\_backups.py}{section.12.5}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.6}remove\_objects.py}{39}{section.12.6}} -\newlabel{contrib:remove-objects}{{12.6}{39}{remove\_objects.py}{section.12.6}{}} \newlabel{contrib:remove-objects-py}{{12.6}{39}{remove\_objects.py}{section.12.6}{}} +\newlabel{contrib:remove-objects}{{12.6}{39}{remove\_objects.py}{section.12.6}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {13}Tips \& Tricks}{41}{chapter.13}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} @@ -228,8 +230,8 @@ \newlabel{tips::doc}{{13}{41}{Tips \& Tricks}{chapter.13}{}} \newlabel{tips:tips-tricks}{{13}{41}{Tips \& Tricks}{chapter.13}{}} \@writefile{toc}{\contentsline {section}{\numberline {13.1}SSH Backend}{41}{section.13.1}} -\newlabel{tips:ssh-tipp}{{13.1}{41}{SSH Backend}{section.13.1}{}} \newlabel{tips:ssh-backend}{{13.1}{41}{SSH Backend}{section.13.1}{}} +\newlabel{tips:ssh-tipp}{{13.1}{41}{SSH Backend}{section.13.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {13.2}Permanently mounted backup file system}{41}{section.13.2}} \newlabel{tips:permanently-mounted-backup-file-system}{{13.2}{41}{Permanently mounted backup file system}{section.13.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {13.3}Improving copy performance}{41}{section.13.3}} @@ -448,8 +450,8 @@ \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{impl_details::doc}{{17}{67}{Implementation Details}{chapter.17}{}} -\newlabel{impl_details:impl-details}{{17}{67}{Implementation Details}{chapter.17}{}} \newlabel{impl_details:implementation-details}{{17}{67}{Implementation Details}{chapter.17}{}} +\newlabel{impl_details:impl-details}{{17}{67}{Implementation Details}{chapter.17}{}} \@writefile{toc}{\contentsline {section}{\numberline {17.1}Metadata Storage}{67}{section.17.1}} \newlabel{impl_details:metadata-storage}{{17.1}{67}{Metadata Storage}{section.17.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {17.2}Data Storage}{67}{section.17.2}} diff --git a/doc/latex/manual.out b/doc/latex/manual.out index 29e23e3..1cf7a2e 100644 --- a/doc/latex/manual.out +++ b/doc/latex/manual.out @@ -30,51 +30,52 @@ \BOOKMARK [1][-]{section.7.1}{\376\377\000P\000e\000r\000m\000i\000s\000s\000i\000o\000n\000\040\000C\000h\000e\000c\000k\000i\000n\000g}{chapter.7}% 30 \BOOKMARK [1][-]{section.7.2}{\376\377\000C\000o\000m\000p\000r\000e\000s\000s\000i\000o\000n\000\040\000A\000l\000g\000o\000r\000i\000t\000h\000m\000s}{chapter.7}% 31 \BOOKMARK [1][-]{section.7.3}{\376\377\000N\000o\000t\000e\000s\000\040\000a\000b\000o\000u\000t\000\040\000C\000a\000c\000h\000i\000n\000g}{chapter.7}% 32 -\BOOKMARK [1][-]{section.7.4}{\376\377\000F\000a\000i\000l\000u\000r\000e\000\040\000M\000o\000d\000e\000s}{chapter.7}% 33 -\BOOKMARK [1][-]{section.7.5}{\376\377\000A\000u\000t\000o\000m\000a\000t\000i\000c\000\040\000M\000o\000u\000n\000t\000i\000n\000g}{chapter.7}% 34 -\BOOKMARK [0][-]{chapter.8}{\376\377\000A\000d\000v\000a\000n\000c\000e\000d\000\040\000S\0003\000Q\000L\000\040\000F\000e\000a\000t\000u\000r\000e\000s}{}% 35 -\BOOKMARK [1][-]{section.8.1}{\376\377\000S\000n\000a\000p\000s\000h\000o\000t\000t\000i\000n\000g\000\040\000a\000n\000d\000\040\000C\000o\000p\000y\000-\000o\000n\000-\000W\000r\000i\000t\000e}{chapter.8}% 36 -\BOOKMARK [1][-]{section.8.2}{\376\377\000G\000e\000t\000t\000i\000n\000g\000\040\000S\000t\000a\000t\000i\000s\000t\000i\000c\000s}{chapter.8}% 37 -\BOOKMARK [1][-]{section.8.3}{\376\377\000I\000m\000m\000u\000t\000a\000b\000l\000e\000\040\000T\000r\000e\000e\000s}{chapter.8}% 38 -\BOOKMARK [1][-]{section.8.4}{\376\377\000F\000a\000s\000t\000\040\000R\000e\000c\000u\000r\000s\000i\000v\000e\000\040\000R\000e\000m\000o\000v\000a\000l}{chapter.8}% 39 -\BOOKMARK [1][-]{section.8.5}{\376\377\000R\000u\000n\000t\000i\000m\000e\000\040\000C\000o\000n\000f\000i\000g\000u\000r\000a\000t\000i\000o\000n}{chapter.8}% 40 -\BOOKMARK [0][-]{chapter.9}{\376\377\000U\000n\000m\000o\000u\000n\000t\000i\000n\000g}{}% 41 -\BOOKMARK [0][-]{chapter.10}{\376\377\000C\000h\000e\000c\000k\000i\000n\000g\000\040\000f\000o\000r\000\040\000E\000r\000r\000o\000r\000s}{}% 42 -\BOOKMARK [1][-]{section.10.1}{\376\377\000C\000h\000e\000c\000k\000i\000n\000g\000\040\000a\000n\000d\000\040\000r\000e\000p\000a\000i\000r\000i\000n\000g\000\040\000i\000n\000t\000e\000r\000n\000a\000l\000\040\000f\000i\000l\000e\000\040\000s\000y\000s\000t\000e\000m\000\040\000e\000r\000r\000o\000r\000s}{chapter.10}% 43 -\BOOKMARK [1][-]{section.10.2}{\376\377\000D\000e\000t\000e\000c\000t\000i\000n\000g\000\040\000a\000n\000d\000\040\000h\000a\000n\000d\000l\000i\000n\000g\000\040\000b\000a\000c\000k\000e\000n\000d\000\040\000d\000a\000t\000a\000\040\000c\000o\000r\000r\000u\000p\000t\000i\000o\000n}{chapter.10}% 44 -\BOOKMARK [0][-]{chapter.11}{\376\377\000S\000t\000o\000r\000i\000n\000g\000\040\000B\000a\000c\000k\000e\000n\000d\000\040\000I\000n\000f\000o\000r\000m\000a\000t\000i\000o\000n}{}% 45 -\BOOKMARK [0][-]{chapter.12}{\376\377\000C\000o\000n\000t\000r\000i\000b\000u\000t\000e\000d\000\040\000P\000r\000o\000g\000r\000a\000m\000s}{}% 46 -\BOOKMARK [1][-]{section.12.1}{\376\377\000b\000e\000n\000c\000h\000m\000a\000r\000k\000.\000p\000y}{chapter.12}% 47 -\BOOKMARK [1][-]{section.12.2}{\376\377\000c\000l\000o\000n\000e\000\137\000f\000s\000.\000p\000y}{chapter.12}% 48 -\BOOKMARK [1][-]{section.12.3}{\376\377\000p\000c\000p\000.\000p\000y}{chapter.12}% 49 -\BOOKMARK [1][-]{section.12.4}{\376\377\000s\0003\000q\000l\000\137\000b\000a\000c\000k\000u\000p\000.\000s\000h}{chapter.12}% 50 -\BOOKMARK [1][-]{section.12.5}{\376\377\000e\000x\000p\000i\000r\000e\000\137\000b\000a\000c\000k\000u\000p\000s\000.\000p\000y}{chapter.12}% 51 -\BOOKMARK [1][-]{section.12.6}{\376\377\000r\000e\000m\000o\000v\000e\000\137\000o\000b\000j\000e\000c\000t\000s\000.\000p\000y}{chapter.12}% 52 -\BOOKMARK [0][-]{chapter.13}{\376\377\000T\000i\000p\000s\000\040\000\046\000\040\000T\000r\000i\000c\000k\000s}{}% 53 -\BOOKMARK [1][-]{section.13.1}{\376\377\000S\000S\000H\000\040\000B\000a\000c\000k\000e\000n\000d}{chapter.13}% 54 -\BOOKMARK [1][-]{section.13.2}{\376\377\000P\000e\000r\000m\000a\000n\000e\000n\000t\000l\000y\000\040\000m\000o\000u\000n\000t\000e\000d\000\040\000b\000a\000c\000k\000u\000p\000\040\000f\000i\000l\000e\000\040\000s\000y\000s\000t\000e\000m}{chapter.13}% 55 -\BOOKMARK [1][-]{section.13.3}{\376\377\000I\000m\000p\000r\000o\000v\000i\000n\000g\000\040\000c\000o\000p\000y\000\040\000p\000e\000r\000f\000o\000r\000m\000a\000n\000c\000e}{chapter.13}% 56 -\BOOKMARK [0][-]{chapter.14}{\376\377\000K\000n\000o\000w\000n\000\040\000I\000s\000s\000u\000e\000s}{}% 57 -\BOOKMARK [0][-]{chapter.15}{\376\377\000M\000a\000n\000p\000a\000g\000e\000s}{}% 58 -\BOOKMARK [1][-]{section.15.1}{\376\377\000T\000h\000e\000\040\000m\000k\000f\000s\000.\000s\0003\000q\000l\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 59 -\BOOKMARK [1][-]{section.15.2}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000a\000d\000m\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 60 -\BOOKMARK [1][-]{section.15.3}{\376\377\000T\000h\000e\000\040\000m\000o\000u\000n\000t\000.\000s\0003\000q\000l\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 61 -\BOOKMARK [1][-]{section.15.4}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000s\000t\000a\000t\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 62 -\BOOKMARK [1][-]{section.15.5}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000c\000t\000r\000l\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 63 -\BOOKMARK [1][-]{section.15.6}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000c\000p\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 64 -\BOOKMARK [1][-]{section.15.7}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000r\000m\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 65 -\BOOKMARK [1][-]{section.15.8}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000l\000o\000c\000k\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 66 -\BOOKMARK [1][-]{section.15.9}{\376\377\000T\000h\000e\000\040\000u\000m\000o\000u\000n\000t\000.\000s\0003\000q\000l\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 67 -\BOOKMARK [1][-]{section.15.10}{\376\377\000T\000h\000e\000\040\000f\000s\000c\000k\000.\000s\0003\000q\000l\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 68 -\BOOKMARK [1][-]{section.15.11}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000\137\000o\000a\000u\000t\000h\000\137\000c\000l\000i\000e\000n\000t\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 69 -\BOOKMARK [1][-]{section.15.12}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000\137\000v\000e\000r\000i\000f\000y\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 70 -\BOOKMARK [1][-]{section.15.13}{\376\377\000T\000h\000e\000\040\000p\000c\000p\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 71 -\BOOKMARK [1][-]{section.15.14}{\376\377\000T\000h\000e\000\040\000e\000x\000p\000i\000r\000e\000\137\000b\000a\000c\000k\000u\000p\000s\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 72 -\BOOKMARK [0][-]{chapter.16}{\376\377\000F\000u\000r\000t\000h\000e\000r\000\040\000R\000e\000s\000o\000u\000r\000c\000e\000s\000\040\000/\000\040\000G\000e\000t\000t\000i\000n\000g\000\040\000H\000e\000l\000p}{}% 73 -\BOOKMARK [0][-]{chapter.17}{\376\377\000I\000m\000p\000l\000e\000m\000e\000n\000t\000a\000t\000i\000o\000n\000\040\000D\000e\000t\000a\000i\000l\000s}{}% 74 -\BOOKMARK [1][-]{section.17.1}{\376\377\000M\000e\000t\000a\000d\000a\000t\000a\000\040\000S\000t\000o\000r\000a\000g\000e}{chapter.17}% 75 -\BOOKMARK [1][-]{section.17.2}{\376\377\000D\000a\000t\000a\000\040\000S\000t\000o\000r\000a\000g\000e}{chapter.17}% 76 -\BOOKMARK [1][-]{section.17.3}{\376\377\000D\000a\000t\000a\000\040\000D\000e\000-\000D\000u\000p\000l\000i\000c\000a\000t\000i\000o\000n}{chapter.17}% 77 -\BOOKMARK [1][-]{section.17.4}{\376\377\000C\000a\000c\000h\000i\000n\000g}{chapter.17}% 78 -\BOOKMARK [1][-]{section.17.5}{\376\377\000E\000v\000e\000n\000t\000u\000a\000l\000\040\000C\000o\000n\000s\000i\000s\000t\000e\000n\000c\000y\000\040\000H\000a\000n\000d\000l\000i\000n\000g}{chapter.17}% 79 -\BOOKMARK [1][-]{section.17.6}{\376\377\000E\000n\000c\000r\000y\000p\000t\000i\000o\000n}{chapter.17}% 80 +\BOOKMARK [1][-]{section.7.4}{\376\377\000N\000F\000S\000\040\000S\000u\000p\000p\000o\000r\000t}{chapter.7}% 33 +\BOOKMARK [1][-]{section.7.5}{\376\377\000F\000a\000i\000l\000u\000r\000e\000\040\000M\000o\000d\000e\000s}{chapter.7}% 34 +\BOOKMARK [1][-]{section.7.6}{\376\377\000A\000u\000t\000o\000m\000a\000t\000i\000c\000\040\000M\000o\000u\000n\000t\000i\000n\000g}{chapter.7}% 35 +\BOOKMARK [0][-]{chapter.8}{\376\377\000A\000d\000v\000a\000n\000c\000e\000d\000\040\000S\0003\000Q\000L\000\040\000F\000e\000a\000t\000u\000r\000e\000s}{}% 36 +\BOOKMARK [1][-]{section.8.1}{\376\377\000S\000n\000a\000p\000s\000h\000o\000t\000t\000i\000n\000g\000\040\000a\000n\000d\000\040\000C\000o\000p\000y\000-\000o\000n\000-\000W\000r\000i\000t\000e}{chapter.8}% 37 +\BOOKMARK [1][-]{section.8.2}{\376\377\000G\000e\000t\000t\000i\000n\000g\000\040\000S\000t\000a\000t\000i\000s\000t\000i\000c\000s}{chapter.8}% 38 +\BOOKMARK [1][-]{section.8.3}{\376\377\000I\000m\000m\000u\000t\000a\000b\000l\000e\000\040\000T\000r\000e\000e\000s}{chapter.8}% 39 +\BOOKMARK [1][-]{section.8.4}{\376\377\000F\000a\000s\000t\000\040\000R\000e\000c\000u\000r\000s\000i\000v\000e\000\040\000R\000e\000m\000o\000v\000a\000l}{chapter.8}% 40 +\BOOKMARK [1][-]{section.8.5}{\376\377\000R\000u\000n\000t\000i\000m\000e\000\040\000C\000o\000n\000f\000i\000g\000u\000r\000a\000t\000i\000o\000n}{chapter.8}% 41 +\BOOKMARK [0][-]{chapter.9}{\376\377\000U\000n\000m\000o\000u\000n\000t\000i\000n\000g}{}% 42 +\BOOKMARK [0][-]{chapter.10}{\376\377\000C\000h\000e\000c\000k\000i\000n\000g\000\040\000f\000o\000r\000\040\000E\000r\000r\000o\000r\000s}{}% 43 +\BOOKMARK [1][-]{section.10.1}{\376\377\000C\000h\000e\000c\000k\000i\000n\000g\000\040\000a\000n\000d\000\040\000r\000e\000p\000a\000i\000r\000i\000n\000g\000\040\000i\000n\000t\000e\000r\000n\000a\000l\000\040\000f\000i\000l\000e\000\040\000s\000y\000s\000t\000e\000m\000\040\000e\000r\000r\000o\000r\000s}{chapter.10}% 44 +\BOOKMARK [1][-]{section.10.2}{\376\377\000D\000e\000t\000e\000c\000t\000i\000n\000g\000\040\000a\000n\000d\000\040\000h\000a\000n\000d\000l\000i\000n\000g\000\040\000b\000a\000c\000k\000e\000n\000d\000\040\000d\000a\000t\000a\000\040\000c\000o\000r\000r\000u\000p\000t\000i\000o\000n}{chapter.10}% 45 +\BOOKMARK [0][-]{chapter.11}{\376\377\000S\000t\000o\000r\000i\000n\000g\000\040\000B\000a\000c\000k\000e\000n\000d\000\040\000I\000n\000f\000o\000r\000m\000a\000t\000i\000o\000n}{}% 46 +\BOOKMARK [0][-]{chapter.12}{\376\377\000C\000o\000n\000t\000r\000i\000b\000u\000t\000e\000d\000\040\000P\000r\000o\000g\000r\000a\000m\000s}{}% 47 +\BOOKMARK [1][-]{section.12.1}{\376\377\000b\000e\000n\000c\000h\000m\000a\000r\000k\000.\000p\000y}{chapter.12}% 48 +\BOOKMARK [1][-]{section.12.2}{\376\377\000c\000l\000o\000n\000e\000\137\000f\000s\000.\000p\000y}{chapter.12}% 49 +\BOOKMARK [1][-]{section.12.3}{\376\377\000p\000c\000p\000.\000p\000y}{chapter.12}% 50 +\BOOKMARK [1][-]{section.12.4}{\376\377\000s\0003\000q\000l\000\137\000b\000a\000c\000k\000u\000p\000.\000s\000h}{chapter.12}% 51 +\BOOKMARK [1][-]{section.12.5}{\376\377\000e\000x\000p\000i\000r\000e\000\137\000b\000a\000c\000k\000u\000p\000s\000.\000p\000y}{chapter.12}% 52 +\BOOKMARK [1][-]{section.12.6}{\376\377\000r\000e\000m\000o\000v\000e\000\137\000o\000b\000j\000e\000c\000t\000s\000.\000p\000y}{chapter.12}% 53 +\BOOKMARK [0][-]{chapter.13}{\376\377\000T\000i\000p\000s\000\040\000\046\000\040\000T\000r\000i\000c\000k\000s}{}% 54 +\BOOKMARK [1][-]{section.13.1}{\376\377\000S\000S\000H\000\040\000B\000a\000c\000k\000e\000n\000d}{chapter.13}% 55 +\BOOKMARK [1][-]{section.13.2}{\376\377\000P\000e\000r\000m\000a\000n\000e\000n\000t\000l\000y\000\040\000m\000o\000u\000n\000t\000e\000d\000\040\000b\000a\000c\000k\000u\000p\000\040\000f\000i\000l\000e\000\040\000s\000y\000s\000t\000e\000m}{chapter.13}% 56 +\BOOKMARK [1][-]{section.13.3}{\376\377\000I\000m\000p\000r\000o\000v\000i\000n\000g\000\040\000c\000o\000p\000y\000\040\000p\000e\000r\000f\000o\000r\000m\000a\000n\000c\000e}{chapter.13}% 57 +\BOOKMARK [0][-]{chapter.14}{\376\377\000K\000n\000o\000w\000n\000\040\000I\000s\000s\000u\000e\000s}{}% 58 +\BOOKMARK [0][-]{chapter.15}{\376\377\000M\000a\000n\000p\000a\000g\000e\000s}{}% 59 +\BOOKMARK [1][-]{section.15.1}{\376\377\000T\000h\000e\000\040\000m\000k\000f\000s\000.\000s\0003\000q\000l\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 60 +\BOOKMARK [1][-]{section.15.2}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000a\000d\000m\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 61 +\BOOKMARK [1][-]{section.15.3}{\376\377\000T\000h\000e\000\040\000m\000o\000u\000n\000t\000.\000s\0003\000q\000l\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 62 +\BOOKMARK [1][-]{section.15.4}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000s\000t\000a\000t\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 63 +\BOOKMARK [1][-]{section.15.5}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000c\000t\000r\000l\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 64 +\BOOKMARK [1][-]{section.15.6}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000c\000p\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 65 +\BOOKMARK [1][-]{section.15.7}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000r\000m\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 66 +\BOOKMARK [1][-]{section.15.8}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000l\000o\000c\000k\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 67 +\BOOKMARK [1][-]{section.15.9}{\376\377\000T\000h\000e\000\040\000u\000m\000o\000u\000n\000t\000.\000s\0003\000q\000l\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 68 +\BOOKMARK [1][-]{section.15.10}{\376\377\000T\000h\000e\000\040\000f\000s\000c\000k\000.\000s\0003\000q\000l\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 69 +\BOOKMARK [1][-]{section.15.11}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000\137\000o\000a\000u\000t\000h\000\137\000c\000l\000i\000e\000n\000t\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 70 +\BOOKMARK [1][-]{section.15.12}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000\137\000v\000e\000r\000i\000f\000y\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 71 +\BOOKMARK [1][-]{section.15.13}{\376\377\000T\000h\000e\000\040\000p\000c\000p\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 72 +\BOOKMARK [1][-]{section.15.14}{\376\377\000T\000h\000e\000\040\000e\000x\000p\000i\000r\000e\000\137\000b\000a\000c\000k\000u\000p\000s\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 73 +\BOOKMARK [0][-]{chapter.16}{\376\377\000F\000u\000r\000t\000h\000e\000r\000\040\000R\000e\000s\000o\000u\000r\000c\000e\000s\000\040\000/\000\040\000G\000e\000t\000t\000i\000n\000g\000\040\000H\000e\000l\000p}{}% 74 +\BOOKMARK [0][-]{chapter.17}{\376\377\000I\000m\000p\000l\000e\000m\000e\000n\000t\000a\000t\000i\000o\000n\000\040\000D\000e\000t\000a\000i\000l\000s}{}% 75 +\BOOKMARK [1][-]{section.17.1}{\376\377\000M\000e\000t\000a\000d\000a\000t\000a\000\040\000S\000t\000o\000r\000a\000g\000e}{chapter.17}% 76 +\BOOKMARK [1][-]{section.17.2}{\376\377\000D\000a\000t\000a\000\040\000S\000t\000o\000r\000a\000g\000e}{chapter.17}% 77 +\BOOKMARK [1][-]{section.17.3}{\376\377\000D\000a\000t\000a\000\040\000D\000e\000-\000D\000u\000p\000l\000i\000c\000a\000t\000i\000o\000n}{chapter.17}% 78 +\BOOKMARK [1][-]{section.17.4}{\376\377\000C\000a\000c\000h\000i\000n\000g}{chapter.17}% 79 +\BOOKMARK [1][-]{section.17.5}{\376\377\000E\000v\000e\000n\000t\000u\000a\000l\000\040\000C\000o\000n\000s\000i\000s\000t\000e\000n\000c\000y\000\040\000H\000a\000n\000d\000l\000i\000n\000g}{chapter.17}% 80 +\BOOKMARK [1][-]{section.17.6}{\376\377\000E\000n\000c\000r\000y\000p\000t\000i\000o\000n}{chapter.17}% 81 diff --git a/doc/latex/manual.tex b/doc/latex/manual.tex index f1c384a..45d8eb1 100644 --- a/doc/latex/manual.tex +++ b/doc/latex/manual.tex @@ -32,8 +32,8 @@ \title{S3QL Documentation} -\date{Jun 14, 2018} -\release{2.28} +\date{Jul 21, 2018} +\release{2.29} \author{Nikolaus Rath} \newcommand{\sphinxlogo}{} \renewcommand{\releasename}{Release} @@ -50,73 +50,73 @@ \PYG@it{\PYG@bf{\PYG@ff{#1}}}}}}} \def\PYG#1#2{\PYG@reset\PYG@toks#1+\relax+\PYG@do{#2}} -\expandafter\def\csname PYG@tok@nl\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.60,0.47,0.00}{##1}}} -\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@sc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} +\expandafter\def\csname PYG@tok@gs\endcsname{\let\PYG@bf=\textbf} \expandafter\def\csname PYG@tok@se\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@si\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.93,0.93,0.93}{\strut ##1}}} +\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} +\expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.20}{##1}}} +\expandafter\def\csname PYG@tok@s\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@il\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} +\expandafter\def\csname PYG@tok@cs\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.80,0.00,0.00}{##1}}} \expandafter\def\csname PYG@tok@vc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.40,0.60}{##1}}} -\expandafter\def\csname PYG@tok@ow\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@kr\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@c\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@gu\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}} \expandafter\def\csname PYG@tok@kc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@sa\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}} -\expandafter\def\csname PYG@tok@il\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} +\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.33,0.33,0.33}{##1}}} +\expandafter\def\csname PYG@tok@vg\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.87,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@sr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,1.00}{\strut ##1}}} \expandafter\def\csname PYG@tok@gp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}} +\expandafter\def\csname PYG@tok@kr\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@sc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} \expandafter\def\csname PYG@tok@no\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.40}{##1}}} +\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} +\expandafter\def\csname PYG@tok@nl\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.60,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@sa\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.73,0.00,0.40}{##1}}} +\expandafter\def\csname PYG@tok@dl\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.33,0.53}{##1}}} +\expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@vm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} +\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} +\expandafter\def\csname PYG@tok@bp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} +\expandafter\def\csname PYG@tok@mb\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@gh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}} +\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}} +\expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.73}{##1}}} +\expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.33,0.47,0.60}{##1}}} +\expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} \expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@ch\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.67,0.40,0.00}{##1}}} -\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit} -\expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}} \expandafter\def\csname PYG@tok@mo\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.27,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@cpf\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@mi\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} +\expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} \expandafter\def\csname PYG@tok@fm\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} +\expandafter\def\csname PYG@tok@cpf\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.67,0.40,0.00}{##1}}} +\expandafter\def\csname PYG@tok@c1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@ow\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}} \expandafter\def\csname PYG@tok@nt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.47,0.00}{##1}}} -\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} -\expandafter\def\csname PYG@tok@sr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,1.00}{\strut ##1}}} -\expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.73}{##1}}} -\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}} -\expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.80}{##1}}} -\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} -\expandafter\def\csname PYG@tok@vm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} -\expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} -\expandafter\def\csname PYG@tok@s\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@bp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} -\expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.73,0.00,0.40}{##1}}} -\expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.20}{##1}}} -\expandafter\def\csname PYG@tok@kt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.60}{##1}}} -\expandafter\def\csname PYG@tok@gu\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}} -\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}} -\expandafter\def\csname PYG@tok@mf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} \expandafter\def\csname PYG@tok@kp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.53}{##1}}} -\expandafter\def\csname PYG@tok@gs\endcsname{\let\PYG@bf=\textbf} -\expandafter\def\csname PYG@tok@vg\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.87,0.47,0.00}{##1}}} -\expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.33,0.33,0.33}{##1}}} -\expandafter\def\csname PYG@tok@dl\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.33,0.53}{##1}}} -\expandafter\def\csname PYG@tok@mi\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} +\expandafter\def\csname PYG@tok@si\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.93,0.93,0.93}{\strut ##1}}} \expandafter\def\csname PYG@tok@ne\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@cs\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.80,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.33,0.47,0.60}{##1}}} -\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@gh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}} +\expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} +\expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.80}{##1}}} +\expandafter\def\csname PYG@tok@ch\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} \expandafter\def\csname PYG@tok@err\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.67,0.67}{\strut ##1}}} -\expandafter\def\csname PYG@tok@sd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.27,0.13}{##1}}} -\expandafter\def\csname PYG@tok@c\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@c1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@mb\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} \expandafter\def\csname PYG@tok@sh\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit} +\expandafter\def\csname PYG@tok@kt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.60}{##1}}} +\expandafter\def\csname PYG@tok@mf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}} +\expandafter\def\csname PYG@tok@sd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.27,0.13}{##1}}} \def\PYGZbs{\char`\\} \def\PYGZus{\char`\_} @@ -256,12 +256,12 @@ The S3QL source code is available both on \href{https://github.com/s3ql/main}{Gi \chapter{Installation} -\label{installation::doc}\label{installation:installation}\label{installation:github} +\label{installation::doc}\label{installation:github}\label{installation:installation} S3QL depends on several other programs and libraries that have to be installed first. The best method to satisfy these dependencies depends on your distribution. -The following instructions are for S3QL 2.28 and should be +The following instructions are for S3QL 2.29 and should be applicable to any system. The \href{https://bitbucket.org/nikratio/s3ql/wiki/Home}{S3QL Wiki} contains \href{https://bitbucket.org/nikratio/s3ql/wiki/Installation}{additional help} help for specific distributions and operating systems. Note, however, that @@ -343,7 +343,7 @@ installed version if the module is installed. \section{Installing S3QL} -\label{installation:installing-s3ql}\label{installation:inst-s3ql} +\label{installation:inst-s3ql}\label{installation:installing-s3ql} To build and install S3QL itself, proceed as follows: \begin{enumerate} \item {} @@ -463,7 +463,7 @@ being skipped by passing the \sphinxcode{-rs} argument to \chapter{Storage Backends} -\label{backends::doc}\label{backends:id1}\label{backends:storage-backends}\label{backends:sphinx} +\label{backends::doc}\label{backends:storage-backends}\label{backends:sphinx}\label{backends:id1} S3QL supports different \emph{backends} to store data at different service providers and using different protocols. A \emph{storage url} specifies a backend together with some backend-specific information and uniquely @@ -568,7 +568,7 @@ TCP connection is closed and re-established (default: 20 seconds). \section{Amazon S3} -\label{backends:amazon-s3}\label{backends:google-storage-manager} +\label{backends:google-storage-manager}\label{backends:amazon-s3} \href{http://aws.amazon.com/s3}{Amazon S3} is the online storage service offered by \href{http://aws.amazon.com/}{Amazon Web Services (AWS)}. To use the S3 backend, you first need to sign up for an AWS account. The @@ -670,7 +670,7 @@ even later in time due to the data de-duplication feature of S3QL (see \section{OpenStack/Swift} -\label{backends:openstack-swift}\label{backends:openstack-backend} +\label{backends:openstack-backend}\label{backends:openstack-swift} \href{http://www.openstack.org/}{OpenStack} is an open-source cloud server application suite. \href{http://openstack.org/projects/storage/}{Swift} is the cloud storage module of OpenStack. Swift/OpenStack storage is offered by many different companies. @@ -872,7 +872,7 @@ option should only be used if you are certain that your storage server only returns \sphinxcode{200 OK} when the copy operation has been completely and successfully carried out. Using this option may be neccessary if your storage server does not return a valid response -body for a succesfull copy operation. +body for a successful copy operation. \end{fulllineitems} @@ -905,7 +905,7 @@ The local backend does not accept any backend options. \chapter{Important Rules to Avoid Losing Data} -\label{durability:sshfs}\label{durability:durability}\label{durability::doc}\label{durability:important-rules-to-avoid-losing-data} +\label{durability::doc}\label{durability:important-rules-to-avoid-losing-data}\label{durability:sshfs}\label{durability:durability} Most S3QL backends store data in distributed storage systems. These systems differ from a traditional, local hard disk in several important ways. In order to avoid losing data, this section should be @@ -1096,7 +1096,7 @@ rise up to hours (\href{http://forums.aws.amazon.com/message.jspa?messageID=3847 \section{Data Durability} -\label{durability:backend-reliability}\label{durability:data-durability} +\label{durability:data-durability}\label{durability:backend-reliability} The durability of a storage service a measure of the average probability of a storage object to become corrupted over time. The lower the chance of data loss, the higher the durability. Storage @@ -1193,9 +1193,6 @@ This command accepts the following options: \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: \sphinxcode{\textasciitilde{}/.s3ql)} -\item [-{-}authfile \textless{}path\textgreater{}] -Read authentication credentials from this file -(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages @@ -1212,6 +1209,9 @@ Backend specific options (separate by commas). See backend documentation for available options. \item [-{-}version] just print program version and exit +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-L \textless{}name\textgreater{}] Filesystem label \item [-{-}max-obj-size \textless{}size\textgreater{}] @@ -1274,6 +1274,9 @@ The \textbf{\texttt{s3qladm}} accepts the following general options, no matter what specific action is being invoked: \begin{quote} \begin{optionlist}{3cm} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages @@ -1292,9 +1295,6 @@ daemon. Anything else will be interpreted as a file name. Log files will be rotated when they reach 1 MiB, and at most 5 old log files will be kept. Default: \sphinxcode{None} -\item [-{-}authfile \textless{}path\textgreater{}] -Read authentication credentials from this file -(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}backend-options \textless{}options\textgreater{}] Backend specific options (separate by commas). See backend documentation for available options. @@ -1319,6 +1319,16 @@ subcommand: \PYG{l}{s3qladm passphrase }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} \end{Verbatim} +That this will not actually re-encrypt all the stored data with the +new passphrase. Rather, S3QL generates an internal ``master key'' when +the file system is created and uses this key to encrypt all stored +data. The user-supplied passphrase is used to encrypt/decrypt only the +master key, and can therefore be changed. This means, however, that if +a passphrase has been disclosed to someone untrusted, changing it +afterwards will \textbf{not revoke access to people who had access to the +old passphrase} (because they could have decrypted the master key and +retained a copy). + \section{Upgrading the file system} \label{adm:upgrading-the-file-system} @@ -1402,9 +1412,6 @@ and at most 5 old log files will be kept. Default: \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: \sphinxcode{\textasciitilde{}/.s3ql)} -\item [-{-}authfile \textless{}path\textgreater{}] -Read authentication credentials from this file -(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages @@ -1421,6 +1428,9 @@ Backend specific options (separate by commas). See backend documentation for available options. \item [-{-}version] just print program version and exit +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}cachesize \textless{}size\textgreater{}] Cache size in KiB (default: autodetect). \item [-{-}max-cache-entries \textless{}num\textgreater{}] @@ -1541,6 +1551,26 @@ when the maximum cache size is reached. S3QL always expires the least recently used blocks first. +\section{NFS Support} +\label{mount:nfs-support} +S3QL filesystems can be exported over NFS. The \sphinxcode{-{-}nfs} option +is recommended to improve performance when NFS is used, but no harm +will occur when it is not specified. + +NFS supports persistence of client mounts across server restarts. This +means that if a client has mounted an S3QL file system over NFS, the +server may unmount and remount the S3QL filesystem (or even reboot) +without the client being affected beyond temporarily becoming +unavailable. This poses several challenges, but is supported by S3QL +as long as no \sphinxcode{fsck.s3ql} operation is run: + +\begin{notice}{warning}{Warning:} +If \sphinxcode{fsck.s3ql} modifies a file system in any way, all NFS +clients must unmount and re-mount the NFS share before the +S3QL file system is re-mounted on the server. +\end{notice} + + \section{Failure Modes} \label{mount:failure-modes} Once an S3QL file system has been mounted, there is a multitude of @@ -1730,7 +1760,7 @@ For a full list of available options, run \sphinxcode{s3qlstat -{-}help}. \section{Immutable Trees} -\label{special:immutable-trees}\label{special:s3qllock} +\label{special:s3qllock}\label{special:immutable-trees} The command \textbf{\texttt{s3qllock}} can be used to make a directory tree immutable. Immutable trees can no longer be changed in any way whatsoever. You can not add new files or directories and you can not @@ -1777,7 +1807,7 @@ changed after they have been made immutable. \section{Fast Recursive Removal} -\label{special:s3qlrm}\label{special:fast-recursive-removal} +\label{special:fast-recursive-removal}\label{special:s3qlrm} The \sphinxcode{s3qlrm} command can be used to recursively delete files and directories on an S3QL file system. Although \sphinxcode{s3qlrm} is faster than using e.g. \sphinxcode{rm -r}, the main reason for its existence is that it @@ -1924,9 +1954,6 @@ and at most 5 old log files will be kept. Default: \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: \sphinxcode{\textasciitilde{}/.s3ql)} -\item [-{-}authfile \textless{}path\textgreater{}] -Read authentication credentials from this file -(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages @@ -1943,6 +1970,9 @@ Backend specific options (separate by commas). See backend documentation for available options. \item [-{-}version] just print program version and exit +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}batch] If user input is required, exit without prompting. \item [-{-}force] @@ -2010,12 +2040,12 @@ just print program version and exit \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: \sphinxcode{\textasciitilde{}/.s3ql)} -\item [-{-}authfile \textless{}path\textgreater{}] -Read authentication credentials from this file -(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}backend-options \textless{}options\textgreater{}] Backend specific options (separate by commas). See backend documentation for available options. +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}missing-file \textless{}name\textgreater{}] File to store keys of missing objects. \item [-{-}corrupted-file \textless{}name\textgreater{}] @@ -2261,7 +2291,7 @@ For a full list of available options, run \textbf{\texttt{expire\_backups.py \section{remove\_objects.py} -\label{contrib:remove-objects}\label{contrib:remove-objects-py} +\label{contrib:remove-objects-py}\label{contrib:remove-objects} \textbf{\texttt{remove\_objects.py}} is a program to remove a list of objects from a storage backend. Since it acts on the backend-level, the backend need not contain an S3QL file system. @@ -2271,7 +2301,7 @@ backend need not contain an S3QL file system. \label{tips::doc}\label{tips:tips-tricks} \section{SSH Backend} -\label{tips:ssh-tipp}\label{tips:ssh-backend} +\label{tips:ssh-backend}\label{tips:ssh-tipp} By combining S3QL's local backend with \href{http://fuse.sourceforge.net/sshfs.html}{sshfs}, it is possible to store an S3QL file system on arbitrary SSH servers: first mount the remote target directory into the local filesystem, @@ -2513,9 +2543,6 @@ The \textbf{\texttt{mkfs.s3ql}} command accepts the following options. \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: \sphinxcode{\textasciitilde{}/.s3ql)} -\item [-{-}authfile \textless{}path\textgreater{}] -Read authentication credentials from this file -(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages @@ -2532,6 +2559,9 @@ Backend specific options (separate by commas). See backend documentation for available options. \item [-{-}version] just print program version and exit +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-L \textless{}name\textgreater{}] Filesystem label \item [-{-}max-obj-size \textless{}size\textgreater{}] @@ -2627,6 +2657,9 @@ Guide should be consulted for a description of the available backends. The \textbf{\texttt{s3qladm}} command accepts the following options. \begin{quote} \begin{optionlist}{3cm} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages @@ -2645,9 +2678,6 @@ daemon. Anything else will be interpreted as a file name. Log files will be rotated when they reach 1 MiB, and at most 5 old log files will be kept. Default: \sphinxcode{None} -\item [-{-}authfile \textless{}path\textgreater{}] -Read authentication credentials from this file -(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}backend-options \textless{}options\textgreater{}] Backend specific options (separate by commas). See backend documentation for available options. @@ -2777,9 +2807,6 @@ and at most 5 old log files will be kept. Default: \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: \sphinxcode{\textasciitilde{}/.s3ql)} -\item [-{-}authfile \textless{}path\textgreater{}] -Read authentication credentials from this file -(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages @@ -2796,6 +2823,9 @@ Backend specific options (separate by commas). See backend documentation for available options. \item [-{-}version] just print program version and exit +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}cachesize \textless{}size\textgreater{}] Cache size in KiB (default: autodetect). \item [-{-}max-cache-entries \textless{}num\textgreater{}] @@ -3526,9 +3556,6 @@ and at most 5 old log files will be kept. Default: \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: \sphinxcode{\textasciitilde{}/.s3ql)} -\item [-{-}authfile \textless{}path\textgreater{}] -Read authentication credentials from this file -(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages @@ -3545,6 +3572,9 @@ Backend specific options (separate by commas). See backend documentation for available options. \item [-{-}version] just print program version and exit +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}batch] If user input is required, exit without prompting. \item [-{-}force] @@ -3762,12 +3792,12 @@ just print program version and exit \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: \sphinxcode{\textasciitilde{}/.s3ql)} -\item [-{-}authfile \textless{}path\textgreater{}] -Read authentication credentials from this file -(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}backend-options \textless{}options\textgreater{}] Backend specific options (separate by commas). See backend documentation for available options. +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}missing-file \textless{}name\textgreater{}] File to store keys of missing objects. \item [-{-}corrupted-file \textless{}name\textgreater{}] @@ -4089,7 +4119,7 @@ Please report any bugs you may encounter in the \href{https://bitbucket.org/nikr \chapter{Implementation Details} -\label{impl_details::doc}\label{impl_details:impl-details}\label{impl_details:implementation-details} +\label{impl_details::doc}\label{impl_details:implementation-details}\label{impl_details:impl-details} This section provides some background information on how S3QL works internally. Reading this section is not necessary to use S3QL. diff --git a/doc/latex/manual.toc b/doc/latex/manual.toc index c8e8a9f..8792d8b 100644 --- a/doc/latex/manual.toc +++ b/doc/latex/manual.toc @@ -33,8 +33,9 @@ \contentsline {section}{\numberline {7.3}Notes about Caching}{23}{section.7.3} \contentsline {subsection}{\numberline {7.3.1}Maximum Number of Cache Entries}{23}{subsection.7.3.1} \contentsline {subsection}{\numberline {7.3.2}Cache Flushing and Expiration}{23}{subsection.7.3.2} -\contentsline {section}{\numberline {7.4}Failure Modes}{23}{section.7.4} -\contentsline {section}{\numberline {7.5}Automatic Mounting}{24}{section.7.5} +\contentsline {section}{\numberline {7.4}NFS Support}{23}{section.7.4} +\contentsline {section}{\numberline {7.5}Failure Modes}{23}{section.7.5} +\contentsline {section}{\numberline {7.6}Automatic Mounting}{24}{section.7.6} \contentsline {chapter}{\numberline {8}Advanced S3QL Features}{25}{chapter.8} \contentsline {section}{\numberline {8.1}Snapshotting and Copy-on-Write}{25}{section.8.1} \contentsline {subsection}{\numberline {8.1.1}Snapshotting vs Hardlinking}{25}{subsection.8.1.1} diff --git a/doc/man/fsck.s3ql.1 b/doc/man/fsck.s3ql.1 index d1175e1..2c72ac7 100644 --- a/doc/man/fsck.s3ql.1 +++ b/doc/man/fsck.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "FSCK.S3QL" "1" "Jun 14, 2018" "2.28" "S3QL" +.TH "FSCK.S3QL" "1" "Jul 21, 2018" "2.29" "S3QL" .SH NAME fsck.s3ql \- Check an S3QL file system for errors . @@ -70,10 +70,6 @@ and at most 5 old log files will be kept. Default: Store cached data in this directory (default: \fB~/.s3ql)\fP .TP -.BI \-\-authfile \ <path> -Read authentication credentials from this file -(default: \fB~/.s3ql/authinfo2)\fP -.TP .BI \-\-debug\-modules \ <modules> Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages @@ -95,6 +91,10 @@ backend documentation for available options. .B \-\-version just print program version and exit .TP +.BI \-\-authfile \ <path> +Read authentication credentials from this file +(default: \fB~/.s3ql/authinfo2)\fP +.TP .B \-\-batch If user input is required, exit without prompting. .TP diff --git a/doc/man/mkfs.s3ql.1 b/doc/man/mkfs.s3ql.1 index 9cc14f6..848e706 100644 --- a/doc/man/mkfs.s3ql.1 +++ b/doc/man/mkfs.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "MKFS.S3QL" "1" "Jun 14, 2018" "2.28" "S3QL" +.TH "MKFS.S3QL" "1" "Jul 21, 2018" "2.29" "S3QL" .SH NAME mkfs.s3ql \- Create an S3QL file system . @@ -67,10 +67,6 @@ The \fBmkfs.s3ql\fP command accepts the following options. Store cached data in this directory (default: \fB~/.s3ql)\fP .TP -.BI \-\-authfile \ <path> -Read authentication credentials from this file -(default: \fB~/.s3ql/authinfo2)\fP -.TP .BI \-\-debug\-modules \ <modules> Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages @@ -92,6 +88,10 @@ backend documentation for available options. .B \-\-version just print program version and exit .TP +.BI \-\-authfile \ <path> +Read authentication credentials from this file +(default: \fB~/.s3ql/authinfo2)\fP +.TP .BI \-L \ <name> Filesystem label .TP diff --git a/doc/man/mount.s3ql.1 b/doc/man/mount.s3ql.1 index 3b245df..e466fec 100644 --- a/doc/man/mount.s3ql.1 +++ b/doc/man/mount.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "MOUNT.S3QL" "1" "Jun 14, 2018" "2.28" "S3QL" +.TH "MOUNT.S3QL" "1" "Jul 21, 2018" "2.29" "S3QL" .SH NAME mount.s3ql \- Mount an S3QL file system . @@ -70,10 +70,6 @@ and at most 5 old log files will be kept. Default: Store cached data in this directory (default: \fB~/.s3ql)\fP .TP -.BI \-\-authfile \ <path> -Read authentication credentials from this file -(default: \fB~/.s3ql/authinfo2)\fP -.TP .BI \-\-debug\-modules \ <modules> Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages @@ -95,6 +91,10 @@ backend documentation for available options. .B \-\-version just print program version and exit .TP +.BI \-\-authfile \ <path> +Read authentication credentials from this file +(default: \fB~/.s3ql/authinfo2)\fP +.TP .BI \-\-cachesize \ <size> Cache size in KiB (default: autodetect). .TP diff --git a/doc/man/s3ql_oauth_client.1 b/doc/man/s3ql_oauth_client.1 index 5af4f9a..33c0e46 100644 --- a/doc/man/s3ql_oauth_client.1 +++ b/doc/man/s3ql_oauth_client.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QL_OAUTH_CLIENT" "1" "Jun 14, 2018" "2.28" "S3QL" +.TH "S3QL_OAUTH_CLIENT" "1" "Jul 21, 2018" "2.29" "S3QL" .SH NAME s3ql_oauth_client \- Obtain Google Storage OAuth2 tokens . diff --git a/doc/man/s3ql_verify.1 b/doc/man/s3ql_verify.1 index b5e82de..f83c41e 100644 --- a/doc/man/s3ql_verify.1 +++ b/doc/man/s3ql_verify.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QL_VERIFY" "1" "Jun 14, 2018" "2.28" "S3QL" +.TH "S3QL_VERIFY" "1" "Jul 21, 2018" "2.29" "S3QL" .SH NAME s3ql_verify \- Verify data in an S3QL file system . @@ -83,14 +83,14 @@ just print program version and exit Store cached data in this directory (default: \fB~/.s3ql)\fP .TP -.BI \-\-authfile \ <path> -Read authentication credentials from this file -(default: \fB~/.s3ql/authinfo2)\fP -.TP .BI \-\-backend\-options \ <options> Backend specific options (separate by commas). See backend documentation for available options. .TP +.BI \-\-authfile \ <path> +Read authentication credentials from this file +(default: \fB~/.s3ql/authinfo2)\fP +.TP .BI \-\-missing\-file \ <name> File to store keys of missing objects. .TP diff --git a/doc/man/s3qladm.1 b/doc/man/s3qladm.1 index 7aa23bd..0c8161c 100644 --- a/doc/man/s3qladm.1 +++ b/doc/man/s3qladm.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLADM" "1" "Jun 14, 2018" "2.28" "S3QL" +.TH "S3QLADM" "1" "Jul 21, 2018" "2.29" "S3QL" .SH NAME s3qladm \- Manage S3QL file systems . @@ -63,6 +63,10 @@ The \fBs3qladm\fP command accepts the following options. .INDENT 3.5 .INDENT 0.0 .TP +.BI \-\-authfile \ <path> +Read authentication credentials from this file +(default: \fB~/.s3ql/authinfo2)\fP +.TP .BI \-\-debug\-modules \ <modules> Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages @@ -85,10 +89,6 @@ name. Log files will be rotated when they reach 1 MiB, and at most 5 old log files will be kept. Default: \fBNone\fP .TP -.BI \-\-authfile \ <path> -Read authentication credentials from this file -(default: \fB~/.s3ql/authinfo2)\fP -.TP .BI \-\-backend\-options \ <options> Backend specific options (separate by commas). See backend documentation for available options. diff --git a/doc/man/s3qlcp.1 b/doc/man/s3qlcp.1 index fcac131..909e19f 100644 --- a/doc/man/s3qlcp.1 +++ b/doc/man/s3qlcp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLCP" "1" "Jun 14, 2018" "2.28" "S3QL" +.TH "S3QLCP" "1" "Jul 21, 2018" "2.29" "S3QL" .SH NAME s3qlcp \- Copy-on-write replication on S3QL file systems . diff --git a/doc/man/s3qlctrl.1 b/doc/man/s3qlctrl.1 index 6585cfc..8c5a63c 100644 --- a/doc/man/s3qlctrl.1 +++ b/doc/man/s3qlctrl.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLCTRL" "1" "Jun 14, 2018" "2.28" "S3QL" +.TH "S3QLCTRL" "1" "Jul 21, 2018" "2.29" "S3QL" .SH NAME s3qlctrl \- Control a mounted S3QL file system . diff --git a/doc/man/s3qllock.1 b/doc/man/s3qllock.1 index 93e4bbc..6297927 100644 --- a/doc/man/s3qllock.1 +++ b/doc/man/s3qllock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLLOCK" "1" "Jun 14, 2018" "2.28" "S3QL" +.TH "S3QLLOCK" "1" "Jul 21, 2018" "2.29" "S3QL" .SH NAME s3qllock \- Make trees on an S3QL file system immutable . diff --git a/doc/man/s3qlrm.1 b/doc/man/s3qlrm.1 index 0106558..d1d57e4 100644 --- a/doc/man/s3qlrm.1 +++ b/doc/man/s3qlrm.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLRM" "1" "Jun 14, 2018" "2.28" "S3QL" +.TH "S3QLRM" "1" "Jul 21, 2018" "2.29" "S3QL" .SH NAME s3qlrm \- Fast tree removal on S3QL file systems . diff --git a/doc/man/s3qlstat.1 b/doc/man/s3qlstat.1 index 50c6877..88ef1e2 100644 --- a/doc/man/s3qlstat.1 +++ b/doc/man/s3qlstat.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLSTAT" "1" "Jun 14, 2018" "2.28" "S3QL" +.TH "S3QLSTAT" "1" "Jul 21, 2018" "2.29" "S3QL" .SH NAME s3qlstat \- Gather S3QL file system statistics . diff --git a/doc/man/umount.s3ql.1 b/doc/man/umount.s3ql.1 index 3f618b9..70f4fb4 100644 --- a/doc/man/umount.s3ql.1 +++ b/doc/man/umount.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "UMOUNT.S3QL" "1" "Jun 14, 2018" "2.28" "S3QL" +.TH "UMOUNT.S3QL" "1" "Jul 21, 2018" "2.29" "S3QL" .SH NAME umount.s3ql \- Unmount an S3QL file system . diff --git a/doc/manual.pdf b/doc/manual.pdf Binary files differindex b7cfe47..0f305ca 100644 --- a/doc/manual.pdf +++ b/doc/manual.pdf diff --git a/rst/adm.rst b/rst/adm.rst index 811eb0d..613e5b8 100644 --- a/rst/adm.rst +++ b/rst/adm.rst @@ -30,6 +30,16 @@ subcommand:: s3qladm passphrase <storage url> +That this will not actually re-encrypt all the stored data with the +new passphrase. Rather, S3QL generates an internal "master key" when +the file system is created and uses this key to encrypt all stored +data. The user-supplied passphrase is used to encrypt/decrypt only the +master key, and can therefore be changed. This means, however, that if +a passphrase has been disclosed to someone untrusted, changing it +afterwards will **not revoke access to people who had access to the +old passphrase** (because they could have decrypted the master key and +retained a copy). + Upgrading the file system ------------------------- diff --git a/rst/backends.rst b/rst/backends.rst index 8f7719f..0bd7a57 100644 --- a/rst/backends.rst +++ b/rst/backends.rst @@ -349,7 +349,7 @@ The S3 compatible backend accepts the following backend options: server only returns ``200 OK`` when the copy operation has been completely and successfully carried out. Using this option may be neccessary if your storage server does not return a valid response - body for a succesfull copy operation. + body for a successful copy operation. .. _`S3 COPY API`: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html .. __: https://doc.s3.amazonaws.com/proposals/copy.html diff --git a/rst/mount.rst b/rst/mount.rst index ed275f3..112a748 100644 --- a/rst/mount.rst +++ b/rst/mount.rst @@ -103,6 +103,26 @@ Cache expiration (i.e., removal of blocks from the cache) is only done when the maximum cache size is reached. S3QL always expires the least recently used blocks first. +NFS Support +=========== + +S3QL filesystems can be exported over NFS. The :cmdopt:`--nfs` option +is recommended to improve performance when NFS is used, but no harm +will occur when it is not specified. + +NFS supports persistence of client mounts across server restarts. This +means that if a client has mounted an S3QL file system over NFS, the +server may unmount and remount the S3QL filesystem (or even reboot) +without the client being affected beyond temporarily becoming +unavailable. This poses several challenges, but is supported by S3QL +as long as no `fsck.s3ql` operation is run: + +.. WARNING:: + + If `fsck.s3ql` modifies a file system in any way, all NFS + clients must unmount and re-mount the NFS share before the + S3QL file system is re-mounted on the server. + Failure Modes ============= @@ -112,21 +112,21 @@ def main(): compile_args = ['-Wall', '-Wextra', '-Wconversion', '-Wsign-compare'] - # Value-changing conversions should always be explicit. - compile_args.append('-Werror=conversion') - - # Note that (i > -1) is false if i is unsigned (-1 will be converted to - # a large positive value). We certainly don't want to do this by - # accident. - compile_args.append('-Werror=sign-compare') - # Enable all fatal warnings only when compiling from Mercurial tip. # (otherwise we break forward compatibility because compilation with newer # compiler may fail if additional warnings are added) if DEVELOPER_MODE: if os.environ.get('CI') != 'true': compile_args.append('-Werror') - compile_args.append('-Wfatal-errors') + + # Value-changing conversions should always be explicit. + compile_args.append('-Werror=conversion') + + # Note that (i > -1) is false if i is unsigned (-1 will be converted to + # a large positive value). We certainly don't want to do this by + # accident. + compile_args.append('-Werror=sign-compare') + compile_args.append('-Wno-unused-function') required_pkgs = ['apsw >= 3.7.0', diff --git a/src/s3ql.egg-info/PKG-INFO b/src/s3ql.egg-info/PKG-INFO index 70d2335..ae42fb4 100644 --- a/src/s3ql.egg-info/PKG-INFO +++ b/src/s3ql.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: s3ql -Version: 2.28 +Version: 2.29 Summary: a full-featured file system for online data storage Home-page: https://bitbucket.org/nikratio/s3ql/ Author: Nikolaus Rath @@ -144,7 +144,7 @@ Description: .. `s3ql+subscribe@googlegroups.com <mailto:s3ql+subscribe@googlegroups.com>`_. - Please report any bugs you may encounter in the `Bitbucket Issue Tracker`_. + Please report any bugs you may encounter in the `GitHub Issue Tracker`_. Contributing ============ @@ -161,7 +161,7 @@ Description: .. .. _`Installation Instructions`: https://bitbucket.org/nikratio/s3ql/wiki/Installation .. _`S3QL FAQ`: https://bitbucket.org/nikratio/s3ql/wiki/FAQ .. _`S3QL Mailing List`: http://groups.google.com/group/s3ql - .. _`Bitbucket Issue Tracker`: https://bitbucket.org/nikratio/s3ql/issues + .. _`GitHub Issue Tracker`: https://github.com/s3ql/s3ql/issues .. _BitBucket: https://bitbucket.org/nikratio/s3ql/ .. _GitHub: https://github.com/s3ql/main .. _`Rath Consulting`: http://www.rath-consulting.biz/ diff --git a/src/s3ql/__init__.py b/src/s3ql/__init__.py index 2e19b9a..185766f 100644 --- a/src/s3ql/__init__.py +++ b/src/s3ql/__init__.py @@ -22,7 +22,7 @@ __all__ = [ 'adm', 'backends', 'block_cache', 'common', 'calc_mro', 'REV_VER_MAP', 'RELEASE', 'BUFSIZE', 'CTRL_NAME', 'CTRL_INODE' ] -VERSION = '2.28' +VERSION = '2.29' RELEASE = '%s' % VERSION # TODO: On next revision bump, remove upgrade code from backend/comprenc.py and diff --git a/src/s3ql/adm.py b/src/s3ql/adm.py index 99fac6f..5544cb6 100644 --- a/src/s3ql/adm.py +++ b/src/s3ql/adm.py @@ -44,7 +44,6 @@ def parse_args(args): pparser = ArgumentParser(add_help=False, epilog=textwrap.dedent('''\ Hint: run `%(prog)s --help` to get help on other available actions and optional arguments that can be used with all actions.''')) - pparser.add_storage_url() subparsers = parser.add_subparsers(metavar='<action>', dest='action', help='may be either of') @@ -61,10 +60,10 @@ def parse_args(args): sparser.add_argument("--threads", type=int, default=20, help='Number of threads to use') + parser.add_storage_url() parser.add_debug() parser.add_quiet() parser.add_log() - parser.add_authfile() parser.add_backend_options() parser.add_cachedir() parser.add_version() @@ -155,6 +154,11 @@ def change_passphrase(backend): data_pw = backend.passphrase + print(textwrap.dedent('''\ + NOTE: If your password has been compromised already, then changing + it WILL NOT PROTECT YOUR DATA, because an attacker may have already + retrieved the master key. + ''')) if sys.stdin.isatty(): wrap_pw = getpass("Enter new encryption password: ") if not wrap_pw == getpass("Confirm new encryption password: "): diff --git a/src/s3ql/block_cache.py b/src/s3ql/block_cache.py index 4678260..a8ebb64 100644 --- a/src/s3ql/block_cache.py +++ b/src/s3ql/block_cache.py @@ -321,14 +321,14 @@ class BlockCache(object): else: self.mlock.release(obj_id, noerror=noerror) - def _lock_entry(self, inode, blockno, release_global=False): + def _lock_entry(self, inode, blockno, release_global=False, timeout=None): '''Acquire lock on cache entry''' if release_global: with lock_released: - self.mlock.acquire((inode, blockno)) + return self.mlock.acquire((inode, blockno), timeout=timeout) else: - self.mlock.acquire((inode, blockno)) + return self.mlock.acquire((inode, blockno), timeout=timeout) def _unlock_entry(self, inode, blockno, release_global=False, noerror=False): @@ -887,30 +887,39 @@ class BlockCache(object): if end_no is None: end_no = start_no + 1 - - for blockno in range(start_no, end_no): - self._lock_entry(inode, blockno, release_global=True) - try: - if (inode, blockno) in self.cache: - log.debug('removing from cache') - self.cache.remove((inode, blockno)) - - try: - block_id = self.db.get_val('SELECT block_id FROM inode_blocks ' - 'WHERE inode=? AND blockno=?', (inode, blockno)) - except NoSuchRowError: - log.debug('block not in db') + blocknos = set(range(start_no, end_no)) + + # First do an opportunistic pass and remove everything where we can + # immediately get a lock. This is important when removing a file right + # after it has been created. If the upload of the first block has + # already started , removal would be stuck behind the upload procedure, + # waiting for every block to be uploaded only to remove it afterwards. + for timeout in (0, None): + for blockno in list(blocknos): + if not self._lock_entry(inode, blockno, release_global=True, timeout=timeout): continue + blocknos.remove(blockno) + try: + if (inode, blockno) in self.cache: + log.debug('removing from cache') + self.cache.remove((inode, blockno)) + + try: + block_id = self.db.get_val('SELECT block_id FROM inode_blocks ' + 'WHERE inode=? AND blockno=?', (inode, blockno)) + except NoSuchRowError: + log.debug('block not in db') + continue - # Detach inode from block - self.db.execute('DELETE FROM inode_blocks WHERE inode=? AND blockno=?', - (inode, blockno)) + # Detach inode from block + self.db.execute('DELETE FROM inode_blocks WHERE inode=? AND blockno=?', + (inode, blockno)) - finally: - self._unlock_entry(inode, blockno, release_global=True) + finally: + self._unlock_entry(inode, blockno, release_global=True) - # Decrease block refcount - self._deref_block(block_id) + # Decrease block refcount + self._deref_block(block_id) log.debug('finished') diff --git a/src/s3ql/fs.py b/src/s3ql/fs.py index 2b46e7f..5bd3518 100644 --- a/src/s3ql/fs.py +++ b/src/s3ql/fs.py @@ -226,9 +226,8 @@ class Operations(llfuse.Operations): def listxattr(self, id_, ctx): log.debug('started with %d', id_) names = list() - with self.db.query('SELECT name FROM ext_attributes_v WHERE inode=?', (id_,)) as res: - for (name,) in res: - names.append(name) + for (name,) in self.db.query('SELECT name FROM ext_attributes_v WHERE inode=?', (id_,)): + names.append(name) return names def setxattr(self, id_, name, value, ctx): diff --git a/src/s3ql/fsck.py b/src/s3ql/fsck.py index 1017613..b3cd834 100644 --- a/src/s3ql/fsck.py +++ b/src/s3ql/fsck.py @@ -544,8 +544,8 @@ class Fsck(object): continue self.moved_inodes.add(inode) - affected_entries = list(self.conn.query('SELECT name, name_id, parent_inode ' - 'FROM contents_v WHERE inode=?', (inode,))) + affected_entries = self.conn.get_list('SELECT name, name_id, parent_inode ' + 'FROM contents_v WHERE inode=?', (inode,)) for (name, name_id, id_p) in affected_entries: path = get_path(id_p, self.conn, name) self.log_error("File may lack data, moved to /lost+found: %s", to_str(path)) @@ -691,8 +691,8 @@ class Fsck(object): log.info('Checking blocks (checksums)...') - for (block_id, obj_id) in list(self.conn.query('SELECT id, obj_id FROM blocks ' - 'WHERE hash IS NULL')): + for (block_id, obj_id) in self.conn.get_list('SELECT id, obj_id FROM blocks ' + 'WHERE hash IS NULL'): self.found_errors = True # This should only happen when there was an error during upload, @@ -989,8 +989,8 @@ class Fsck(object): # Copy the list, or we may pick up the same entry again and again # (first from the original location, then from lost+found) - affected_entries = list(self.conn.query('SELECT name, name_id, parent_inode ' - 'FROM contents_v WHERE inode=?', (id_,))) + affected_entries = self.conn.get_list('SELECT name, name_id, parent_inode ' + 'FROM contents_v WHERE inode=?', (id_,)) for (name, name_id, id_p) in affected_entries: path = get_path(id_p, self.conn, name) self.log_error("File may lack data, moved to /lost+found: %s", to_str(path)) @@ -1169,7 +1169,6 @@ def parse_args(args): parser.add_log('~/.s3ql/fsck.log') parser.add_cachedir() - parser.add_authfile() parser.add_debug() parser.add_quiet() parser.add_backend_options() diff --git a/src/s3ql/metadata.py b/src/s3ql/metadata.py index f465c6b..5a8bffb 100644 --- a/src/s3ql/metadata.py +++ b/src/s3ql/metadata.py @@ -177,7 +177,7 @@ def create_tables(conn): conn.execute(""" CREATE TABLE blocks ( id INTEGER PRIMARY KEY, - hash BLOB(16) UNIQUE, + hash BLOB(32) UNIQUE, refcount INT, size INT NOT NULL, obj_id INTEGER NOT NULL REFERENCES objects(id) diff --git a/src/s3ql/mkfs.py b/src/s3ql/mkfs.py index ecf81d0..f5cdf04 100644 --- a/src/s3ql/mkfs.py +++ b/src/s3ql/mkfs.py @@ -31,7 +31,6 @@ def parse_args(args): description="Initializes an S3QL file system") parser.add_cachedir() - parser.add_authfile() parser.add_debug() parser.add_quiet() parser.add_backend_options() @@ -95,7 +94,7 @@ def main(args=None): atexit.register(plain_backend.close) log.info("Before using S3QL, make sure to read the user's guide, especially\n" - "the 'Important Rules to Avoid Loosing Data' section.") + "the 'Important Rules to Avoid Losing Data' section.") if isinstance(plain_backend, s3.Backend) and '.' in plain_backend.bucket_name: log.warning('S3 Buckets with names containing dots cannot be ' diff --git a/src/s3ql/mount.py b/src/s3ql/mount.py index 8c61345..2773fc4 100644 --- a/src/s3ql/mount.py +++ b/src/s3ql/mount.py @@ -514,7 +514,6 @@ def parse_args(args): parser.add_log('~/.s3ql/mount.log') parser.add_cachedir() - parser.add_authfile() parser.add_debug() parser.add_quiet() parser.add_backend_options() diff --git a/src/s3ql/multi_lock.py b/src/s3ql/multi_lock.py index 59b23a8..1177d1d 100644 --- a/src/s3ql/multi_lock.py +++ b/src/s3ql/multi_lock.py @@ -52,6 +52,8 @@ class MultiLock: self.locked_keys.add(key) + return True + def release(self, *key, noerror=False): """Release lock on given key diff --git a/src/s3ql/parse_args.py b/src/s3ql/parse_args.py index f557c6d..b2f5be0 100644 --- a/src/s3ql/parse_args.py +++ b/src/s3ql/parse_args.py @@ -162,12 +162,6 @@ class ArgumentParser(argparse.ArgumentParser): help="Activate debugging output from all S3QL modules. " + destnote) - def add_authfile(self): - self.add_argument("--authfile", type=str, metavar='<path>', - default=os.path.expanduser("~/.s3ql/authinfo2"), - help='Read authentication credentials from this file ' - '(default: `~/.s3ql/authinfo2)`') - def add_cachedir(self): self.add_argument("--cachedir", type=str, metavar='<path>', default=os.path.expanduser("~/.s3ql"), @@ -186,6 +180,10 @@ class ArgumentParser(argparse.ArgumentParser): self.add_argument("storage_url", metavar='<storage-url>', type=storage_url_type, help='Storage URL of the backend that contains the file system') + self.add_argument("--authfile", type=str, metavar='<path>', + default=os.path.expanduser("~/.s3ql/authinfo2"), + help='Read authentication credentials from this file ' + '(default: `~/.s3ql/authinfo2)`') def add_subparsers(self, **kw): '''Pass parent and set prog to default usage message''' @@ -211,9 +209,8 @@ class ArgumentParser(argparse.ArgumentParser): except ArgumentError as exc: self.error(str(exc)) - if hasattr(options, 'authfile'): - assert options.storage_url - self._read_authfile(options) + if hasattr(options, 'storage_url'): + self._init_backend_factory(options) if hasattr(options, 'cachedir'): assert options.storage_url @@ -232,7 +229,7 @@ class ArgumentParser(argparse.ArgumentParser): return options - def _read_authfile(self, options): + def _init_backend_factory(self, options): storage_url = options.storage_url hit = re.match(r'^([a-zA-Z0-9]+)://', storage_url) if not hit: diff --git a/src/s3ql/verify.py b/src/s3ql/verify.py index ca3075d..b8913a2 100644 --- a/src/s3ql/verify.py +++ b/src/s3ql/verify.py @@ -52,7 +52,6 @@ def parse_args(args): parser.add_quiet() parser.add_version() parser.add_cachedir() - parser.add_authfile() parser.add_backend_options() parser.add_storage_url() diff --git a/tests/common.py b/tests/common.py index 2b1e8b3..8c3be59 100644 --- a/tests/common.py +++ b/tests/common.py @@ -213,9 +213,13 @@ def populate_dir(path, entries=1000, size=20*1024*1024, srcname = os.path.join(pooldir, poolnames[idx]) if not os.path.isfile(srcname): continue - with open(srcname, 'rb') as src: - buf = src.read(size) - dst.write(buf) + try: + with open(srcname, 'rb') as src: + buf = src.read(size) + except PermissionError: + # Source not readable.. + continue + dst.write(buf) size -= len(buf) files.append(name) diff --git a/tests/t2_block_cache.py b/tests/t2_block_cache.py index 453a58b..ebdc7fe 100755 --- a/tests/t2_block_cache.py +++ b/tests/t2_block_cache.py @@ -541,6 +541,8 @@ class MockMultiLock: self.release(*key) def acquire(self, *key, timeout=None): + if timeout == 0: + return False me = threading.current_thread() log.debug('%s blocked in acquire()', me.name) with self.cond: @@ -548,6 +550,7 @@ class MockMultiLock: pytest.fail('Timeout waiting for lock') self.real_mlock.locked_keys.add(key) log.debug('%s got lock', me.name) + return True def release(self, *key, noerror=False): me = threading.current_thread() diff --git a/tests/t4_fuse.py b/tests/t4_fuse.py index d529c77..75f8322 100755 --- a/tests/t4_fuse.py +++ b/tests/t4_fuse.py @@ -114,29 +114,18 @@ class TestFuse: assert not os.path.ismount(self.mnt_dir) def fsck(self): - # Use fsck to test authinfo reading - with tempfile.NamedTemporaryFile('wt') as authinfo_fh: - print('[entry1]', - 'storage-url: %s' % self.storage_url[:6], - 'fs-passphrase: clearly wrong', - 'backend-login: bla', - 'backend-password: not much better', - '', - '[entry2]', - 'storage-url: %s' % self.storage_url, - 'fs-passphrase: %s' % self.passphrase, - 'backend-login: %s' % self.backend_login, - 'backend-password:%s' % self.backend_passphrase, - file=authinfo_fh, sep='\n') - authinfo_fh.flush() - - proc = subprocess.Popen(self.s3ql_cmd_argv('fsck.s3ql') + - [ '--force', '--quiet', '--log', 'none', '--cachedir', - self.cache_dir, '--authfile', - authinfo_fh.name, self.storage_url ], - stdin=subprocess.PIPE, universal_newlines=True) - proc.stdin.close() - assert proc.wait() == 0 + proc = subprocess.Popen(self.s3ql_cmd_argv('fsck.s3ql') + + [ '--force', '--quiet', '--log', 'none', '--cachedir', + self.cache_dir, '--authfile', '/dev/null', + self.storage_url ], + stdin=subprocess.PIPE, universal_newlines=True) + if self.backend_login is not None: + print(self.backend_login, file=proc.stdin) + print(self.backend_passphrase, file=proc.stdin) + if self.passphrase is not None: + print(self.passphrase, file=proc.stdin) + proc.stdin.close() + assert proc.wait() == 0 def teardown_method(self, method): with open('/dev/null', 'wb') as devnull: @@ -178,6 +167,17 @@ class TestFuse: self.umount() self.fsck() + # Test metadata recovery + shutil.rmtree(self.cache_dir) + self.cache_dir = tempfile.mkdtemp(prefix='s3ql-cache-') + self.fsck() + + shutil.rmtree(self.cache_dir) + self.cache_dir = tempfile.mkdtemp(prefix='s3ql-cache-') + self.mount() + self.umount() + + def newname(self): self.name_cnt += 1 return "s3ql_%d" % self.name_cnt diff --git a/tests/t5_failsafe.py b/tests/t5_failsafe.py index af7fbea..ba9c83a 100755 --- a/tests/t5_failsafe.py +++ b/tests/t5_failsafe.py @@ -42,6 +42,7 @@ class TestFailsafe(t4_fuse.TestFuse): (backend_login, backend_pw, self.storage_url) = get_remote_test_info('gs-test') except NoTestSection as exc: + super().teardown_method(method) pytest.skip(exc.reason) self.backend_login = backend_login diff --git a/tests/t5_full.py b/tests/t5_full.py index e3011e2..5dcf5e8 100755 --- a/tests/t5_full.py +++ b/tests/t5_full.py @@ -74,6 +74,7 @@ class RemoteTest: (backend_login, backend_pw, self.storage_url) = get_remote_test_info(name) except NoTestSection as exc: + super().teardown_method(method) pytest.skip(exc.reason) self.backend_login = backend_login self.backend_passphrase = backend_pw diff --git a/tests/t6_upgrade.py b/tests/t6_upgrade.py index ffbc387..d58eea5 100755 --- a/tests/t6_upgrade.py +++ b/tests/t6_upgrade.py @@ -185,6 +185,7 @@ class RemoteUpgradeTest: (backend_login, backend_pw, self.storage_url) = get_remote_test_info(name) except NoTestSection as exc: + super().teardown_method(method) pytest.skip(exc.reason) self.backend_login = backend_login self.backend_passphrase = backend_pw |