diff options
author | Nikolaus Rath <Nikolaus@rath.org> | 2018-01-05 11:57:29 +0100 |
---|---|---|
committer | Nikolaus Rath <Nikolaus@rath.org> | 2018-01-05 11:57:29 +0100 |
commit | f0a962d37508caa5c59402b0353d3347be170415 (patch) | |
tree | 17611c2bb9757c1fb853847b60279c6fdd75d2cd | |
parent | cff3c35a423c68dadec70a45a039c2db8dc19e32 (diff) |
Import s3ql_2.25+dfsg.orig.tar.gz
-rw-r--r-- | Changes.txt | 7 | ||||
-rw-r--r-- | PKG-INFO | 13 | ||||
-rw-r--r-- | README.rst | 11 | ||||
-rw-r--r-- | contrib/expire_backups.1 | 2 | ||||
-rw-r--r-- | contrib/pcp.1 | 2 | ||||
-rw-r--r-- | doc/latex/manual.aux | 64 | ||||
-rw-r--r-- | doc/latex/manual.tex | 174 | ||||
-rw-r--r-- | doc/man/fsck.s3ql.1 | 2 | ||||
-rw-r--r-- | doc/man/mkfs.s3ql.1 | 2 | ||||
-rw-r--r-- | doc/man/mount.s3ql.1 | 2 | ||||
-rw-r--r-- | doc/man/s3ql_oauth_client.1 | 2 | ||||
-rw-r--r-- | doc/man/s3ql_verify.1 | 2 | ||||
-rw-r--r-- | doc/man/s3qladm.1 | 2 | ||||
-rw-r--r-- | doc/man/s3qlcp.1 | 2 | ||||
-rw-r--r-- | doc/man/s3qlctrl.1 | 2 | ||||
-rw-r--r-- | doc/man/s3qllock.1 | 2 | ||||
-rw-r--r-- | doc/man/s3qlrm.1 | 2 | ||||
-rw-r--r-- | doc/man/s3qlstat.1 | 2 | ||||
-rw-r--r-- | doc/man/umount.s3ql.1 | 2 | ||||
-rw-r--r-- | doc/manual.pdf | bin | 283819 -> 284062 bytes | |||
-rw-r--r-- | rst/special.rst | 7 | ||||
-rw-r--r-- | src/s3ql.egg-info/PKG-INFO | 13 | ||||
-rw-r--r-- | src/s3ql/__init__.py | 2 | ||||
-rw-r--r-- | src/s3ql/block_cache.py | 86 | ||||
-rw-r--r-- | src/s3ql/common.py | 5 | ||||
-rw-r--r-- | src/s3ql/ctrl.py | 7 | ||||
-rw-r--r-- | src/s3ql/fs.py | 4 | ||||
-rw-r--r-- | src/s3ql/inode_cache.py | 7 | ||||
-rw-r--r-- | src/s3ql/mount.py | 7 | ||||
-rw-r--r-- | tests/pytest.ini | 3 | ||||
-rwxr-xr-x | tests/t2_block_cache.py | 167 | ||||
-rwxr-xr-x | tests/t5_ctrl.py | 8 |
32 files changed, 376 insertions, 237 deletions
diff --git a/Changes.txt b/Changes.txt index 77871ec..a2a188b 100644 --- a/Changes.txt +++ b/Changes.txt @@ -1,3 +1,10 @@ +2018-01-05, S3QL 2.25 + + * s3qlctrl now accepts a new *dropcache* command. + * Fixed a race condition that resulted in mount.s3ql crashing with + "I/O operation on closed file". Thanks to www.imCode.com for + sponsoring this work! + 2017-11-04, S3QL 2.24 * fsck.s3ql now accepts a new --force-remote parameter. It does @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: s3ql -Version: 2.24 +Version: 2.25 Summary: a full-featured file system for online data storage Home-page: https://bitbucket.org/nikratio/s3ql/ Author: Nikolaus Rath @@ -146,17 +146,15 @@ Description: .. Please report any bugs you may encounter in the `Bitbucket Issue Tracker`_. - Professional Support - ==================== - - Professional support is available. Please contact Nikolaus Rath - <Nikolaus@rath.org> for details. - Contributing ============ The S3QL source code is available both on GitHub_ and BitBucket_. + Professional Support + -------------------- + + Professional support is offered via `Rath Consulting`_. .. _`S3QL User's Guide`: http://www.rath.org/s3ql-docs/index.html .. _`S3QL Wiki`: https://bitbucket.org/nikratio/s3ql/wiki/ @@ -166,6 +164,7 @@ Description: .. .. _`Bitbucket Issue Tracker`: https://bitbucket.org/nikratio/s3ql/issues .. _BitBucket: https://bitbucket.org/nikratio/s3ql/ .. _GitHub: https://github.com/s3ql/main + .. _`Rath Consulting`: http://www.rath-consulting.biz/ Keywords: FUSE,backup,archival,compression,encryption,deduplication,aws,s3 Platform: POSIX @@ -137,17 +137,15 @@ The following resources are available: Please report any bugs you may encounter in the `Bitbucket Issue Tracker`_. -Professional Support -==================== - -Professional support is available. Please contact Nikolaus Rath -<Nikolaus@rath.org> for details. - Contributing ============ The S3QL source code is available both on GitHub_ and BitBucket_. +Professional Support +-------------------- + +Professional support is offered via `Rath Consulting`_. .. _`S3QL User's Guide`: http://www.rath.org/s3ql-docs/index.html .. _`S3QL Wiki`: https://bitbucket.org/nikratio/s3ql/wiki/ @@ -157,3 +155,4 @@ The S3QL source code is available both on GitHub_ and BitBucket_. .. _`Bitbucket Issue Tracker`: https://bitbucket.org/nikratio/s3ql/issues .. _BitBucket: https://bitbucket.org/nikratio/s3ql/ .. _GitHub: https://github.com/s3ql/main +.. _`Rath Consulting`: http://www.rath-consulting.biz/ diff --git a/contrib/expire_backups.1 b/contrib/expire_backups.1 index cb1e7e2..2aed509 100644 --- a/contrib/expire_backups.1 +++ b/contrib/expire_backups.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "EXPIRE_BACKUPS" "1" "Nov 04, 2017" "2.24" "S3QL" +.TH "EXPIRE_BACKUPS" "1" "Jan 05, 2018" "2.25" "S3QL" .SH NAME expire_backups \- Intelligently expire old backups . diff --git a/contrib/pcp.1 b/contrib/pcp.1 index c95f74c..e5d5f52 100644 --- a/contrib/pcp.1 +++ b/contrib/pcp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "PCP" "1" "Nov 04, 2017" "2.24" "S3QL" +.TH "PCP" "1" "Jan 05, 2018" "2.25" "S3QL" .SH NAME pcp \- Recursive, parallel copy of directory trees . diff --git a/doc/latex/manual.aux b/doc/latex/manual.aux index ee2490b..1b0d1cf 100644 --- a/doc/latex/manual.aux +++ b/doc/latex/manual.aux @@ -25,12 +25,12 @@ \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{about::doc}{{1}{1}{S3QL}{chapter.1}{}} \newlabel{about:s3ql-user-s-guide}{{1}{1}{S3QL}{chapter.1}{}} +\newlabel{about::doc}{{1}{1}{S3QL}{chapter.1}{}} \newlabel{about:s3ql}{{1}{1}{S3QL}{chapter.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {1.1}Features}{1}{section.1.1}} -\newlabel{about:openstack}{{1.1}{1}{Features}{section.1.1}{}} \newlabel{about:features}{{1.1}{1}{Features}{section.1.1}{}} +\newlabel{about:openstack}{{1.1}{1}{Features}{section.1.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {1.2}Development Status}{2}{section.1.2}} \newlabel{about:development-status}{{1.2}{2}{Development Status}{section.1.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {1.3}Supported Platforms}{2}{section.1.3}} @@ -41,14 +41,14 @@ \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{installation::doc}{{2}{3}{Installation}{chapter.2}{}} -\newlabel{installation:installation}{{2}{3}{Installation}{chapter.2}{}} \newlabel{installation:github}{{2}{3}{Installation}{chapter.2}{}} +\newlabel{installation:installation}{{2}{3}{Installation}{chapter.2}{}} +\newlabel{installation::doc}{{2}{3}{Installation}{chapter.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.1}Dependencies}{3}{section.2.1}} \newlabel{installation:dependencies}{{2.1}{3}{Dependencies}{section.2.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.2}Installing S3QL}{4}{section.2.2}} -\newlabel{installation:inst-s3ql}{{2.2}{4}{Installing S3QL}{section.2.2}{}} \newlabel{installation:installing-s3ql}{{2.2}{4}{Installing S3QL}{section.2.2}{}} +\newlabel{installation:inst-s3ql}{{2.2}{4}{Installing S3QL}{section.2.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.3}Development Version}{4}{section.2.3}} \newlabel{installation:development-version}{{2.3}{4}{Development Version}{section.2.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.4}Running tests requiring remote servers}{4}{section.2.4}} @@ -57,18 +57,18 @@ \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} +\newlabel{backends:sphinx}{{3}{7}{Storage Backends}{chapter.3}{}} \newlabel{backends:id1}{{3}{7}{Storage Backends}{chapter.3}{}} -\newlabel{backends::doc}{{3}{7}{Storage Backends}{chapter.3}{}} \newlabel{backends:storage-backends}{{3}{7}{Storage Backends}{chapter.3}{}} -\newlabel{backends:sphinx}{{3}{7}{Storage Backends}{chapter.3}{}} +\newlabel{backends::doc}{{3}{7}{Storage Backends}{chapter.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.1}Google Storage}{7}{section.3.1}} \newlabel{backends:google-storage}{{3.1}{7}{Google Storage}{section.3.1}{}} \newlabel{backends:cmdoption-gs_backend-arg-no-ssl}{{3.1}{7}{Google Storage}{section*.3}{}} \newlabel{backends:cmdoption-gs_backend-arg-ssl-ca-path}{{3.1}{8}{Google Storage}{section*.4}{}} \newlabel{backends:cmdoption-gs_backend-arg-tcp-timeout}{{3.1}{8}{Google Storage}{section*.5}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.2}Amazon S3}{8}{section.3.2}} -\newlabel{backends:amazon-s3}{{3.2}{8}{Amazon S3}{section.3.2}{}} \newlabel{backends:google-storage-manager}{{3.2}{8}{Amazon S3}{section.3.2}{}} +\newlabel{backends:amazon-s3}{{3.2}{8}{Amazon S3}{section.3.2}{}} \newlabel{backends:cmdoption-s3_backend-arg-no-ssl}{{3.2}{8}{Amazon S3}{section*.6}{}} \newlabel{backends:cmdoption-s3_backend-arg-ssl-ca-path}{{3.2}{8}{Amazon S3}{section*.7}{}} \newlabel{backends:cmdoption-s3_backend-arg-tcp-timeout}{{3.2}{8}{Amazon S3}{section*.8}{}} @@ -86,24 +86,24 @@ \@writefile{toc}{\contentsline {section}{\numberline {3.4}Rackspace CloudFiles}{10}{section.3.4}} \newlabel{backends:rackspace-cloudfiles}{{3.4}{10}{Rackspace CloudFiles}{section.3.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.5}S3 compatible}{10}{section.3.5}} -\newlabel{backends:rackspace}{{3.5}{10}{S3 compatible}{section.3.5}{}} \newlabel{backends:s3-compatible}{{3.5}{10}{S3 compatible}{section.3.5}{}} +\newlabel{backends:rackspace}{{3.5}{10}{S3 compatible}{section.3.5}{}} \newlabel{backends:cmdoption-s3c_backend-arg-no-ssl}{{3.5}{10}{S3 compatible}{section*.17}{}} \newlabel{backends:cmdoption-s3c_backend-arg-ssl-ca-path}{{3.5}{10}{S3 compatible}{section*.18}{}} \newlabel{backends:cmdoption-s3c_backend-arg-tcp-timeout}{{3.5}{10}{S3 compatible}{section*.19}{}} \newlabel{backends:cmdoption-s3c_backend-arg-disable-expect100}{{3.5}{10}{S3 compatible}{section*.20}{}} \newlabel{backends:cmdoption-s3c_backend-arg-dumb-copy}{{3.5}{10}{S3 compatible}{section*.21}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.6}Local}{11}{section.3.6}} -\newlabel{backends:local}{{3.6}{11}{Local}{section.3.6}{}} \newlabel{backends:id6}{{3.6}{11}{Local}{section.3.6}{}} +\newlabel{backends:local}{{3.6}{11}{Local}{section.3.6}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {4}Important Rules to Avoid Losing Data}{13}{chapter.4}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{durability:durability}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} -\newlabel{durability:sshfs}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} -\newlabel{durability::doc}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} \newlabel{durability:important-rules-to-avoid-losing-data}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} +\newlabel{durability::doc}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} +\newlabel{durability:sshfs}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.1}Rules in a Nutshell}{13}{section.4.1}} \newlabel{durability:rules-in-a-nutshell}{{4.1}{13}{Rules in a Nutshell}{section.4.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.2}Consistency Window List}{14}{section.4.2}} @@ -111,20 +111,20 @@ \@writefile{toc}{\contentsline {section}{\numberline {4.3}Data Consistency}{14}{section.4.3}} \newlabel{durability:data-consistency}{{4.3}{14}{Data Consistency}{section.4.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.4}Data Durability}{15}{section.4.4}} -\newlabel{durability:data-durability}{{4.4}{15}{Data Durability}{section.4.4}{}} \newlabel{durability:backend-reliability}{{4.4}{15}{Data Durability}{section.4.4}{}} +\newlabel{durability:data-durability}{{4.4}{15}{Data Durability}{section.4.4}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {5}File System Creation}{17}{chapter.5}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{mkfs::doc}{{5}{17}{File System Creation}{chapter.5}{}} \newlabel{mkfs:file-system-creation}{{5}{17}{File System Creation}{chapter.5}{}} +\newlabel{mkfs::doc}{{5}{17}{File System Creation}{chapter.5}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {6}Managing File Systems}{19}{chapter.6}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{adm::doc}{{6}{19}{Managing File Systems}{chapter.6}{}} \newlabel{adm:managing-file-systems}{{6}{19}{Managing File Systems}{chapter.6}{}} +\newlabel{adm::doc}{{6}{19}{Managing File Systems}{chapter.6}{}} \@writefile{toc}{\contentsline {section}{\numberline {6.1}Changing the Passphrase}{19}{section.6.1}} \newlabel{adm:changing-the-passphrase}{{6.1}{19}{Changing the Passphrase}{section.6.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {6.2}Upgrading the file system}{20}{section.6.2}} @@ -137,8 +137,8 @@ \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{mount::doc}{{7}{21}{Mounting}{chapter.7}{}} \newlabel{mount:mounting}{{7}{21}{Mounting}{chapter.7}{}} +\newlabel{mount::doc}{{7}{21}{Mounting}{chapter.7}{}} \@writefile{toc}{\contentsline {section}{\numberline {7.1}Permission Checking}{22}{section.7.1}} \newlabel{mount:permission-checking}{{7.1}{22}{Permission Checking}{section.7.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {7.2}Compression Algorithms}{22}{section.7.2}} @@ -152,8 +152,8 @@ \@writefile{toc}{\contentsline {section}{\numberline {7.4}Failure Modes}{23}{section.7.4}} \newlabel{mount:failure-modes}{{7.4}{23}{Failure Modes}{section.7.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {7.5}Automatic Mounting}{24}{section.7.5}} -\newlabel{mount:automatic-mounting}{{7.5}{24}{Automatic Mounting}{section.7.5}{}} \newlabel{mount:logcheck}{{7.5}{24}{Automatic Mounting}{section.7.5}{}} +\newlabel{mount:automatic-mounting}{{7.5}{24}{Automatic Mounting}{section.7.5}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {8}Advanced S3QL Features}{25}{chapter.8}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} @@ -161,52 +161,52 @@ \newlabel{special::doc}{{8}{25}{Advanced S3QL Features}{chapter.8}{}} \newlabel{special:advanced-s3ql-features}{{8}{25}{Advanced S3QL Features}{chapter.8}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.1}Snapshotting and Copy-on-Write}{25}{section.8.1}} -\newlabel{special:snapshotting-and-copy-on-write}{{8.1}{25}{Snapshotting and Copy-on-Write}{section.8.1}{}} \newlabel{special:s3qlcp}{{8.1}{25}{Snapshotting and Copy-on-Write}{section.8.1}{}} +\newlabel{special:snapshotting-and-copy-on-write}{{8.1}{25}{Snapshotting and Copy-on-Write}{section.8.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.1.1}Snapshotting vs Hardlinking}{25}{subsection.8.1.1}} \newlabel{special:snapshotting-vs-hardlinking}{{8.1.1}{25}{Snapshotting vs Hardlinking}{subsection.8.1.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.2}Getting Statistics}{26}{section.8.2}} \newlabel{special:s3qlstat}{{8.2}{26}{Getting Statistics}{section.8.2}{}} \newlabel{special:getting-statistics}{{8.2}{26}{Getting Statistics}{section.8.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.3}Immutable Trees}{26}{section.8.3}} -\newlabel{special:s3qllock}{{8.3}{26}{Immutable Trees}{section.8.3}{}} \newlabel{special:immutable-trees}{{8.3}{26}{Immutable Trees}{section.8.3}{}} +\newlabel{special:s3qllock}{{8.3}{26}{Immutable Trees}{section.8.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.4}Fast Recursive Removal}{27}{section.8.4}} \newlabel{special:s3qlrm}{{8.4}{27}{Fast Recursive Removal}{section.8.4}{}} \newlabel{special:fast-recursive-removal}{{8.4}{27}{Fast Recursive Removal}{section.8.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.5}Runtime Configuration}{27}{section.8.5}} -\newlabel{special:s3qlctrl}{{8.5}{27}{Runtime Configuration}{section.8.5}{}} \newlabel{special:runtime-configuration}{{8.5}{27}{Runtime Configuration}{section.8.5}{}} +\newlabel{special:s3qlctrl}{{8.5}{27}{Runtime Configuration}{section.8.5}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {9}Unmounting}{29}{chapter.9}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{umount::doc}{{9}{29}{Unmounting}{chapter.9}{}} \newlabel{umount:unmounting}{{9}{29}{Unmounting}{chapter.9}{}} +\newlabel{umount::doc}{{9}{29}{Unmounting}{chapter.9}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {10}Checking for Errors}{31}{chapter.10}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{fsck::doc}{{10}{31}{Checking for Errors}{chapter.10}{}} \newlabel{fsck:checking-for-errors}{{10}{31}{Checking for Errors}{chapter.10}{}} +\newlabel{fsck::doc}{{10}{31}{Checking for Errors}{chapter.10}{}} \@writefile{toc}{\contentsline {section}{\numberline {10.1}Checking and repairing internal file system errors}{31}{section.10.1}} \newlabel{fsck:checking-and-repairing-internal-file-system-errors}{{10.1}{31}{Checking and repairing internal file system errors}{section.10.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {10.2}Detecting and handling backend data corruption}{32}{section.10.2}} -\newlabel{fsck:s3ql-verify}{{10.2}{32}{Detecting and handling backend data corruption}{section.10.2}{}} \newlabel{fsck:detecting-and-handling-backend-data-corruption}{{10.2}{32}{Detecting and handling backend data corruption}{section.10.2}{}} +\newlabel{fsck:s3ql-verify}{{10.2}{32}{Detecting and handling backend data corruption}{section.10.2}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {11}Storing Authentication Information}{35}{chapter.11}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} +\newlabel{authinfo:storing-authentication-information}{{11}{35}{Storing Authentication Information}{chapter.11}{}} \newlabel{authinfo:authinfo}{{11}{35}{Storing Authentication Information}{chapter.11}{}} \newlabel{authinfo::doc}{{11}{35}{Storing Authentication Information}{chapter.11}{}} -\newlabel{authinfo:storing-authentication-information}{{11}{35}{Storing Authentication Information}{chapter.11}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {12}Contributed Programs}{37}{chapter.12}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{contrib::doc}{{12}{37}{Contributed Programs}{chapter.12}{}} \newlabel{contrib:contributed-programs}{{12}{37}{Contributed Programs}{chapter.12}{}} +\newlabel{contrib::doc}{{12}{37}{Contributed Programs}{chapter.12}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.1}benchmark.py}{37}{section.12.1}} \newlabel{contrib:benchmark-py}{{12.1}{37}{benchmark.py}{section.12.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.2}clone\_fs.py}{37}{section.12.2}} @@ -225,11 +225,11 @@ \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{tips::doc}{{13}{41}{Tips \& Tricks}{chapter.13}{}} \newlabel{tips:tips-tricks}{{13}{41}{Tips \& Tricks}{chapter.13}{}} +\newlabel{tips::doc}{{13}{41}{Tips \& Tricks}{chapter.13}{}} \@writefile{toc}{\contentsline {section}{\numberline {13.1}SSH Backend}{41}{section.13.1}} -\newlabel{tips:ssh-backend}{{13.1}{41}{SSH Backend}{section.13.1}{}} \newlabel{tips:ssh-tipp}{{13.1}{41}{SSH Backend}{section.13.1}{}} +\newlabel{tips:ssh-backend}{{13.1}{41}{SSH Backend}{section.13.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {13.2}Permanently mounted backup file system}{41}{section.13.2}} \newlabel{tips:permanently-mounted-backup-file-system}{{13.2}{41}{Permanently mounted backup file system}{section.13.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {13.3}Improving copy performance}{41}{section.13.3}} @@ -239,8 +239,8 @@ \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{issues::doc}{{14}{43}{Known Issues}{chapter.14}{}} \newlabel{issues:known-issues}{{14}{43}{Known Issues}{chapter.14}{}} +\newlabel{issues::doc}{{14}{43}{Known Issues}{chapter.14}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {15}Manpages}{45}{chapter.15}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} @@ -384,9 +384,9 @@ \@writefile{toc}{\contentsline {subsection}{\numberline {15.10.5}See Also}{59}{subsection.15.10.5}} \newlabel{man/fsck:see-also}{{15.10.5}{59}{See Also}{subsection.15.10.5}{}} \@writefile{toc}{\contentsline {section}{\numberline {15.11}The \textbf {\texttt {s3ql\_oauth\_client}} command}{59}{section.15.11}} +\newlabel{man/oauth_client:oauth-client}{{15.11}{59}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}} \newlabel{man/oauth_client:the-command-command}{{15.11}{59}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}} \newlabel{man/oauth_client::doc}{{15.11}{59}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}} -\newlabel{man/oauth_client:oauth-client}{{15.11}{59}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.11.1}Synopsis}{59}{subsection.15.11.1}} \newlabel{man/oauth_client:synopsis}{{15.11.1}{59}{Synopsis}{subsection.15.11.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.11.2}Description}{59}{subsection.15.11.2}} @@ -440,16 +440,16 @@ \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} +\newlabel{resources:resources}{{16}{65}{Further Resources / Getting Help}{chapter.16}{}} \newlabel{resources::doc}{{16}{65}{Further Resources / Getting Help}{chapter.16}{}} \newlabel{resources:further-resources-getting-help}{{16}{65}{Further Resources / Getting Help}{chapter.16}{}} -\newlabel{resources:resources}{{16}{65}{Further Resources / Getting Help}{chapter.16}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {17}Implementation Details}{67}{chapter.17}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{impl_details::doc}{{17}{67}{Implementation Details}{chapter.17}{}} -\newlabel{impl_details:implementation-details}{{17}{67}{Implementation Details}{chapter.17}{}} \newlabel{impl_details:impl-details}{{17}{67}{Implementation Details}{chapter.17}{}} +\newlabel{impl_details:implementation-details}{{17}{67}{Implementation Details}{chapter.17}{}} +\newlabel{impl_details::doc}{{17}{67}{Implementation Details}{chapter.17}{}} \@writefile{toc}{\contentsline {section}{\numberline {17.1}Metadata Storage}{67}{section.17.1}} \newlabel{impl_details:metadata-storage}{{17.1}{67}{Metadata Storage}{section.17.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {17.2}Data Storage}{67}{section.17.2}} diff --git a/doc/latex/manual.tex b/doc/latex/manual.tex index d6f917a..929d707 100644 --- a/doc/latex/manual.tex +++ b/doc/latex/manual.tex @@ -32,8 +32,8 @@ \title{S3QL Documentation} -\date{Nov 04, 2017} -\release{2.24} +\date{Jan 05, 2018} +\release{2.25} \author{Nikolaus Rath} \newcommand{\sphinxlogo}{} \renewcommand{\releasename}{Release} @@ -50,73 +50,73 @@ \PYG@it{\PYG@bf{\PYG@ff{#1}}}}}}} \def\PYG#1#2{\PYG@reset\PYG@toks#1+\relax+\PYG@do{#2}} -\expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.20}{##1}}} -\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.73,0.00,0.40}{##1}}} -\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} -\expandafter\def\csname PYG@tok@gp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}} -\expandafter\def\csname PYG@tok@kt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.60}{##1}}} -\expandafter\def\csname PYG@tok@mi\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} -\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.33,0.47,0.60}{##1}}} -\expandafter\def\csname PYG@tok@dl\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@c1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@vm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} \expandafter\def\csname PYG@tok@nl\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.60,0.47,0.00}{##1}}} -\expandafter\def\csname PYG@tok@nt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.47,0.00}{##1}}} -\expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}} \expandafter\def\csname PYG@tok@cs\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.80,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@mf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.80}{##1}}} +\expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}} +\expandafter\def\csname PYG@tok@sh\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} \expandafter\def\csname PYG@tok@kr\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@sd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.27,0.13}{##1}}} -\expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@mb\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} -\expandafter\def\csname PYG@tok@sc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} -\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}} +\expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} +\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@kp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.53}{##1}}} +\expandafter\def\csname PYG@tok@nt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@vm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} \expandafter\def\csname PYG@tok@vg\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.87,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@il\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} +\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.33,0.47,0.60}{##1}}} +\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} \expandafter\def\csname PYG@tok@gu\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}} -\expandafter\def\csname PYG@tok@mo\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.27,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.33,0.33,0.33}{##1}}} +\expandafter\def\csname PYG@tok@mi\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} +\expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit} +\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} \expandafter\def\csname PYG@tok@gs\endcsname{\let\PYG@bf=\textbf} -\expandafter\def\csname PYG@tok@kc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}} -\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.73}{##1}}} -\expandafter\def\csname PYG@tok@vc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.40,0.60}{##1}}} +\expandafter\def\csname PYG@tok@no\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.40}{##1}}} +\expandafter\def\csname PYG@tok@ch\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@c1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@sc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} +\expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.73,0.00,0.40}{##1}}} +\expandafter\def\csname PYG@tok@sr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,1.00}{\strut ##1}}} +\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} +\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.33,0.33,0.33}{##1}}} +\expandafter\def\csname PYG@tok@c\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} \expandafter\def\csname PYG@tok@bp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} -\expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} -\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.67,0.40,0.00}{##1}}} \expandafter\def\csname PYG@tok@gh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}} -\expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@no\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.40}{##1}}} -\expandafter\def\csname PYG@tok@ne\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@gp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}} +\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}} +\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}} \expandafter\def\csname PYG@tok@sa\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@sh\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@kc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} \expandafter\def\csname PYG@tok@err\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.67,0.67}{\strut ##1}}} -\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.33,0.53}{##1}}} -\expandafter\def\csname PYG@tok@kp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.53}{##1}}} -\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} -\expandafter\def\csname PYG@tok@fm\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} -\expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.80}{##1}}} -\expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@s\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@si\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.93,0.93,0.93}{\strut ##1}}} -\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@c\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@sr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,1.00}{\strut ##1}}} +\expandafter\def\csname PYG@tok@mb\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@kt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.60}{##1}}} +\expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.20}{##1}}} \expandafter\def\csname PYG@tok@ow\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit} -\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@il\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} -\expandafter\def\csname PYG@tok@ch\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} \expandafter\def\csname PYG@tok@se\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@mo\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.27,0.00,0.93}{##1}}} \expandafter\def\csname PYG@tok@cpf\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@fm\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} +\expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@s\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.33,0.53}{##1}}} +\expandafter\def\csname PYG@tok@sd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.27,0.13}{##1}}} +\expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.73}{##1}}} +\expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@ne\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@dl\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} +\expandafter\def\csname PYG@tok@si\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.93,0.93,0.93}{\strut ##1}}} +\expandafter\def\csname PYG@tok@mf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.67,0.40,0.00}{##1}}} +\expandafter\def\csname PYG@tok@vc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.40,0.60}{##1}}} \def\PYGZbs{\char`\\} \def\PYGZus{\char`\_} @@ -150,7 +150,7 @@ \chapter{S3QL} -\label{about::doc}\label{about:s3ql-user-s-guide}\label{about:s3ql} +\label{about:s3ql-user-s-guide}\label{about::doc}\label{about:s3ql} S3QL is a file system that stores all its data online using storage services like \href{http://code.google.com/apis/storage/}{Google Storage}, \href{http://aws.amazon.com/s3}{Amazon S3}, or \href{http://openstack.org/projects/storage/}{OpenStack}. S3QL effectively provides a hard disk of dynamic, infinite capacity that @@ -170,7 +170,7 @@ with extensive automated test cases for all its components. \section{Features} -\label{about:openstack}\label{about:features}\begin{itemize} +\label{about:features}\label{about:openstack}\begin{itemize} \item {} \textbf{Transparency.} Conceptually, S3QL is indistinguishable from a local file system. For example, it supports hardlinks, symlinks, @@ -256,12 +256,12 @@ The S3QL source code is available both on \href{https://github.com/s3ql/main}{Gi \chapter{Installation} -\label{installation::doc}\label{installation:installation}\label{installation:github} +\label{installation:github}\label{installation:installation}\label{installation::doc} S3QL depends on several other programs and libraries that have to be installed first. The best method to satisfy these dependencies depends on your distribution. -The following instructions are for S3QL 2.24 and should be +The following instructions are for S3QL 2.25 and should be applicable to any system. The \href{https://bitbucket.org/nikratio/s3ql/wiki/Home}{S3QL Wiki} contains \href{https://bitbucket.org/nikratio/s3ql/wiki/Installation}{additional help} help for specific distributions and operating systems. Note, however, that @@ -347,7 +347,7 @@ installed version if the module is installed. \section{Installing S3QL} -\label{installation:inst-s3ql}\label{installation:installing-s3ql} +\label{installation:installing-s3ql}\label{installation:inst-s3ql} To build and install S3QL itself, proceed as follows: \begin{enumerate} \item {} @@ -467,7 +467,7 @@ being skipped by passing the \sphinxcode{-rs} argument to \chapter{Storage Backends} -\label{backends:id1}\label{backends::doc}\label{backends:storage-backends}\label{backends:sphinx} +\label{backends:sphinx}\label{backends:id1}\label{backends:storage-backends}\label{backends::doc} S3QL supports different \emph{backends} to store data at different service providers and using different protocols. A \emph{storage url} specifies a backend together with some backend-specific information and uniquely @@ -572,7 +572,7 @@ TCP connection is closed and re-established (default: 20 seconds). \section{Amazon S3} -\label{backends:amazon-s3}\label{backends:google-storage-manager} +\label{backends:google-storage-manager}\label{backends:amazon-s3} \href{http://aws.amazon.com/s3}{Amazon S3} is the online storage service offered by \href{http://aws.amazon.com/}{Amazon Web Services (AWS)}. To use the S3 backend, you first need to sign up for an AWS account. The @@ -806,7 +806,7 @@ The Rackspace backend accepts the same backend options as the \section{S3 compatible} -\label{backends:rackspace}\label{backends:s3-compatible} +\label{backends:s3-compatible}\label{backends:rackspace} The S3 compatible backend allows S3QL to access any storage service that uses the same protocol as Amazon S3. The storage URL has the form @@ -883,7 +883,7 @@ body for a succesfull copy operation. \section{Local} -\label{backends:local}\label{backends:id6} +\label{backends:id6}\label{backends:local} S3QL is also able to store its data on the local file system. This can be used to backup data on external media, or to access external services that S3QL can not talk to directly (e.g., it is possible to @@ -909,7 +909,7 @@ The local backend does not accept any backend options. \chapter{Important Rules to Avoid Losing Data} -\label{durability:durability}\label{durability:sshfs}\label{durability::doc}\label{durability:important-rules-to-avoid-losing-data} +\label{durability:durability}\label{durability:important-rules-to-avoid-losing-data}\label{durability::doc}\label{durability:sshfs} Most S3QL backends store data in distributed storage systems. These systems differ from a traditional, local hard disk in several important ways. In order to avoid losing data, this section should be @@ -1100,7 +1100,7 @@ rise up to hours (\href{http://forums.aws.amazon.com/message.jspa?messageID=3847 \section{Data Durability} -\label{durability:data-durability}\label{durability:backend-reliability} +\label{durability:backend-reliability}\label{durability:data-durability} The durability of a storage service a measure of the average probability of a storage object to become corrupted over time. The lower the chance of data loss, the higher the durability. Storage @@ -1183,7 +1183,7 @@ neglected over long periods of time. \chapter{File System Creation} -\label{mkfs::doc}\label{mkfs:file-system-creation} +\label{mkfs:file-system-creation}\label{mkfs::doc} A S3QL file system is created with the \textbf{\texttt{mkfs.s3ql}} command. It has the following syntax: @@ -1260,7 +1260,7 @@ one at \sphinxcode{s3://bucketname/outerprefix/innerprefix}. \chapter{Managing File Systems} -\label{adm::doc}\label{adm:managing-file-systems} +\label{adm:managing-file-systems}\label{adm::doc} The \sphinxcode{s3qladm} command performs various operations on \emph{unmounted} S3QL file systems. The file system \emph{must not be mounted} when using \sphinxcode{s3qladm} or things will go wrong badly. @@ -1379,7 +1379,7 @@ for help on the mailing list first (see {\hyperref[resources:resources]{\sphinxc \chapter{Mounting} -\label{mount::doc}\label{mount:mounting} +\label{mount:mounting}\label{mount::doc} A S3QL file system is mounted with the \textbf{\texttt{mount.s3ql}} command. It has the following syntax: @@ -1590,7 +1590,7 @@ messages is the only way to find out about them. \section{Automatic Mounting} -\label{mount:automatic-mounting}\label{mount:logcheck} +\label{mount:logcheck}\label{mount:automatic-mounting} If you want to mount and umount an S3QL file system automatically at system startup and shutdown, you should do so with a dedicated S3QL init job (instead of using \sphinxcode{/etc/fstab}. When using systemd, @@ -1626,7 +1626,7 @@ mounted. \label{special::doc}\label{special:advanced-s3ql-features} \section{Snapshotting and Copy-on-Write} -\label{special:snapshotting-and-copy-on-write}\label{special:s3qlcp} +\label{special:s3qlcp}\label{special:snapshotting-and-copy-on-write} The command \sphinxcode{s3qlcp} can be used to duplicate a directory tree without physically copying the file contents. This is made possible by the data de-duplication feature of S3QL. @@ -1734,7 +1734,7 @@ For a full list of available options, run \sphinxcode{s3qlstat -{-}help}. \section{Immutable Trees} -\label{special:s3qllock}\label{special:immutable-trees} +\label{special:immutable-trees}\label{special:s3qllock} The command \textbf{\texttt{s3qllock}} can be used to make a directory tree immutable. Immutable trees can no longer be changed in any way whatsoever. You can not add new files or directories and you can not @@ -1797,7 +1797,7 @@ be removed entirely and immediately. \section{Runtime Configuration} -\label{special:s3qlctrl}\label{special:runtime-configuration} +\label{special:runtime-configuration}\label{special:s3qlctrl} The \sphinxcode{s3qlctrl} can be used to control a mounted S3QL file system. Its syntax is @@ -1814,6 +1814,10 @@ may be either of: Flush file system cache. The command blocks until the cache has been flushed. +\item[{dropcache}] \leavevmode +Flush, and then drop file system cache. The command +blocks until the cache has been flushed and dropped. + \item[{log}] \leavevmode Change log level. @@ -1828,7 +1832,7 @@ Trigger a metadata upload. \chapter{Unmounting} -\label{umount::doc}\label{umount:unmounting} +\label{umount:unmounting}\label{umount::doc} To unmount an S3QL file system, use the command: \begin{Verbatim}[commandchars=\\\{\}] @@ -1877,7 +1881,7 @@ upload data in the background for a while longer. \chapter{Checking for Errors} -\label{fsck::doc}\label{fsck:checking-for-errors} +\label{fsck:checking-for-errors}\label{fsck::doc} It is recommended to periodically run the \textbf{\texttt{fsck.s3ql}} and \textbf{\texttt{s3ql\_verify}} commands (in this order) to ensure that the file system is consistent, and that there has been no data corruption @@ -1955,7 +1959,7 @@ likely result in data loss. \section{Detecting and handling backend data corruption} -\label{fsck:s3ql-verify}\label{fsck:detecting-and-handling-backend-data-corruption} +\label{fsck:detecting-and-handling-backend-data-corruption}\label{fsck:s3ql-verify} The \textbf{\texttt{s3ql\_verify}} command verifies all data in the file system. In contrast to \textbf{\texttt{fsck.s3ql}}, \textbf{\texttt{s3ql\_verify}} does not trust the object listing returned by the backend, but @@ -2033,7 +2037,7 @@ Skip over first \textless{}n\textgreater{} objects and with verifying object \chapter{Storing Authentication Information} -\label{authinfo:authinfo}\label{authinfo::doc}\label{authinfo:storing-authentication-information} +\label{authinfo:storing-authentication-information}\label{authinfo:authinfo}\label{authinfo::doc} Normally, S3QL reads username and password for the backend as well as an encryption passphrase for the file system from the terminal. Most commands also accept an \sphinxcode{-{-}authfile} parameter that can be @@ -2100,7 +2104,7 @@ module}. \chapter{Contributed Programs} -\label{contrib::doc}\label{contrib:contributed-programs} +\label{contrib:contributed-programs}\label{contrib::doc} S3QL comes with a few contributed programs that are not part of the core distribution (and are therefore not installed automatically by default), but which may nevertheless be useful. These programs are in @@ -2261,10 +2265,10 @@ backend need not contain an S3QL file system. \chapter{Tips \& Tricks} -\label{tips::doc}\label{tips:tips-tricks} +\label{tips:tips-tricks}\label{tips::doc} \section{SSH Backend} -\label{tips:ssh-backend}\label{tips:ssh-tipp} +\label{tips:ssh-tipp}\label{tips:ssh-backend} By combining S3QL's local backend with \href{http://fuse.sourceforge.net/sshfs.html}{sshfs}, it is possible to store an S3QL file system on arbitrary SSH servers: first mount the remote target directory into the local filesystem, @@ -2362,7 +2366,7 @@ details. \chapter{Known Issues} -\label{issues::doc}\label{issues:known-issues}\begin{itemize} +\label{issues:known-issues}\label{issues::doc}\begin{itemize} \item {} S3QL de-duplicates data blocks based solely only on SHA256 checksums, without doing a byte-by-byte comparison of the blocks. @@ -3635,7 +3639,7 @@ system, common locations are \sphinxcode{/usr/share/doc/s3ql} or \section{The \textbf{\texttt{s3ql\_oauth\_client}} command} -\label{man/oauth_client:the-command-command}\label{man/oauth_client::doc}\label{man/oauth_client:oauth-client} +\label{man/oauth_client:oauth-client}\label{man/oauth_client:the-command-command}\label{man/oauth_client::doc} \subsection{Synopsis} \label{man/oauth_client:synopsis} @@ -4055,7 +4059,7 @@ Invalid command line argument. \chapter{Further Resources / Getting Help} -\label{resources::doc}\label{resources:further-resources-getting-help}\label{resources:resources} +\label{resources:resources}\label{resources::doc}\label{resources:further-resources-getting-help} If you have questions or problems with S3QL that you weren't able to resolve with this manual, you might want to consider the following other resources: \begin{itemize} @@ -4076,7 +4080,7 @@ Please report any bugs you may encounter in the \href{https://bitbucket.org/nikr \chapter{Implementation Details} -\label{impl_details::doc}\label{impl_details:implementation-details}\label{impl_details:impl-details} +\label{impl_details:impl-details}\label{impl_details:implementation-details}\label{impl_details::doc} This section provides some background information on how S3QL works internally. Reading this section is not necessary to use S3QL. diff --git a/doc/man/fsck.s3ql.1 b/doc/man/fsck.s3ql.1 index afa100d..8a0bf95 100644 --- a/doc/man/fsck.s3ql.1 +++ b/doc/man/fsck.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "FSCK.S3QL" "1" "Nov 04, 2017" "2.24" "S3QL" +.TH "FSCK.S3QL" "1" "Jan 05, 2018" "2.25" "S3QL" .SH NAME fsck.s3ql \- Check an S3QL file system for errors . diff --git a/doc/man/mkfs.s3ql.1 b/doc/man/mkfs.s3ql.1 index 4c2b38b..eea88f7 100644 --- a/doc/man/mkfs.s3ql.1 +++ b/doc/man/mkfs.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "MKFS.S3QL" "1" "Nov 04, 2017" "2.24" "S3QL" +.TH "MKFS.S3QL" "1" "Jan 05, 2018" "2.25" "S3QL" .SH NAME mkfs.s3ql \- Create an S3QL file system . diff --git a/doc/man/mount.s3ql.1 b/doc/man/mount.s3ql.1 index ba28133..ece39a1 100644 --- a/doc/man/mount.s3ql.1 +++ b/doc/man/mount.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "MOUNT.S3QL" "1" "Nov 04, 2017" "2.24" "S3QL" +.TH "MOUNT.S3QL" "1" "Jan 05, 2018" "2.25" "S3QL" .SH NAME mount.s3ql \- Mount an S3QL file system . diff --git a/doc/man/s3ql_oauth_client.1 b/doc/man/s3ql_oauth_client.1 index 1556ae6..513b13d 100644 --- a/doc/man/s3ql_oauth_client.1 +++ b/doc/man/s3ql_oauth_client.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QL_OAUTH_CLIENT" "1" "Nov 04, 2017" "2.24" "S3QL" +.TH "S3QL_OAUTH_CLIENT" "1" "Jan 05, 2018" "2.25" "S3QL" .SH NAME s3ql_oauth_client \- Obtain Google Storage OAuth2 tokens . diff --git a/doc/man/s3ql_verify.1 b/doc/man/s3ql_verify.1 index 38ebddf..3ef5ccb 100644 --- a/doc/man/s3ql_verify.1 +++ b/doc/man/s3ql_verify.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QL_VERIFY" "1" "Nov 04, 2017" "2.24" "S3QL" +.TH "S3QL_VERIFY" "1" "Jan 05, 2018" "2.25" "S3QL" .SH NAME s3ql_verify \- Verify data in an S3QL file system . diff --git a/doc/man/s3qladm.1 b/doc/man/s3qladm.1 index 5ac553c..e60167b 100644 --- a/doc/man/s3qladm.1 +++ b/doc/man/s3qladm.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLADM" "1" "Nov 04, 2017" "2.24" "S3QL" +.TH "S3QLADM" "1" "Jan 05, 2018" "2.25" "S3QL" .SH NAME s3qladm \- Manage S3QL file systems . diff --git a/doc/man/s3qlcp.1 b/doc/man/s3qlcp.1 index dc12671..8e81f07 100644 --- a/doc/man/s3qlcp.1 +++ b/doc/man/s3qlcp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLCP" "1" "Nov 04, 2017" "2.24" "S3QL" +.TH "S3QLCP" "1" "Jan 05, 2018" "2.25" "S3QL" .SH NAME s3qlcp \- Copy-on-write replication on S3QL file systems . diff --git a/doc/man/s3qlctrl.1 b/doc/man/s3qlctrl.1 index a764420..75ad2bd 100644 --- a/doc/man/s3qlctrl.1 +++ b/doc/man/s3qlctrl.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLCTRL" "1" "Nov 04, 2017" "2.24" "S3QL" +.TH "S3QLCTRL" "1" "Jan 05, 2018" "2.25" "S3QL" .SH NAME s3qlctrl \- Control a mounted S3QL file system . diff --git a/doc/man/s3qllock.1 b/doc/man/s3qllock.1 index a789752..24faaba 100644 --- a/doc/man/s3qllock.1 +++ b/doc/man/s3qllock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLLOCK" "1" "Nov 04, 2017" "2.24" "S3QL" +.TH "S3QLLOCK" "1" "Jan 05, 2018" "2.25" "S3QL" .SH NAME s3qllock \- Make trees on an S3QL file system immutable . diff --git a/doc/man/s3qlrm.1 b/doc/man/s3qlrm.1 index 61bfd1d..dc091d3 100644 --- a/doc/man/s3qlrm.1 +++ b/doc/man/s3qlrm.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLRM" "1" "Nov 04, 2017" "2.24" "S3QL" +.TH "S3QLRM" "1" "Jan 05, 2018" "2.25" "S3QL" .SH NAME s3qlrm \- Fast tree removal on S3QL file systems . diff --git a/doc/man/s3qlstat.1 b/doc/man/s3qlstat.1 index 53629ad..a05ccc5 100644 --- a/doc/man/s3qlstat.1 +++ b/doc/man/s3qlstat.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLSTAT" "1" "Nov 04, 2017" "2.24" "S3QL" +.TH "S3QLSTAT" "1" "Jan 05, 2018" "2.25" "S3QL" .SH NAME s3qlstat \- Gather S3QL file system statistics . diff --git a/doc/man/umount.s3ql.1 b/doc/man/umount.s3ql.1 index e1e6fbe..3bc5e9f 100644 --- a/doc/man/umount.s3ql.1 +++ b/doc/man/umount.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "UMOUNT.S3QL" "1" "Nov 04, 2017" "2.24" "S3QL" +.TH "UMOUNT.S3QL" "1" "Jan 05, 2018" "2.25" "S3QL" .SH NAME umount.s3ql \- Unmount an S3QL file system . diff --git a/doc/manual.pdf b/doc/manual.pdf Binary files differindex 1363116..a7e1a67 100644 --- a/doc/manual.pdf +++ b/doc/manual.pdf diff --git a/rst/special.rst b/rst/special.rst index 05c22e1..534d1ef 100644 --- a/rst/special.rst +++ b/rst/special.rst @@ -112,8 +112,11 @@ For a list of valid options, run `s3qlctrl --help`. `<action>` may be either of: :flushcache: - Flush file system cache. The command blocks until the cache has - been flushed. + Flush file system cache. The command blocks until the cache has + been flushed. + :dropcache: + Flush, and then drop file system cache. The command + blocks until the cache has been flushed and dropped. :log: Change log level. :cachesize: diff --git a/src/s3ql.egg-info/PKG-INFO b/src/s3ql.egg-info/PKG-INFO index 71674df..c9580d2 100644 --- a/src/s3ql.egg-info/PKG-INFO +++ b/src/s3ql.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: s3ql -Version: 2.24 +Version: 2.25 Summary: a full-featured file system for online data storage Home-page: https://bitbucket.org/nikratio/s3ql/ Author: Nikolaus Rath @@ -146,17 +146,15 @@ Description: .. Please report any bugs you may encounter in the `Bitbucket Issue Tracker`_. - Professional Support - ==================== - - Professional support is available. Please contact Nikolaus Rath - <Nikolaus@rath.org> for details. - Contributing ============ The S3QL source code is available both on GitHub_ and BitBucket_. + Professional Support + -------------------- + + Professional support is offered via `Rath Consulting`_. .. _`S3QL User's Guide`: http://www.rath.org/s3ql-docs/index.html .. _`S3QL Wiki`: https://bitbucket.org/nikratio/s3ql/wiki/ @@ -166,6 +164,7 @@ Description: .. .. _`Bitbucket Issue Tracker`: https://bitbucket.org/nikratio/s3ql/issues .. _BitBucket: https://bitbucket.org/nikratio/s3ql/ .. _GitHub: https://github.com/s3ql/main + .. _`Rath Consulting`: http://www.rath-consulting.biz/ Keywords: FUSE,backup,archival,compression,encryption,deduplication,aws,s3 Platform: POSIX diff --git a/src/s3ql/__init__.py b/src/s3ql/__init__.py index 3a1a67d..5681bc6 100644 --- a/src/s3ql/__init__.py +++ b/src/s3ql/__init__.py @@ -22,7 +22,7 @@ __all__ = [ 'adm', 'backends', 'block_cache', 'common', 'calc_mro', 'REV_VER_MAP', 'RELEASE', 'BUFSIZE', 'CTRL_NAME', 'CTRL_INODE' ] -VERSION = '2.24' +VERSION = '2.25' RELEASE = '%s' % VERSION # TODO: On next revision bump, consider removing support for TIME diff --git a/src/s3ql/block_cache.py b/src/s3ql/block_cache.py index e8a02b4..a82ade4 100644 --- a/src/s3ql/block_cache.py +++ b/src/s3ql/block_cache.py @@ -492,29 +492,39 @@ class BlockCache(object): if self.transfer_completed.wait(timeout=5): return - def upload(self, el): - '''Upload cache entry `el` asynchronously + def upload_if_dirty(self, el): + '''Upload cache entry asynchronously - This method releases the global lock. + This method releases the global lock. Return True if the object + is actually scheduled for upload. ''' log.debug('started with %s', el) + if el in self.in_transit or not el.dirty: + return False + # Calculate checksum with lock_released: + self._lock_entry(el.inode, el.blockno) + added_to_transit = False try: - self._lock_entry(el.inode, el.blockno) - assert el not in self.in_transit - if not el.dirty: - log.debug('not dirty, returning') + if el is not self.cache.get((el.inode, el.blockno), None): + log.debug('%s got removed while waiting for lock', el) self._unlock_entry(el.inode, el.blockno) - return - if (el.inode, el.blockno) not in self.cache: - log.debug('%s removed while waiting for lock', el) + return False + if el in self.in_transit: + log.debug('%s already in transit', el) self._unlock_entry(el.inode, el.blockno) - return + return False + if not el.dirty: + log.debug('no longer dirty, returning') + self._unlock_entry(el.inode, el.blockno) + return False + log.debug('uploading %s..', el) self.in_transit.add(el) + added_to_transit = True sha = hashlib.sha256() el.seek(0) while True: @@ -524,7 +534,8 @@ class BlockCache(object): sha.update(buf) hash_ = sha.digest() except: - self.in_transit.discard(el) + if added_to_transit: + self.in_transit.discard(el) self._unlock_entry(el.inode, el.blockno) raise @@ -575,7 +586,7 @@ class BlockCache(object): if old_block_id == block_id: log.debug('unchanged, block_id=%d', block_id) - return + return False except: self.in_transit.discard(el) @@ -585,12 +596,13 @@ class BlockCache(object): self._unlock_obj(obj_id) raise - # Check if we have to remove an old block - if not old_block_id: - log.debug('no old block, returning') - return + if old_block_id: + self._deref_block(old_block_id) + else: + log.debug('no old block') + + return obj_lock_taken - self._deref_block(old_block_id) def _queue_upload(self, obj): '''Put *obj* into upload queue''' @@ -829,23 +841,20 @@ class BlockCache(object): need_entries -= 1 need_size -= el.size - if el.dirty: - if el not in self.in_transit: - log.debug('uploading %s..', el) - self.upload(el) # Releases global lock + if self.upload_if_dirty(el): # Releases global lock sth_in_transit = True continue - log.debug('removing inode %d, block %d from cache', el.inode, el.blockno) self._lock_entry(el.inode, el.blockno, release_global=True) try: # May have changed while we were waiting for lock + if el is not self.cache.get((el.inode, el.blockno), None): + log.debug('%s removed while waiting for lock', el) + continue if el.dirty: log.debug('%s got dirty while waiting for lock', el) continue - if (el.inode, el.blockno) not in self.cache: - log.debug('%s removed while waiting for lock', el) - continue + log.debug('removing %s from cache', el) self.cache.remove((el.inode, el.blockno)) finally: self._unlock_entry(el.inode, el.blockno, release_global=True) @@ -918,14 +927,11 @@ class BlockCache(object): This method releases the global lock. """ - # Need to make copy, since we aren't allowed to change dict while - # iterating through it. Look at the comments in CommitThread.run() - # (mount.py) for an estimate of the resulting performance hit. + # Need to make copy, since dict() may change while global lock is + # released. Look at the comments in CommitThread.run() (mount.py) for an + # estimate of the performance impact. for el in list(self.cache.values()): - if not el.dirty or el in self.in_transit: - continue - - self.upload(el) # Releases global lock + self.upload_if_dirty(el) # Releases global lock def flush(self): """Upload all dirty blocks @@ -937,13 +943,13 @@ class BlockCache(object): while True: sth_in_transit = False - for el in self.cache.values(): - if not el.dirty: - continue - if el not in self.in_transit: - log.debug('uploading %s..', el) - self.upload(el) # Releases global lock - sth_in_transit = True + + # Need to make copy, since dict() may change while global lock is + # released. Look at the comments in CommitThread.run() (mount.py) + # for an estimate of the performance impact. + for el in list(self.cache.values()): + if self.upload_if_dirty(el): # Releases global lock + sth_in_transit = True if not sth_in_transit: break diff --git a/src/s3ql/common.py b/src/s3ql/common.py index 6a7825d..a5359c2 100644 --- a/src/s3ql/common.py +++ b/src/s3ql/common.py @@ -448,18 +448,19 @@ class ExceptionStoringThread(threading.Thread): except: # This creates a circular reference chain self._exc_info = sys.exc_info() + log.exception('Thread %s terminated with exception', self.name) def join_get_exc(self): self._joined = True self.join() return self._exc_info - def join_and_raise(self): + def join_and_raise(self, timeout=None): '''Wait for the thread to finish, raise any occurred exceptions''' self._joined = True if self.is_alive(): - self.join() + self.join(timeout=timeout) if self._exc_info is not None: # Break reference chain diff --git a/src/s3ql/ctrl.py b/src/s3ql/ctrl.py index 30852d4..35bd530 100644 --- a/src/s3ql/ctrl.py +++ b/src/s3ql/ctrl.py @@ -40,6 +40,8 @@ def parse_args(args): subparsers.required = True subparsers.add_parser('flushcache', help='flush file system cache', parents=[pparser]) + subparsers.add_parser('dropcache', help='drop file system cache', + parents=[pparser]) subparsers.add_parser('upload-meta', help='Upload metadata', parents=[pparser]) @@ -85,7 +87,10 @@ def main(args=None): if options.action == 'flushcache': llfuse.setxattr(ctrlfile, 's3ql_flushcache!', b'dummy') - if options.action == 'upload-meta': + elif options.action == 'dropcache': + llfuse.setxattr(ctrlfile, 's3ql_dropcache!', b'dummy') + + elif options.action == 'upload-meta': llfuse.setxattr(ctrlfile, 'upload-meta', b'dummy') elif options.action == 'log': diff --git a/src/s3ql/fs.py b/src/s3ql/fs.py index dff2b59..2b46e7f 100644 --- a/src/s3ql/fs.py +++ b/src/s3ql/fs.py @@ -240,6 +240,10 @@ class Operations(llfuse.Operations): self.inodes.flush() self.cache.flush() + elif name == b's3ql_dropcache!': + self.inodes.drop() + self.cache.drop() + elif name == b'copy': try: tup = parse_literal(value, (int, int)) diff --git a/src/s3ql/inode_cache.py b/src/s3ql/inode_cache.py index 77c8985..d82c08b 100644 --- a/src/s3ql/inode_cache.py +++ b/src/s3ql/inode_cache.py @@ -219,6 +219,8 @@ class InodeCache(object): def destroy(self): '''Flush all entries and empty cache''' + # Note: this method is currently also used for dropping the cache + for i in range(len(self.cached_rows)): id_ = self.cached_rows[i] self.cached_rows[i] = None @@ -250,6 +252,11 @@ class InodeCache(object): else: self.setattr(inode) + def drop(self): + '''Drop cache (after flushing)''' + + self.destroy() + def __del__(self): if len(self.attrs) == 0: return diff --git a/src/s3ql/mount.py b/src/s3ql/mount.py index 8dfd0da..7bbcc0e 100644 --- a/src/s3ql/mount.py +++ b/src/s3ql/mount.py @@ -740,11 +740,8 @@ class CommitThread(Thread): break if stamp - el.last_access < 10: break - if not el.dirty or el in self.block_cache.in_transit: - continue - - self.block_cache.upload(el) - did_sth = True + if self.block_cache.upload_if_dirty(el): + did_sth = True if not did_sth: self.stop_event.wait(5) diff --git a/tests/pytest.ini b/tests/pytest.ini index a87c1f8..7be3c8d 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,3 +1,6 @@ [pytest] addopts = --verbose --assert=rewrite --exitfirst --tb=native python_files = t?_*.py +log_cli_level = 100 +log_format = %(asctime)s.%(msecs)03d %(threadName)s %(name)s.%(funcName)s: %(message)s +log_date_format = %H:%M:%S diff --git a/tests/t2_block_cache.py b/tests/t2_block_cache.py index 87224c7..ba52b1b 100755 --- a/tests/t2_block_cache.py +++ b/tests/t2_block_cache.py @@ -21,9 +21,10 @@ from s3ql.mkfs import init_tables from s3ql.metadata import create_tables from s3ql.database import Connection from s3ql.common import AsyncFn, time_ns +import s3ql.block_cache from common import safe_sleep from pytest_checklogs import assert_logs -import llfuse +from unittest.mock import patch import errno import os import logging @@ -35,6 +36,8 @@ import unittest import queue import pytest +log = logging.getLogger(__name__) + # A dummy removal queue to monkeypatch around the need for removal and upload # threads class DummyQueue: @@ -107,10 +110,10 @@ class cache_tests(unittest.TestCase): # Tested methods assume that they are called from # file system request handler - llfuse.lock.acquire() + s3ql.block_cache.lock = MockLock() + s3ql.block_cache.lock_released = MockLock() def tearDown(self): - llfuse.lock.release() self.cache.backend_pool = self.backend_pool self.cache.destroy() shutil.rmtree(self.cachedir) @@ -163,8 +166,7 @@ class cache_tests(unittest.TestCase): try: # Try to clean-up (implicitly calls expire) - with llfuse.lock_released, \ - assert_logs('Unable to drop cache, no upload threads left alive', + with assert_logs('Unable to drop cache, no upload threads left alive', level=logging.ERROR, count=1): with pytest.raises(OSError) as exc_info: self.cache.destroy() @@ -259,7 +261,7 @@ class cache_tests(unittest.TestCase): fh.seek(0) fh.write(data1) el1 = fh - self.cache.upload(el1) + assert self.cache.upload_if_dirty(el1) self.cache.backend_pool.verify() # Case 2: Link new object @@ -268,7 +270,7 @@ class cache_tests(unittest.TestCase): fh.seek(0) fh.write(data1) el2 = fh - self.cache.upload(el2) + assert not self.cache.upload_if_dirty(el2) self.cache.backend_pool.verify() # Case 3: Upload old object, still has references @@ -276,7 +278,7 @@ class cache_tests(unittest.TestCase): with self.cache.get(inode, blockno1) as fh: fh.seek(0) fh.write(data2) - self.cache.upload(el1) + assert self.cache.upload_if_dirty(el1) self.cache.backend_pool.verify() # Case 4: Upload old object, no references left @@ -284,7 +286,7 @@ class cache_tests(unittest.TestCase): with self.cache.get(inode, blockno2) as fh: fh.seek(0) fh.write(data3) - self.cache.upload(el2) + assert self.cache.upload_if_dirty(el2) self.cache.backend_pool.verify() # Case 5: Link old object, no references left @@ -292,7 +294,7 @@ class cache_tests(unittest.TestCase): with self.cache.get(inode, blockno2) as fh: fh.seek(0) fh.write(data2) - self.cache.upload(el2) + assert not self.cache.upload_if_dirty(el2) self.cache.backend_pool.verify() # Case 6: Link old object, still has references @@ -302,14 +304,14 @@ class cache_tests(unittest.TestCase): fh.seek(0) fh.write(data1) el3 = fh - self.cache.upload(el3) + assert self.cache.upload_if_dirty(el3) self.cache.backend_pool.verify() self.cache.backend_pool = MockBackendPool(self.backend_pool) with self.cache.get(inode, blockno1) as fh: fh.seek(0) fh.write(data1) - self.cache.upload(el1) + assert not self.cache.upload_if_dirty(el1) self.cache.drop() self.cache.backend_pool.verify() @@ -359,7 +361,7 @@ class cache_tests(unittest.TestCase): self.cache.remove(inode, blockno) # Try to upload it, may happen if CommitThread is interrupted - self.cache.upload(fh) + self.cache.upload_if_dirty(fh) def test_expire_race(self): # Create element @@ -369,32 +371,27 @@ class cache_tests(unittest.TestCase): with self.cache.get(inode, blockno) as fh: fh.seek(0) fh.write(data1) - self.cache.upload(fh) + assert self.cache.upload_if_dirty(fh) # Make sure entry will be expired self.cache.cache.max_entries = 0 - def e_w_l(): - with llfuse.lock: - self.cache.expire() # Lock it self.cache._lock_entry(inode, blockno, release_global=True) try: # Start expiration, will block on lock - t1 = AsyncFn(e_w_l) + t1 = AsyncFn(self.cache.expire) t1.start() # Start second expiration, will block - t2 = AsyncFn(e_w_l) + t2 = AsyncFn(self.cache.expire) t2.start() # Release lock - with llfuse.lock_released: - safe_sleep(0.1) - self.cache._unlock_entry(inode, blockno) - t1.join_and_raise() - t2.join_and_raise() + self.cache._unlock_entry(inode, blockno) + t1.join_and_raise() + t2.join_and_raise() assert len(self.cache.cache) == 0 finally: @@ -413,28 +410,23 @@ class cache_tests(unittest.TestCase): # We want to expire just one element, but have # several threads running expire() simultaneously self.cache.cache.max_entries = 4 - def e_w_l(): - with llfuse.lock: - self.cache.expire() # Lock first element so that we have time to start threads self.cache._lock_entry(inode, 0, release_global=True) try: # Start expiration, will block on lock - t1 = AsyncFn(e_w_l) + t1 = AsyncFn(self.cache.expire) t1.start() # Start second expiration, will block - t2 = AsyncFn(e_w_l) + t2 = AsyncFn(self.cache.expire) t2.start() # Release lock - with llfuse.lock_released: - safe_sleep(0.1) - self.cache._unlock_entry(inode, 0) - t1.join_and_raise() - t2.join_and_raise() + self.cache._unlock_entry(inode, 0) + t1.join_and_raise() + t2.join_and_raise() assert len(self.cache.cache) == 4 finally: @@ -480,6 +472,94 @@ class cache_tests(unittest.TestCase): fh.seek(0) self.assertEqual(fh.read(42), b'') + def test_issue_241(self): + + inode = self.inode + + # Create block + with self.cache.get(inode, 0) as fh: + fh.write(self.random_data(500)) + + # "Fill" cache + self.cache.cache.max_entries = 0 + + # Mock locking to reproduce race condition + mlock = MockMultiLock(self.cache.mlock) + with patch.object(self.cache, 'mlock', mlock): + # Start first expiration run, will block in upload + thread1 = AsyncFn(self.cache.expire) + thread1.start() + + # Remove the object while the expiration thread waits + # for it to become available. + thread2 = AsyncFn(self.cache.remove, inode, 0, 1) + thread2.start() + mlock.yield_to(thread2) + thread2.join_and_raise(timeout=10) + assert not thread2.is_alive() + + # Create a new object for the same block + with self.cache.get(inode, 0) as fh: + fh.write(self.random_data(500)) + + # Continue first expiration run + mlock.yield_to(thread1, block=False) + thread1.join_and_raise(timeout=10) + assert not thread1.is_alive() + + +class MockMultiLock: + def __init__(self, real_mlock): + self.cond = real_mlock.cond + self.cleared = set() + self.real_mlock = real_mlock + + def yield_to(self, thread, block=True): + '''Allow *thread* to proceed''' + + me = threading.current_thread() + log.debug('%s blocked in yield_to(), phase 1', me.name) + with self.cond: + self.cleared.add(thread) + self.cond.notify_all() + + if not block: + return + log.debug('%s blocked in yield_to(), phase 2', me.name) + with self.cond: + if not self.cond.wait_for(lambda: thread not in self.cleared, 10): + pytest.fail('Timeout waiting for lock') + log.debug('%s completed yield_to()', me.name) + + @contextmanager + def __call__(self, *key): + self.acquire(*key) + try: + yield + finally: + self.release(*key) + + def acquire(self, *key, timeout=None): + me = threading.current_thread() + log.debug('%s blocked in acquire()', me.name) + with self.cond: + if not self.cond.wait_for(lambda: me in self.cleared, 10): + pytest.fail('Timeout waiting for lock') + self.real_mlock.locked_keys.add(key) + log.debug('%s got lock', me.name) + + def release(self, *key, noerror=False): + me = threading.current_thread() + log.debug('%s blocked in release()', me.name) + with self.cond: + self.cleared.remove(me) + self.cond.notify_all() + if noerror: + self.real_mlock.locked_keys.discard(key) + else: + self.real_mlock.locked_keys.remove(key) + log.debug('%s released lock', me.name) + class MockBackendPool(AbstractBackend): has_native_rename = False @@ -581,4 +661,21 @@ def start_flush(cache, inode, block=None): if block is not None and el.blockno != block: continue - cache.upload(el) + cache.upload_if_dirty(el) + + +class MockLock(): + def __call__(self): + return self + + def acquire(self, timeout=None): + pass + + def release(self): + pass + + def __enter__(self): + pass + + def __exit__(self, *args): + pass diff --git a/tests/t5_ctrl.py b/tests/t5_ctrl.py index 29834ce..48f1cbe 100755 --- a/tests/t5_ctrl.py +++ b/tests/t5_ctrl.py @@ -22,6 +22,7 @@ class TestCtrl(t4_fuse.TestFuse): self.mkfs() self.mount() self.tst_ctrl_flush() + self.tst_ctrl_drop() self.tst_ctrl_log() self.tst_ctrl_cachesize() self.umount() @@ -34,6 +35,13 @@ class TestCtrl(t4_fuse.TestFuse): sys.excepthook(*sys.exc_info()) pytest.fail("s3qlctrl raised exception") + def tst_ctrl_drop(self): + try: + s3ql.ctrl.main(['dropcache', self.mnt_dir]) + except: + sys.excepthook(*sys.exc_info()) + pytest.fail("s3qlctrl raised exception") + def tst_ctrl_log(self): try: s3ql.ctrl.main(['log', self.mnt_dir, 'warn']) |