summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorNikolaus Rath <Nikolaus@rath.org>2018-01-05 11:57:29 +0100
committerNikolaus Rath <Nikolaus@rath.org>2018-01-05 11:57:29 +0100
commitf0a962d37508caa5c59402b0353d3347be170415 (patch)
tree17611c2bb9757c1fb853847b60279c6fdd75d2cd
parentcff3c35a423c68dadec70a45a039c2db8dc19e32 (diff)
Import s3ql_2.25+dfsg.orig.tar.gz
-rw-r--r--Changes.txt7
-rw-r--r--PKG-INFO13
-rw-r--r--README.rst11
-rw-r--r--contrib/expire_backups.12
-rw-r--r--contrib/pcp.12
-rw-r--r--doc/latex/manual.aux64
-rw-r--r--doc/latex/manual.tex174
-rw-r--r--doc/man/fsck.s3ql.12
-rw-r--r--doc/man/mkfs.s3ql.12
-rw-r--r--doc/man/mount.s3ql.12
-rw-r--r--doc/man/s3ql_oauth_client.12
-rw-r--r--doc/man/s3ql_verify.12
-rw-r--r--doc/man/s3qladm.12
-rw-r--r--doc/man/s3qlcp.12
-rw-r--r--doc/man/s3qlctrl.12
-rw-r--r--doc/man/s3qllock.12
-rw-r--r--doc/man/s3qlrm.12
-rw-r--r--doc/man/s3qlstat.12
-rw-r--r--doc/man/umount.s3ql.12
-rw-r--r--doc/manual.pdfbin283819 -> 284062 bytes
-rw-r--r--rst/special.rst7
-rw-r--r--src/s3ql.egg-info/PKG-INFO13
-rw-r--r--src/s3ql/__init__.py2
-rw-r--r--src/s3ql/block_cache.py86
-rw-r--r--src/s3ql/common.py5
-rw-r--r--src/s3ql/ctrl.py7
-rw-r--r--src/s3ql/fs.py4
-rw-r--r--src/s3ql/inode_cache.py7
-rw-r--r--src/s3ql/mount.py7
-rw-r--r--tests/pytest.ini3
-rwxr-xr-xtests/t2_block_cache.py167
-rwxr-xr-xtests/t5_ctrl.py8
32 files changed, 376 insertions, 237 deletions
diff --git a/Changes.txt b/Changes.txt
index 77871ec..a2a188b 100644
--- a/Changes.txt
+++ b/Changes.txt
@@ -1,3 +1,10 @@
+2018-01-05, S3QL 2.25
+
+ * s3qlctrl now accepts a new *dropcache* command.
+ * Fixed a race condition that resulted in mount.s3ql crashing with
+ "I/O operation on closed file". Thanks to www.imCode.com for
+ sponsoring this work!
+
2017-11-04, S3QL 2.24
* fsck.s3ql now accepts a new --force-remote parameter. It does
diff --git a/PKG-INFO b/PKG-INFO
index 71674df..c9580d2 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: s3ql
-Version: 2.24
+Version: 2.25
Summary: a full-featured file system for online data storage
Home-page: https://bitbucket.org/nikratio/s3ql/
Author: Nikolaus Rath
@@ -146,17 +146,15 @@ Description: ..
Please report any bugs you may encounter in the `Bitbucket Issue Tracker`_.
- Professional Support
- ====================
-
- Professional support is available. Please contact Nikolaus Rath
- <Nikolaus@rath.org> for details.
-
Contributing
============
The S3QL source code is available both on GitHub_ and BitBucket_.
+ Professional Support
+ --------------------
+
+ Professional support is offered via `Rath Consulting`_.
.. _`S3QL User's Guide`: http://www.rath.org/s3ql-docs/index.html
.. _`S3QL Wiki`: https://bitbucket.org/nikratio/s3ql/wiki/
@@ -166,6 +164,7 @@ Description: ..
.. _`Bitbucket Issue Tracker`: https://bitbucket.org/nikratio/s3ql/issues
.. _BitBucket: https://bitbucket.org/nikratio/s3ql/
.. _GitHub: https://github.com/s3ql/main
+ .. _`Rath Consulting`: http://www.rath-consulting.biz/
Keywords: FUSE,backup,archival,compression,encryption,deduplication,aws,s3
Platform: POSIX
diff --git a/README.rst b/README.rst
index 7b49c06..fd015c9 100644
--- a/README.rst
+++ b/README.rst
@@ -137,17 +137,15 @@ The following resources are available:
Please report any bugs you may encounter in the `Bitbucket Issue Tracker`_.
-Professional Support
-====================
-
-Professional support is available. Please contact Nikolaus Rath
-<Nikolaus@rath.org> for details.
-
Contributing
============
The S3QL source code is available both on GitHub_ and BitBucket_.
+Professional Support
+--------------------
+
+Professional support is offered via `Rath Consulting`_.
.. _`S3QL User's Guide`: http://www.rath.org/s3ql-docs/index.html
.. _`S3QL Wiki`: https://bitbucket.org/nikratio/s3ql/wiki/
@@ -157,3 +155,4 @@ The S3QL source code is available both on GitHub_ and BitBucket_.
.. _`Bitbucket Issue Tracker`: https://bitbucket.org/nikratio/s3ql/issues
.. _BitBucket: https://bitbucket.org/nikratio/s3ql/
.. _GitHub: https://github.com/s3ql/main
+.. _`Rath Consulting`: http://www.rath-consulting.biz/
diff --git a/contrib/expire_backups.1 b/contrib/expire_backups.1
index cb1e7e2..2aed509 100644
--- a/contrib/expire_backups.1
+++ b/contrib/expire_backups.1
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
-.TH "EXPIRE_BACKUPS" "1" "Nov 04, 2017" "2.24" "S3QL"
+.TH "EXPIRE_BACKUPS" "1" "Jan 05, 2018" "2.25" "S3QL"
.SH NAME
expire_backups \- Intelligently expire old backups
.
diff --git a/contrib/pcp.1 b/contrib/pcp.1
index c95f74c..e5d5f52 100644
--- a/contrib/pcp.1
+++ b/contrib/pcp.1
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
-.TH "PCP" "1" "Nov 04, 2017" "2.24" "S3QL"
+.TH "PCP" "1" "Jan 05, 2018" "2.25" "S3QL"
.SH NAME
pcp \- Recursive, parallel copy of directory trees
.
diff --git a/doc/latex/manual.aux b/doc/latex/manual.aux
index ee2490b..1b0d1cf 100644
--- a/doc/latex/manual.aux
+++ b/doc/latex/manual.aux
@@ -25,12 +25,12 @@
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{loliteral-block}{\addvspace {10\p@ }}
-\newlabel{about::doc}{{1}{1}{S3QL}{chapter.1}{}}
\newlabel{about:s3ql-user-s-guide}{{1}{1}{S3QL}{chapter.1}{}}
+\newlabel{about::doc}{{1}{1}{S3QL}{chapter.1}{}}
\newlabel{about:s3ql}{{1}{1}{S3QL}{chapter.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {1.1}Features}{1}{section.1.1}}
-\newlabel{about:openstack}{{1.1}{1}{Features}{section.1.1}{}}
\newlabel{about:features}{{1.1}{1}{Features}{section.1.1}{}}
+\newlabel{about:openstack}{{1.1}{1}{Features}{section.1.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {1.2}Development Status}{2}{section.1.2}}
\newlabel{about:development-status}{{1.2}{2}{Development Status}{section.1.2}{}}
\@writefile{toc}{\contentsline {section}{\numberline {1.3}Supported Platforms}{2}{section.1.3}}
@@ -41,14 +41,14 @@
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{loliteral-block}{\addvspace {10\p@ }}
-\newlabel{installation::doc}{{2}{3}{Installation}{chapter.2}{}}
-\newlabel{installation:installation}{{2}{3}{Installation}{chapter.2}{}}
\newlabel{installation:github}{{2}{3}{Installation}{chapter.2}{}}
+\newlabel{installation:installation}{{2}{3}{Installation}{chapter.2}{}}
+\newlabel{installation::doc}{{2}{3}{Installation}{chapter.2}{}}
\@writefile{toc}{\contentsline {section}{\numberline {2.1}Dependencies}{3}{section.2.1}}
\newlabel{installation:dependencies}{{2.1}{3}{Dependencies}{section.2.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {2.2}Installing S3QL}{4}{section.2.2}}
-\newlabel{installation:inst-s3ql}{{2.2}{4}{Installing S3QL}{section.2.2}{}}
\newlabel{installation:installing-s3ql}{{2.2}{4}{Installing S3QL}{section.2.2}{}}
+\newlabel{installation:inst-s3ql}{{2.2}{4}{Installing S3QL}{section.2.2}{}}
\@writefile{toc}{\contentsline {section}{\numberline {2.3}Development Version}{4}{section.2.3}}
\newlabel{installation:development-version}{{2.3}{4}{Development Version}{section.2.3}{}}
\@writefile{toc}{\contentsline {section}{\numberline {2.4}Running tests requiring remote servers}{4}{section.2.4}}
@@ -57,18 +57,18 @@
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{loliteral-block}{\addvspace {10\p@ }}
+\newlabel{backends:sphinx}{{3}{7}{Storage Backends}{chapter.3}{}}
\newlabel{backends:id1}{{3}{7}{Storage Backends}{chapter.3}{}}
-\newlabel{backends::doc}{{3}{7}{Storage Backends}{chapter.3}{}}
\newlabel{backends:storage-backends}{{3}{7}{Storage Backends}{chapter.3}{}}
-\newlabel{backends:sphinx}{{3}{7}{Storage Backends}{chapter.3}{}}
+\newlabel{backends::doc}{{3}{7}{Storage Backends}{chapter.3}{}}
\@writefile{toc}{\contentsline {section}{\numberline {3.1}Google Storage}{7}{section.3.1}}
\newlabel{backends:google-storage}{{3.1}{7}{Google Storage}{section.3.1}{}}
\newlabel{backends:cmdoption-gs_backend-arg-no-ssl}{{3.1}{7}{Google Storage}{section*.3}{}}
\newlabel{backends:cmdoption-gs_backend-arg-ssl-ca-path}{{3.1}{8}{Google Storage}{section*.4}{}}
\newlabel{backends:cmdoption-gs_backend-arg-tcp-timeout}{{3.1}{8}{Google Storage}{section*.5}{}}
\@writefile{toc}{\contentsline {section}{\numberline {3.2}Amazon S3}{8}{section.3.2}}
-\newlabel{backends:amazon-s3}{{3.2}{8}{Amazon S3}{section.3.2}{}}
\newlabel{backends:google-storage-manager}{{3.2}{8}{Amazon S3}{section.3.2}{}}
+\newlabel{backends:amazon-s3}{{3.2}{8}{Amazon S3}{section.3.2}{}}
\newlabel{backends:cmdoption-s3_backend-arg-no-ssl}{{3.2}{8}{Amazon S3}{section*.6}{}}
\newlabel{backends:cmdoption-s3_backend-arg-ssl-ca-path}{{3.2}{8}{Amazon S3}{section*.7}{}}
\newlabel{backends:cmdoption-s3_backend-arg-tcp-timeout}{{3.2}{8}{Amazon S3}{section*.8}{}}
@@ -86,24 +86,24 @@
\@writefile{toc}{\contentsline {section}{\numberline {3.4}Rackspace CloudFiles}{10}{section.3.4}}
\newlabel{backends:rackspace-cloudfiles}{{3.4}{10}{Rackspace CloudFiles}{section.3.4}{}}
\@writefile{toc}{\contentsline {section}{\numberline {3.5}S3 compatible}{10}{section.3.5}}
-\newlabel{backends:rackspace}{{3.5}{10}{S3 compatible}{section.3.5}{}}
\newlabel{backends:s3-compatible}{{3.5}{10}{S3 compatible}{section.3.5}{}}
+\newlabel{backends:rackspace}{{3.5}{10}{S3 compatible}{section.3.5}{}}
\newlabel{backends:cmdoption-s3c_backend-arg-no-ssl}{{3.5}{10}{S3 compatible}{section*.17}{}}
\newlabel{backends:cmdoption-s3c_backend-arg-ssl-ca-path}{{3.5}{10}{S3 compatible}{section*.18}{}}
\newlabel{backends:cmdoption-s3c_backend-arg-tcp-timeout}{{3.5}{10}{S3 compatible}{section*.19}{}}
\newlabel{backends:cmdoption-s3c_backend-arg-disable-expect100}{{3.5}{10}{S3 compatible}{section*.20}{}}
\newlabel{backends:cmdoption-s3c_backend-arg-dumb-copy}{{3.5}{10}{S3 compatible}{section*.21}{}}
\@writefile{toc}{\contentsline {section}{\numberline {3.6}Local}{11}{section.3.6}}
-\newlabel{backends:local}{{3.6}{11}{Local}{section.3.6}{}}
\newlabel{backends:id6}{{3.6}{11}{Local}{section.3.6}{}}
+\newlabel{backends:local}{{3.6}{11}{Local}{section.3.6}{}}
\@writefile{toc}{\contentsline {chapter}{\numberline {4}Important Rules to Avoid Losing Data}{13}{chapter.4}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{loliteral-block}{\addvspace {10\p@ }}
\newlabel{durability:durability}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}}
-\newlabel{durability:sshfs}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}}
-\newlabel{durability::doc}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}}
\newlabel{durability:important-rules-to-avoid-losing-data}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}}
+\newlabel{durability::doc}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}}
+\newlabel{durability:sshfs}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}}
\@writefile{toc}{\contentsline {section}{\numberline {4.1}Rules in a Nutshell}{13}{section.4.1}}
\newlabel{durability:rules-in-a-nutshell}{{4.1}{13}{Rules in a Nutshell}{section.4.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {4.2}Consistency Window List}{14}{section.4.2}}
@@ -111,20 +111,20 @@
\@writefile{toc}{\contentsline {section}{\numberline {4.3}Data Consistency}{14}{section.4.3}}
\newlabel{durability:data-consistency}{{4.3}{14}{Data Consistency}{section.4.3}{}}
\@writefile{toc}{\contentsline {section}{\numberline {4.4}Data Durability}{15}{section.4.4}}
-\newlabel{durability:data-durability}{{4.4}{15}{Data Durability}{section.4.4}{}}
\newlabel{durability:backend-reliability}{{4.4}{15}{Data Durability}{section.4.4}{}}
+\newlabel{durability:data-durability}{{4.4}{15}{Data Durability}{section.4.4}{}}
\@writefile{toc}{\contentsline {chapter}{\numberline {5}File System Creation}{17}{chapter.5}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{loliteral-block}{\addvspace {10\p@ }}
-\newlabel{mkfs::doc}{{5}{17}{File System Creation}{chapter.5}{}}
\newlabel{mkfs:file-system-creation}{{5}{17}{File System Creation}{chapter.5}{}}
+\newlabel{mkfs::doc}{{5}{17}{File System Creation}{chapter.5}{}}
\@writefile{toc}{\contentsline {chapter}{\numberline {6}Managing File Systems}{19}{chapter.6}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{loliteral-block}{\addvspace {10\p@ }}
-\newlabel{adm::doc}{{6}{19}{Managing File Systems}{chapter.6}{}}
\newlabel{adm:managing-file-systems}{{6}{19}{Managing File Systems}{chapter.6}{}}
+\newlabel{adm::doc}{{6}{19}{Managing File Systems}{chapter.6}{}}
\@writefile{toc}{\contentsline {section}{\numberline {6.1}Changing the Passphrase}{19}{section.6.1}}
\newlabel{adm:changing-the-passphrase}{{6.1}{19}{Changing the Passphrase}{section.6.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {6.2}Upgrading the file system}{20}{section.6.2}}
@@ -137,8 +137,8 @@
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{loliteral-block}{\addvspace {10\p@ }}
-\newlabel{mount::doc}{{7}{21}{Mounting}{chapter.7}{}}
\newlabel{mount:mounting}{{7}{21}{Mounting}{chapter.7}{}}
+\newlabel{mount::doc}{{7}{21}{Mounting}{chapter.7}{}}
\@writefile{toc}{\contentsline {section}{\numberline {7.1}Permission Checking}{22}{section.7.1}}
\newlabel{mount:permission-checking}{{7.1}{22}{Permission Checking}{section.7.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {7.2}Compression Algorithms}{22}{section.7.2}}
@@ -152,8 +152,8 @@
\@writefile{toc}{\contentsline {section}{\numberline {7.4}Failure Modes}{23}{section.7.4}}
\newlabel{mount:failure-modes}{{7.4}{23}{Failure Modes}{section.7.4}{}}
\@writefile{toc}{\contentsline {section}{\numberline {7.5}Automatic Mounting}{24}{section.7.5}}
-\newlabel{mount:automatic-mounting}{{7.5}{24}{Automatic Mounting}{section.7.5}{}}
\newlabel{mount:logcheck}{{7.5}{24}{Automatic Mounting}{section.7.5}{}}
+\newlabel{mount:automatic-mounting}{{7.5}{24}{Automatic Mounting}{section.7.5}{}}
\@writefile{toc}{\contentsline {chapter}{\numberline {8}Advanced S3QL Features}{25}{chapter.8}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
@@ -161,52 +161,52 @@
\newlabel{special::doc}{{8}{25}{Advanced S3QL Features}{chapter.8}{}}
\newlabel{special:advanced-s3ql-features}{{8}{25}{Advanced S3QL Features}{chapter.8}{}}
\@writefile{toc}{\contentsline {section}{\numberline {8.1}Snapshotting and Copy-on-Write}{25}{section.8.1}}
-\newlabel{special:snapshotting-and-copy-on-write}{{8.1}{25}{Snapshotting and Copy-on-Write}{section.8.1}{}}
\newlabel{special:s3qlcp}{{8.1}{25}{Snapshotting and Copy-on-Write}{section.8.1}{}}
+\newlabel{special:snapshotting-and-copy-on-write}{{8.1}{25}{Snapshotting and Copy-on-Write}{section.8.1}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {8.1.1}Snapshotting vs Hardlinking}{25}{subsection.8.1.1}}
\newlabel{special:snapshotting-vs-hardlinking}{{8.1.1}{25}{Snapshotting vs Hardlinking}{subsection.8.1.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {8.2}Getting Statistics}{26}{section.8.2}}
\newlabel{special:s3qlstat}{{8.2}{26}{Getting Statistics}{section.8.2}{}}
\newlabel{special:getting-statistics}{{8.2}{26}{Getting Statistics}{section.8.2}{}}
\@writefile{toc}{\contentsline {section}{\numberline {8.3}Immutable Trees}{26}{section.8.3}}
-\newlabel{special:s3qllock}{{8.3}{26}{Immutable Trees}{section.8.3}{}}
\newlabel{special:immutable-trees}{{8.3}{26}{Immutable Trees}{section.8.3}{}}
+\newlabel{special:s3qllock}{{8.3}{26}{Immutable Trees}{section.8.3}{}}
\@writefile{toc}{\contentsline {section}{\numberline {8.4}Fast Recursive Removal}{27}{section.8.4}}
\newlabel{special:s3qlrm}{{8.4}{27}{Fast Recursive Removal}{section.8.4}{}}
\newlabel{special:fast-recursive-removal}{{8.4}{27}{Fast Recursive Removal}{section.8.4}{}}
\@writefile{toc}{\contentsline {section}{\numberline {8.5}Runtime Configuration}{27}{section.8.5}}
-\newlabel{special:s3qlctrl}{{8.5}{27}{Runtime Configuration}{section.8.5}{}}
\newlabel{special:runtime-configuration}{{8.5}{27}{Runtime Configuration}{section.8.5}{}}
+\newlabel{special:s3qlctrl}{{8.5}{27}{Runtime Configuration}{section.8.5}{}}
\@writefile{toc}{\contentsline {chapter}{\numberline {9}Unmounting}{29}{chapter.9}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{loliteral-block}{\addvspace {10\p@ }}
-\newlabel{umount::doc}{{9}{29}{Unmounting}{chapter.9}{}}
\newlabel{umount:unmounting}{{9}{29}{Unmounting}{chapter.9}{}}
+\newlabel{umount::doc}{{9}{29}{Unmounting}{chapter.9}{}}
\@writefile{toc}{\contentsline {chapter}{\numberline {10}Checking for Errors}{31}{chapter.10}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{loliteral-block}{\addvspace {10\p@ }}
-\newlabel{fsck::doc}{{10}{31}{Checking for Errors}{chapter.10}{}}
\newlabel{fsck:checking-for-errors}{{10}{31}{Checking for Errors}{chapter.10}{}}
+\newlabel{fsck::doc}{{10}{31}{Checking for Errors}{chapter.10}{}}
\@writefile{toc}{\contentsline {section}{\numberline {10.1}Checking and repairing internal file system errors}{31}{section.10.1}}
\newlabel{fsck:checking-and-repairing-internal-file-system-errors}{{10.1}{31}{Checking and repairing internal file system errors}{section.10.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {10.2}Detecting and handling backend data corruption}{32}{section.10.2}}
-\newlabel{fsck:s3ql-verify}{{10.2}{32}{Detecting and handling backend data corruption}{section.10.2}{}}
\newlabel{fsck:detecting-and-handling-backend-data-corruption}{{10.2}{32}{Detecting and handling backend data corruption}{section.10.2}{}}
+\newlabel{fsck:s3ql-verify}{{10.2}{32}{Detecting and handling backend data corruption}{section.10.2}{}}
\@writefile{toc}{\contentsline {chapter}{\numberline {11}Storing Authentication Information}{35}{chapter.11}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{loliteral-block}{\addvspace {10\p@ }}
+\newlabel{authinfo:storing-authentication-information}{{11}{35}{Storing Authentication Information}{chapter.11}{}}
\newlabel{authinfo:authinfo}{{11}{35}{Storing Authentication Information}{chapter.11}{}}
\newlabel{authinfo::doc}{{11}{35}{Storing Authentication Information}{chapter.11}{}}
-\newlabel{authinfo:storing-authentication-information}{{11}{35}{Storing Authentication Information}{chapter.11}{}}
\@writefile{toc}{\contentsline {chapter}{\numberline {12}Contributed Programs}{37}{chapter.12}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{loliteral-block}{\addvspace {10\p@ }}
-\newlabel{contrib::doc}{{12}{37}{Contributed Programs}{chapter.12}{}}
\newlabel{contrib:contributed-programs}{{12}{37}{Contributed Programs}{chapter.12}{}}
+\newlabel{contrib::doc}{{12}{37}{Contributed Programs}{chapter.12}{}}
\@writefile{toc}{\contentsline {section}{\numberline {12.1}benchmark.py}{37}{section.12.1}}
\newlabel{contrib:benchmark-py}{{12.1}{37}{benchmark.py}{section.12.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {12.2}clone\_fs.py}{37}{section.12.2}}
@@ -225,11 +225,11 @@
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{loliteral-block}{\addvspace {10\p@ }}
-\newlabel{tips::doc}{{13}{41}{Tips \& Tricks}{chapter.13}{}}
\newlabel{tips:tips-tricks}{{13}{41}{Tips \& Tricks}{chapter.13}{}}
+\newlabel{tips::doc}{{13}{41}{Tips \& Tricks}{chapter.13}{}}
\@writefile{toc}{\contentsline {section}{\numberline {13.1}SSH Backend}{41}{section.13.1}}
-\newlabel{tips:ssh-backend}{{13.1}{41}{SSH Backend}{section.13.1}{}}
\newlabel{tips:ssh-tipp}{{13.1}{41}{SSH Backend}{section.13.1}{}}
+\newlabel{tips:ssh-backend}{{13.1}{41}{SSH Backend}{section.13.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {13.2}Permanently mounted backup file system}{41}{section.13.2}}
\newlabel{tips:permanently-mounted-backup-file-system}{{13.2}{41}{Permanently mounted backup file system}{section.13.2}{}}
\@writefile{toc}{\contentsline {section}{\numberline {13.3}Improving copy performance}{41}{section.13.3}}
@@ -239,8 +239,8 @@
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{loliteral-block}{\addvspace {10\p@ }}
-\newlabel{issues::doc}{{14}{43}{Known Issues}{chapter.14}{}}
\newlabel{issues:known-issues}{{14}{43}{Known Issues}{chapter.14}{}}
+\newlabel{issues::doc}{{14}{43}{Known Issues}{chapter.14}{}}
\@writefile{toc}{\contentsline {chapter}{\numberline {15}Manpages}{45}{chapter.15}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
@@ -384,9 +384,9 @@
\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.5}See Also}{59}{subsection.15.10.5}}
\newlabel{man/fsck:see-also}{{15.10.5}{59}{See Also}{subsection.15.10.5}{}}
\@writefile{toc}{\contentsline {section}{\numberline {15.11}The \textbf {\texttt {s3ql\_oauth\_client}} command}{59}{section.15.11}}
+\newlabel{man/oauth_client:oauth-client}{{15.11}{59}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}}
\newlabel{man/oauth_client:the-command-command}{{15.11}{59}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}}
\newlabel{man/oauth_client::doc}{{15.11}{59}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}}
-\newlabel{man/oauth_client:oauth-client}{{15.11}{59}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.1}Synopsis}{59}{subsection.15.11.1}}
\newlabel{man/oauth_client:synopsis}{{15.11.1}{59}{Synopsis}{subsection.15.11.1}{}}
\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.2}Description}{59}{subsection.15.11.2}}
@@ -440,16 +440,16 @@
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{loliteral-block}{\addvspace {10\p@ }}
+\newlabel{resources:resources}{{16}{65}{Further Resources / Getting Help}{chapter.16}{}}
\newlabel{resources::doc}{{16}{65}{Further Resources / Getting Help}{chapter.16}{}}
\newlabel{resources:further-resources-getting-help}{{16}{65}{Further Resources / Getting Help}{chapter.16}{}}
-\newlabel{resources:resources}{{16}{65}{Further Resources / Getting Help}{chapter.16}{}}
\@writefile{toc}{\contentsline {chapter}{\numberline {17}Implementation Details}{67}{chapter.17}}
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\@writefile{loliteral-block}{\addvspace {10\p@ }}
-\newlabel{impl_details::doc}{{17}{67}{Implementation Details}{chapter.17}{}}
-\newlabel{impl_details:implementation-details}{{17}{67}{Implementation Details}{chapter.17}{}}
\newlabel{impl_details:impl-details}{{17}{67}{Implementation Details}{chapter.17}{}}
+\newlabel{impl_details:implementation-details}{{17}{67}{Implementation Details}{chapter.17}{}}
+\newlabel{impl_details::doc}{{17}{67}{Implementation Details}{chapter.17}{}}
\@writefile{toc}{\contentsline {section}{\numberline {17.1}Metadata Storage}{67}{section.17.1}}
\newlabel{impl_details:metadata-storage}{{17.1}{67}{Metadata Storage}{section.17.1}{}}
\@writefile{toc}{\contentsline {section}{\numberline {17.2}Data Storage}{67}{section.17.2}}
diff --git a/doc/latex/manual.tex b/doc/latex/manual.tex
index d6f917a..929d707 100644
--- a/doc/latex/manual.tex
+++ b/doc/latex/manual.tex
@@ -32,8 +32,8 @@
\title{S3QL Documentation}
-\date{Nov 04, 2017}
-\release{2.24}
+\date{Jan 05, 2018}
+\release{2.25}
\author{Nikolaus Rath}
\newcommand{\sphinxlogo}{}
\renewcommand{\releasename}{Release}
@@ -50,73 +50,73 @@
\PYG@it{\PYG@bf{\PYG@ff{#1}}}}}}}
\def\PYG#1#2{\PYG@reset\PYG@toks#1+\relax+\PYG@do{#2}}
-\expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.20}{##1}}}
-\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}}
-\expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.73,0.00,0.40}{##1}}}
-\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
-\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
-\expandafter\def\csname PYG@tok@gp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}}
-\expandafter\def\csname PYG@tok@kt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.60}{##1}}}
-\expandafter\def\csname PYG@tok@mi\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}}
-\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
-\expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.33,0.47,0.60}{##1}}}
-\expandafter\def\csname PYG@tok@dl\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
-\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}}
-\expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}}
-\expandafter\def\csname PYG@tok@c1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}}
-\expandafter\def\csname PYG@tok@vm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}}
\expandafter\def\csname PYG@tok@nl\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.60,0.47,0.00}{##1}}}
-\expandafter\def\csname PYG@tok@nt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.47,0.00}{##1}}}
-\expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}}
\expandafter\def\csname PYG@tok@cs\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.80,0.00,0.00}{##1}}}
-\expandafter\def\csname PYG@tok@mf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}}
-\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
+\expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.80}{##1}}}
+\expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}}
+\expandafter\def\csname PYG@tok@sh\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
\expandafter\def\csname PYG@tok@kr\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}}
-\expandafter\def\csname PYG@tok@sd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.27,0.13}{##1}}}
-\expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}}
-\expandafter\def\csname PYG@tok@mb\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}}
-\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}}
-\expandafter\def\csname PYG@tok@sc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}}
-\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}}
+\expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}}
+\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}}
+\expandafter\def\csname PYG@tok@kp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.53}{##1}}}
+\expandafter\def\csname PYG@tok@nt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.47,0.00}{##1}}}
+\expandafter\def\csname PYG@tok@vm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}}
\expandafter\def\csname PYG@tok@vg\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.87,0.47,0.00}{##1}}}
+\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}}
+\expandafter\def\csname PYG@tok@il\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}}
+\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}}
+\expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.33,0.47,0.60}{##1}}}
+\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}}
\expandafter\def\csname PYG@tok@gu\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}}
-\expandafter\def\csname PYG@tok@mo\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.27,0.00,0.93}{##1}}}
-\expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.33,0.33,0.33}{##1}}}
+\expandafter\def\csname PYG@tok@mi\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}}
+\expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit}
+\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
\expandafter\def\csname PYG@tok@gs\endcsname{\let\PYG@bf=\textbf}
-\expandafter\def\csname PYG@tok@kc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}}
-\expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}}
-\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}}
-\expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.73}{##1}}}
-\expandafter\def\csname PYG@tok@vc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.40,0.60}{##1}}}
+\expandafter\def\csname PYG@tok@no\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.40}{##1}}}
+\expandafter\def\csname PYG@tok@ch\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}}
+\expandafter\def\csname PYG@tok@c1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}}
+\expandafter\def\csname PYG@tok@sc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}}
+\expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.73,0.00,0.40}{##1}}}
+\expandafter\def\csname PYG@tok@sr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,1.00}{\strut ##1}}}
+\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}}
+\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
+\expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.33,0.33,0.33}{##1}}}
+\expandafter\def\csname PYG@tok@c\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}}
\expandafter\def\csname PYG@tok@bp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
-\expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}}
-\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.67,0.40,0.00}{##1}}}
\expandafter\def\csname PYG@tok@gh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}}
-\expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
-\expandafter\def\csname PYG@tok@no\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.40}{##1}}}
-\expandafter\def\csname PYG@tok@ne\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}}
+\expandafter\def\csname PYG@tok@gp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}}
+\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}}
+\expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}}
+\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}}
\expandafter\def\csname PYG@tok@sa\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
-\expandafter\def\csname PYG@tok@sh\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
+\expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}}
+\expandafter\def\csname PYG@tok@kc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}}
\expandafter\def\csname PYG@tok@err\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.67,0.67}{\strut ##1}}}
-\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.33,0.53}{##1}}}
-\expandafter\def\csname PYG@tok@kp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.53}{##1}}}
-\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}}
-\expandafter\def\csname PYG@tok@fm\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}}
-\expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.80}{##1}}}
-\expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}}
-\expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}}
-\expandafter\def\csname PYG@tok@s\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
-\expandafter\def\csname PYG@tok@si\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.93,0.93,0.93}{\strut ##1}}}
-\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}}
-\expandafter\def\csname PYG@tok@c\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}}
-\expandafter\def\csname PYG@tok@sr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,1.00}{\strut ##1}}}
+\expandafter\def\csname PYG@tok@mb\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}}
+\expandafter\def\csname PYG@tok@kt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.60}{##1}}}
+\expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.20}{##1}}}
\expandafter\def\csname PYG@tok@ow\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}}
-\expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit}
-\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}}
-\expandafter\def\csname PYG@tok@il\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}}
-\expandafter\def\csname PYG@tok@ch\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}}
\expandafter\def\csname PYG@tok@se\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
+\expandafter\def\csname PYG@tok@mo\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.27,0.00,0.93}{##1}}}
\expandafter\def\csname PYG@tok@cpf\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}}
+\expandafter\def\csname PYG@tok@fm\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}}
+\expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}}
+\expandafter\def\csname PYG@tok@s\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
+\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.33,0.53}{##1}}}
+\expandafter\def\csname PYG@tok@sd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.27,0.13}{##1}}}
+\expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.73}{##1}}}
+\expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}}
+\expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}}
+\expandafter\def\csname PYG@tok@ne\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}}
+\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
+\expandafter\def\csname PYG@tok@dl\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
+\expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}}
+\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}}
+\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
+\expandafter\def\csname PYG@tok@si\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.93,0.93,0.93}{\strut ##1}}}
+\expandafter\def\csname PYG@tok@mf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}}
+\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.67,0.40,0.00}{##1}}}
+\expandafter\def\csname PYG@tok@vc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.40,0.60}{##1}}}
\def\PYGZbs{\char`\\}
\def\PYGZus{\char`\_}
@@ -150,7 +150,7 @@
\chapter{S3QL}
-\label{about::doc}\label{about:s3ql-user-s-guide}\label{about:s3ql}
+\label{about:s3ql-user-s-guide}\label{about::doc}\label{about:s3ql}
S3QL is a file system that stores all its data online using storage
services like \href{http://code.google.com/apis/storage/}{Google Storage}, \href{http://aws.amazon.com/s3}{Amazon S3}, or \href{http://openstack.org/projects/storage/}{OpenStack}. S3QL
effectively provides a hard disk of dynamic, infinite capacity that
@@ -170,7 +170,7 @@ with extensive automated test cases for all its components.
\section{Features}
-\label{about:openstack}\label{about:features}\begin{itemize}
+\label{about:features}\label{about:openstack}\begin{itemize}
\item {}
\textbf{Transparency.} Conceptually, S3QL is indistinguishable from a
local file system. For example, it supports hardlinks, symlinks,
@@ -256,12 +256,12 @@ The S3QL source code is available both on \href{https://github.com/s3ql/main}{Gi
\chapter{Installation}
-\label{installation::doc}\label{installation:installation}\label{installation:github}
+\label{installation:github}\label{installation:installation}\label{installation::doc}
S3QL depends on several other programs and libraries that have to be
installed first. The best method to satisfy these dependencies depends
on your distribution.
-The following instructions are for S3QL 2.24 and should be
+The following instructions are for S3QL 2.25 and should be
applicable to any system. The \href{https://bitbucket.org/nikratio/s3ql/wiki/Home}{S3QL Wiki} contains \href{https://bitbucket.org/nikratio/s3ql/wiki/Installation}{additional
help} help
for specific distributions and operating systems. Note, however, that
@@ -347,7 +347,7 @@ installed version if the module is installed.
\section{Installing S3QL}
-\label{installation:inst-s3ql}\label{installation:installing-s3ql}
+\label{installation:installing-s3ql}\label{installation:inst-s3ql}
To build and install S3QL itself, proceed as follows:
\begin{enumerate}
\item {}
@@ -467,7 +467,7 @@ being skipped by passing the \sphinxcode{-rs} argument to
\chapter{Storage Backends}
-\label{backends:id1}\label{backends::doc}\label{backends:storage-backends}\label{backends:sphinx}
+\label{backends:sphinx}\label{backends:id1}\label{backends:storage-backends}\label{backends::doc}
S3QL supports different \emph{backends} to store data at different service
providers and using different protocols. A \emph{storage url} specifies a
backend together with some backend-specific information and uniquely
@@ -572,7 +572,7 @@ TCP connection is closed and re-established (default: 20 seconds).
\section{Amazon S3}
-\label{backends:amazon-s3}\label{backends:google-storage-manager}
+\label{backends:google-storage-manager}\label{backends:amazon-s3}
\href{http://aws.amazon.com/s3}{Amazon S3} is the online storage service
offered by \href{http://aws.amazon.com/}{Amazon Web Services (AWS)}. To
use the S3 backend, you first need to sign up for an AWS account. The
@@ -806,7 +806,7 @@ The Rackspace backend accepts the same backend options as the
\section{S3 compatible}
-\label{backends:rackspace}\label{backends:s3-compatible}
+\label{backends:s3-compatible}\label{backends:rackspace}
The S3 compatible backend allows S3QL to access any storage service
that uses the same protocol as Amazon S3. The storage URL has the form
@@ -883,7 +883,7 @@ body for a succesfull copy operation.
\section{Local}
-\label{backends:local}\label{backends:id6}
+\label{backends:id6}\label{backends:local}
S3QL is also able to store its data on the local file system. This can
be used to backup data on external media, or to access external
services that S3QL can not talk to directly (e.g., it is possible to
@@ -909,7 +909,7 @@ The local backend does not accept any backend options.
\chapter{Important Rules to Avoid Losing Data}
-\label{durability:durability}\label{durability:sshfs}\label{durability::doc}\label{durability:important-rules-to-avoid-losing-data}
+\label{durability:durability}\label{durability:important-rules-to-avoid-losing-data}\label{durability::doc}\label{durability:sshfs}
Most S3QL backends store data in distributed storage systems. These
systems differ from a traditional, local hard disk in several
important ways. In order to avoid losing data, this section should be
@@ -1100,7 +1100,7 @@ rise up to hours (\href{http://forums.aws.amazon.com/message.jspa?messageID=3847
\section{Data Durability}
-\label{durability:data-durability}\label{durability:backend-reliability}
+\label{durability:backend-reliability}\label{durability:data-durability}
The durability of a storage service a measure of the average
probability of a storage object to become corrupted over time. The
lower the chance of data loss, the higher the durability. Storage
@@ -1183,7 +1183,7 @@ neglected over long periods of time.
\chapter{File System Creation}
-\label{mkfs::doc}\label{mkfs:file-system-creation}
+\label{mkfs:file-system-creation}\label{mkfs::doc}
A S3QL file system is created with the \textbf{\texttt{mkfs.s3ql}} command. It has the
following syntax:
@@ -1260,7 +1260,7 @@ one at \sphinxcode{s3://bucketname/outerprefix/innerprefix}.
\chapter{Managing File Systems}
-\label{adm::doc}\label{adm:managing-file-systems}
+\label{adm:managing-file-systems}\label{adm::doc}
The \sphinxcode{s3qladm} command performs various operations on \emph{unmounted} S3QL
file systems. The file system \emph{must not be mounted} when using
\sphinxcode{s3qladm} or things will go wrong badly.
@@ -1379,7 +1379,7 @@ for help on the mailing list first (see {\hyperref[resources:resources]{\sphinxc
\chapter{Mounting}
-\label{mount::doc}\label{mount:mounting}
+\label{mount:mounting}\label{mount::doc}
A S3QL file system is mounted with the \textbf{\texttt{mount.s3ql}}
command. It has the following syntax:
@@ -1590,7 +1590,7 @@ messages is the only way to find out about them.
\section{Automatic Mounting}
-\label{mount:automatic-mounting}\label{mount:logcheck}
+\label{mount:logcheck}\label{mount:automatic-mounting}
If you want to mount and umount an S3QL file system automatically at
system startup and shutdown, you should do so with a dedicated S3QL
init job (instead of using \sphinxcode{/etc/fstab}. When using systemd,
@@ -1626,7 +1626,7 @@ mounted.
\label{special::doc}\label{special:advanced-s3ql-features}
\section{Snapshotting and Copy-on-Write}
-\label{special:snapshotting-and-copy-on-write}\label{special:s3qlcp}
+\label{special:s3qlcp}\label{special:snapshotting-and-copy-on-write}
The command \sphinxcode{s3qlcp} can be used to duplicate a directory tree without
physically copying the file contents. This is made possible by the
data de-duplication feature of S3QL.
@@ -1734,7 +1734,7 @@ For a full list of available options, run \sphinxcode{s3qlstat -{-}help}.
\section{Immutable Trees}
-\label{special:s3qllock}\label{special:immutable-trees}
+\label{special:immutable-trees}\label{special:s3qllock}
The command \textbf{\texttt{s3qllock}} can be used to make a directory tree
immutable. Immutable trees can no longer be changed in any way
whatsoever. You can not add new files or directories and you can not
@@ -1797,7 +1797,7 @@ be removed entirely and immediately.
\section{Runtime Configuration}
-\label{special:s3qlctrl}\label{special:runtime-configuration}
+\label{special:runtime-configuration}\label{special:s3qlctrl}
The \sphinxcode{s3qlctrl} can be used to control a mounted S3QL file system. Its
syntax is
@@ -1814,6 +1814,10 @@ may be either of:
Flush file system cache. The command blocks until the cache has
been flushed.
+\item[{dropcache}] \leavevmode
+Flush, and then drop file system cache. The command
+blocks until the cache has been flushed and dropped.
+
\item[{log}] \leavevmode
Change log level.
@@ -1828,7 +1832,7 @@ Trigger a metadata upload.
\chapter{Unmounting}
-\label{umount::doc}\label{umount:unmounting}
+\label{umount:unmounting}\label{umount::doc}
To unmount an S3QL file system, use the command:
\begin{Verbatim}[commandchars=\\\{\}]
@@ -1877,7 +1881,7 @@ upload data in the background for a while longer.
\chapter{Checking for Errors}
-\label{fsck::doc}\label{fsck:checking-for-errors}
+\label{fsck:checking-for-errors}\label{fsck::doc}
It is recommended to periodically run the \textbf{\texttt{fsck.s3ql}} and
\textbf{\texttt{s3ql\_verify}} commands (in this order) to ensure that the
file system is consistent, and that there has been no data corruption
@@ -1955,7 +1959,7 @@ likely result in data loss.
\section{Detecting and handling backend data corruption}
-\label{fsck:s3ql-verify}\label{fsck:detecting-and-handling-backend-data-corruption}
+\label{fsck:detecting-and-handling-backend-data-corruption}\label{fsck:s3ql-verify}
The \textbf{\texttt{s3ql\_verify}} command verifies all data in the file
system. In contrast to \textbf{\texttt{fsck.s3ql}}, \textbf{\texttt{s3ql\_verify}}
does not trust the object listing returned by the backend, but
@@ -2033,7 +2037,7 @@ Skip over first \textless{}n\textgreater{} objects and with verifying object
\chapter{Storing Authentication Information}
-\label{authinfo:authinfo}\label{authinfo::doc}\label{authinfo:storing-authentication-information}
+\label{authinfo:storing-authentication-information}\label{authinfo:authinfo}\label{authinfo::doc}
Normally, S3QL reads username and password for the backend as well as
an encryption passphrase for the file system from the terminal. Most
commands also accept an \sphinxcode{-{-}authfile} parameter that can be
@@ -2100,7 +2104,7 @@ module}.
\chapter{Contributed Programs}
-\label{contrib::doc}\label{contrib:contributed-programs}
+\label{contrib:contributed-programs}\label{contrib::doc}
S3QL comes with a few contributed programs that are not part of the
core distribution (and are therefore not installed automatically by
default), but which may nevertheless be useful. These programs are in
@@ -2261,10 +2265,10 @@ backend need not contain an S3QL file system.
\chapter{Tips \& Tricks}
-\label{tips::doc}\label{tips:tips-tricks}
+\label{tips:tips-tricks}\label{tips::doc}
\section{SSH Backend}
-\label{tips:ssh-backend}\label{tips:ssh-tipp}
+\label{tips:ssh-tipp}\label{tips:ssh-backend}
By combining S3QL's local backend with \href{http://fuse.sourceforge.net/sshfs.html}{sshfs}, it is possible to store an
S3QL file system on arbitrary SSH servers: first mount the remote
target directory into the local filesystem,
@@ -2362,7 +2366,7 @@ details.
\chapter{Known Issues}
-\label{issues::doc}\label{issues:known-issues}\begin{itemize}
+\label{issues:known-issues}\label{issues::doc}\begin{itemize}
\item {}
S3QL de-duplicates data blocks based solely only on SHA256
checksums, without doing a byte-by-byte comparison of the blocks.
@@ -3635,7 +3639,7 @@ system, common locations are \sphinxcode{/usr/share/doc/s3ql} or
\section{The \textbf{\texttt{s3ql\_oauth\_client}} command}
-\label{man/oauth_client:the-command-command}\label{man/oauth_client::doc}\label{man/oauth_client:oauth-client}
+\label{man/oauth_client:oauth-client}\label{man/oauth_client:the-command-command}\label{man/oauth_client::doc}
\subsection{Synopsis}
\label{man/oauth_client:synopsis}
@@ -4055,7 +4059,7 @@ Invalid command line argument.
\chapter{Further Resources / Getting Help}
-\label{resources::doc}\label{resources:further-resources-getting-help}\label{resources:resources}
+\label{resources:resources}\label{resources::doc}\label{resources:further-resources-getting-help}
If you have questions or problems with S3QL that you weren't able to
resolve with this manual, you might want to consider the following other resources:
\begin{itemize}
@@ -4076,7 +4080,7 @@ Please report any bugs you may encounter in the \href{https://bitbucket.org/nikr
\chapter{Implementation Details}
-\label{impl_details::doc}\label{impl_details:implementation-details}\label{impl_details:impl-details}
+\label{impl_details:impl-details}\label{impl_details:implementation-details}\label{impl_details::doc}
This section provides some background information on how S3QL works
internally. Reading this section is not necessary to use S3QL.
diff --git a/doc/man/fsck.s3ql.1 b/doc/man/fsck.s3ql.1
index afa100d..8a0bf95 100644
--- a/doc/man/fsck.s3ql.1
+++ b/doc/man/fsck.s3ql.1
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
-.TH "FSCK.S3QL" "1" "Nov 04, 2017" "2.24" "S3QL"
+.TH "FSCK.S3QL" "1" "Jan 05, 2018" "2.25" "S3QL"
.SH NAME
fsck.s3ql \- Check an S3QL file system for errors
.
diff --git a/doc/man/mkfs.s3ql.1 b/doc/man/mkfs.s3ql.1
index 4c2b38b..eea88f7 100644
--- a/doc/man/mkfs.s3ql.1
+++ b/doc/man/mkfs.s3ql.1
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
-.TH "MKFS.S3QL" "1" "Nov 04, 2017" "2.24" "S3QL"
+.TH "MKFS.S3QL" "1" "Jan 05, 2018" "2.25" "S3QL"
.SH NAME
mkfs.s3ql \- Create an S3QL file system
.
diff --git a/doc/man/mount.s3ql.1 b/doc/man/mount.s3ql.1
index ba28133..ece39a1 100644
--- a/doc/man/mount.s3ql.1
+++ b/doc/man/mount.s3ql.1
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
-.TH "MOUNT.S3QL" "1" "Nov 04, 2017" "2.24" "S3QL"
+.TH "MOUNT.S3QL" "1" "Jan 05, 2018" "2.25" "S3QL"
.SH NAME
mount.s3ql \- Mount an S3QL file system
.
diff --git a/doc/man/s3ql_oauth_client.1 b/doc/man/s3ql_oauth_client.1
index 1556ae6..513b13d 100644
--- a/doc/man/s3ql_oauth_client.1
+++ b/doc/man/s3ql_oauth_client.1
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
-.TH "S3QL_OAUTH_CLIENT" "1" "Nov 04, 2017" "2.24" "S3QL"
+.TH "S3QL_OAUTH_CLIENT" "1" "Jan 05, 2018" "2.25" "S3QL"
.SH NAME
s3ql_oauth_client \- Obtain Google Storage OAuth2 tokens
.
diff --git a/doc/man/s3ql_verify.1 b/doc/man/s3ql_verify.1
index 38ebddf..3ef5ccb 100644
--- a/doc/man/s3ql_verify.1
+++ b/doc/man/s3ql_verify.1
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
-.TH "S3QL_VERIFY" "1" "Nov 04, 2017" "2.24" "S3QL"
+.TH "S3QL_VERIFY" "1" "Jan 05, 2018" "2.25" "S3QL"
.SH NAME
s3ql_verify \- Verify data in an S3QL file system
.
diff --git a/doc/man/s3qladm.1 b/doc/man/s3qladm.1
index 5ac553c..e60167b 100644
--- a/doc/man/s3qladm.1
+++ b/doc/man/s3qladm.1
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
-.TH "S3QLADM" "1" "Nov 04, 2017" "2.24" "S3QL"
+.TH "S3QLADM" "1" "Jan 05, 2018" "2.25" "S3QL"
.SH NAME
s3qladm \- Manage S3QL file systems
.
diff --git a/doc/man/s3qlcp.1 b/doc/man/s3qlcp.1
index dc12671..8e81f07 100644
--- a/doc/man/s3qlcp.1
+++ b/doc/man/s3qlcp.1
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
-.TH "S3QLCP" "1" "Nov 04, 2017" "2.24" "S3QL"
+.TH "S3QLCP" "1" "Jan 05, 2018" "2.25" "S3QL"
.SH NAME
s3qlcp \- Copy-on-write replication on S3QL file systems
.
diff --git a/doc/man/s3qlctrl.1 b/doc/man/s3qlctrl.1
index a764420..75ad2bd 100644
--- a/doc/man/s3qlctrl.1
+++ b/doc/man/s3qlctrl.1
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
-.TH "S3QLCTRL" "1" "Nov 04, 2017" "2.24" "S3QL"
+.TH "S3QLCTRL" "1" "Jan 05, 2018" "2.25" "S3QL"
.SH NAME
s3qlctrl \- Control a mounted S3QL file system
.
diff --git a/doc/man/s3qllock.1 b/doc/man/s3qllock.1
index a789752..24faaba 100644
--- a/doc/man/s3qllock.1
+++ b/doc/man/s3qllock.1
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
-.TH "S3QLLOCK" "1" "Nov 04, 2017" "2.24" "S3QL"
+.TH "S3QLLOCK" "1" "Jan 05, 2018" "2.25" "S3QL"
.SH NAME
s3qllock \- Make trees on an S3QL file system immutable
.
diff --git a/doc/man/s3qlrm.1 b/doc/man/s3qlrm.1
index 61bfd1d..dc091d3 100644
--- a/doc/man/s3qlrm.1
+++ b/doc/man/s3qlrm.1
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
-.TH "S3QLRM" "1" "Nov 04, 2017" "2.24" "S3QL"
+.TH "S3QLRM" "1" "Jan 05, 2018" "2.25" "S3QL"
.SH NAME
s3qlrm \- Fast tree removal on S3QL file systems
.
diff --git a/doc/man/s3qlstat.1 b/doc/man/s3qlstat.1
index 53629ad..a05ccc5 100644
--- a/doc/man/s3qlstat.1
+++ b/doc/man/s3qlstat.1
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
-.TH "S3QLSTAT" "1" "Nov 04, 2017" "2.24" "S3QL"
+.TH "S3QLSTAT" "1" "Jan 05, 2018" "2.25" "S3QL"
.SH NAME
s3qlstat \- Gather S3QL file system statistics
.
diff --git a/doc/man/umount.s3ql.1 b/doc/man/umount.s3ql.1
index e1e6fbe..3bc5e9f 100644
--- a/doc/man/umount.s3ql.1
+++ b/doc/man/umount.s3ql.1
@@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
-.TH "UMOUNT.S3QL" "1" "Nov 04, 2017" "2.24" "S3QL"
+.TH "UMOUNT.S3QL" "1" "Jan 05, 2018" "2.25" "S3QL"
.SH NAME
umount.s3ql \- Unmount an S3QL file system
.
diff --git a/doc/manual.pdf b/doc/manual.pdf
index 1363116..a7e1a67 100644
--- a/doc/manual.pdf
+++ b/doc/manual.pdf
Binary files differ
diff --git a/rst/special.rst b/rst/special.rst
index 05c22e1..534d1ef 100644
--- a/rst/special.rst
+++ b/rst/special.rst
@@ -112,8 +112,11 @@ For a list of valid options, run `s3qlctrl --help`. `<action>`
may be either of:
:flushcache:
- Flush file system cache. The command blocks until the cache has
- been flushed.
+ Flush file system cache. The command blocks until the cache has
+ been flushed.
+ :dropcache:
+ Flush, and then drop file system cache. The command
+ blocks until the cache has been flushed and dropped.
:log:
Change log level.
:cachesize:
diff --git a/src/s3ql.egg-info/PKG-INFO b/src/s3ql.egg-info/PKG-INFO
index 71674df..c9580d2 100644
--- a/src/s3ql.egg-info/PKG-INFO
+++ b/src/s3ql.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: s3ql
-Version: 2.24
+Version: 2.25
Summary: a full-featured file system for online data storage
Home-page: https://bitbucket.org/nikratio/s3ql/
Author: Nikolaus Rath
@@ -146,17 +146,15 @@ Description: ..
Please report any bugs you may encounter in the `Bitbucket Issue Tracker`_.
- Professional Support
- ====================
-
- Professional support is available. Please contact Nikolaus Rath
- <Nikolaus@rath.org> for details.
-
Contributing
============
The S3QL source code is available both on GitHub_ and BitBucket_.
+ Professional Support
+ --------------------
+
+ Professional support is offered via `Rath Consulting`_.
.. _`S3QL User's Guide`: http://www.rath.org/s3ql-docs/index.html
.. _`S3QL Wiki`: https://bitbucket.org/nikratio/s3ql/wiki/
@@ -166,6 +164,7 @@ Description: ..
.. _`Bitbucket Issue Tracker`: https://bitbucket.org/nikratio/s3ql/issues
.. _BitBucket: https://bitbucket.org/nikratio/s3ql/
.. _GitHub: https://github.com/s3ql/main
+ .. _`Rath Consulting`: http://www.rath-consulting.biz/
Keywords: FUSE,backup,archival,compression,encryption,deduplication,aws,s3
Platform: POSIX
diff --git a/src/s3ql/__init__.py b/src/s3ql/__init__.py
index 3a1a67d..5681bc6 100644
--- a/src/s3ql/__init__.py
+++ b/src/s3ql/__init__.py
@@ -22,7 +22,7 @@ __all__ = [ 'adm', 'backends', 'block_cache', 'common', 'calc_mro',
'REV_VER_MAP', 'RELEASE', 'BUFSIZE',
'CTRL_NAME', 'CTRL_INODE' ]
-VERSION = '2.24'
+VERSION = '2.25'
RELEASE = '%s' % VERSION
# TODO: On next revision bump, consider removing support for TIME
diff --git a/src/s3ql/block_cache.py b/src/s3ql/block_cache.py
index e8a02b4..a82ade4 100644
--- a/src/s3ql/block_cache.py
+++ b/src/s3ql/block_cache.py
@@ -492,29 +492,39 @@ class BlockCache(object):
if self.transfer_completed.wait(timeout=5):
return
- def upload(self, el):
- '''Upload cache entry `el` asynchronously
+ def upload_if_dirty(self, el):
+ '''Upload cache entry asynchronously
- This method releases the global lock.
+ This method releases the global lock. Return True if the object
+ is actually scheduled for upload.
'''
log.debug('started with %s', el)
+ if el in self.in_transit or not el.dirty:
+ return False
+
# Calculate checksum
with lock_released:
+ self._lock_entry(el.inode, el.blockno)
+ added_to_transit = False
try:
- self._lock_entry(el.inode, el.blockno)
- assert el not in self.in_transit
- if not el.dirty:
- log.debug('not dirty, returning')
+ if el is not self.cache.get((el.inode, el.blockno), None):
+ log.debug('%s got removed while waiting for lock', el)
self._unlock_entry(el.inode, el.blockno)
- return
- if (el.inode, el.blockno) not in self.cache:
- log.debug('%s removed while waiting for lock', el)
+ return False
+ if el in self.in_transit:
+ log.debug('%s already in transit', el)
self._unlock_entry(el.inode, el.blockno)
- return
+ return False
+ if not el.dirty:
+ log.debug('no longer dirty, returning')
+ self._unlock_entry(el.inode, el.blockno)
+ return False
+ log.debug('uploading %s..', el)
self.in_transit.add(el)
+ added_to_transit = True
sha = hashlib.sha256()
el.seek(0)
while True:
@@ -524,7 +534,8 @@ class BlockCache(object):
sha.update(buf)
hash_ = sha.digest()
except:
- self.in_transit.discard(el)
+ if added_to_transit:
+ self.in_transit.discard(el)
self._unlock_entry(el.inode, el.blockno)
raise
@@ -575,7 +586,7 @@ class BlockCache(object):
if old_block_id == block_id:
log.debug('unchanged, block_id=%d', block_id)
- return
+ return False
except:
self.in_transit.discard(el)
@@ -585,12 +596,13 @@ class BlockCache(object):
self._unlock_obj(obj_id)
raise
- # Check if we have to remove an old block
- if not old_block_id:
- log.debug('no old block, returning')
- return
+ if old_block_id:
+ self._deref_block(old_block_id)
+ else:
+ log.debug('no old block')
+
+ return obj_lock_taken
- self._deref_block(old_block_id)
def _queue_upload(self, obj):
'''Put *obj* into upload queue'''
@@ -829,23 +841,20 @@ class BlockCache(object):
need_entries -= 1
need_size -= el.size
- if el.dirty:
- if el not in self.in_transit:
- log.debug('uploading %s..', el)
- self.upload(el) # Releases global lock
+ if self.upload_if_dirty(el): # Releases global lock
sth_in_transit = True
continue
- log.debug('removing inode %d, block %d from cache', el.inode, el.blockno)
self._lock_entry(el.inode, el.blockno, release_global=True)
try:
# May have changed while we were waiting for lock
+ if el is not self.cache.get((el.inode, el.blockno), None):
+ log.debug('%s removed while waiting for lock', el)
+ continue
if el.dirty:
log.debug('%s got dirty while waiting for lock', el)
continue
- if (el.inode, el.blockno) not in self.cache:
- log.debug('%s removed while waiting for lock', el)
- continue
+ log.debug('removing %s from cache', el)
self.cache.remove((el.inode, el.blockno))
finally:
self._unlock_entry(el.inode, el.blockno, release_global=True)
@@ -918,14 +927,11 @@ class BlockCache(object):
This method releases the global lock.
"""
- # Need to make copy, since we aren't allowed to change dict while
- # iterating through it. Look at the comments in CommitThread.run()
- # (mount.py) for an estimate of the resulting performance hit.
+ # Need to make copy, since dict() may change while global lock is
+ # released. Look at the comments in CommitThread.run() (mount.py) for an
+ # estimate of the performance impact.
for el in list(self.cache.values()):
- if not el.dirty or el in self.in_transit:
- continue
-
- self.upload(el) # Releases global lock
+ self.upload_if_dirty(el) # Releases global lock
def flush(self):
"""Upload all dirty blocks
@@ -937,13 +943,13 @@ class BlockCache(object):
while True:
sth_in_transit = False
- for el in self.cache.values():
- if not el.dirty:
- continue
- if el not in self.in_transit:
- log.debug('uploading %s..', el)
- self.upload(el) # Releases global lock
- sth_in_transit = True
+
+ # Need to make copy, since dict() may change while global lock is
+ # released. Look at the comments in CommitThread.run() (mount.py)
+ # for an estimate of the performance impact.
+ for el in list(self.cache.values()):
+ if self.upload_if_dirty(el): # Releases global lock
+ sth_in_transit = True
if not sth_in_transit:
break
diff --git a/src/s3ql/common.py b/src/s3ql/common.py
index 6a7825d..a5359c2 100644
--- a/src/s3ql/common.py
+++ b/src/s3ql/common.py
@@ -448,18 +448,19 @@ class ExceptionStoringThread(threading.Thread):
except:
# This creates a circular reference chain
self._exc_info = sys.exc_info()
+ log.exception('Thread %s terminated with exception', self.name)
def join_get_exc(self):
self._joined = True
self.join()
return self._exc_info
- def join_and_raise(self):
+ def join_and_raise(self, timeout=None):
'''Wait for the thread to finish, raise any occurred exceptions'''
self._joined = True
if self.is_alive():
- self.join()
+ self.join(timeout=timeout)
if self._exc_info is not None:
# Break reference chain
diff --git a/src/s3ql/ctrl.py b/src/s3ql/ctrl.py
index 30852d4..35bd530 100644
--- a/src/s3ql/ctrl.py
+++ b/src/s3ql/ctrl.py
@@ -40,6 +40,8 @@ def parse_args(args):
subparsers.required = True
subparsers.add_parser('flushcache', help='flush file system cache',
parents=[pparser])
+ subparsers.add_parser('dropcache', help='drop file system cache',
+ parents=[pparser])
subparsers.add_parser('upload-meta', help='Upload metadata',
parents=[pparser])
@@ -85,7 +87,10 @@ def main(args=None):
if options.action == 'flushcache':
llfuse.setxattr(ctrlfile, 's3ql_flushcache!', b'dummy')
- if options.action == 'upload-meta':
+ elif options.action == 'dropcache':
+ llfuse.setxattr(ctrlfile, 's3ql_dropcache!', b'dummy')
+
+ elif options.action == 'upload-meta':
llfuse.setxattr(ctrlfile, 'upload-meta', b'dummy')
elif options.action == 'log':
diff --git a/src/s3ql/fs.py b/src/s3ql/fs.py
index dff2b59..2b46e7f 100644
--- a/src/s3ql/fs.py
+++ b/src/s3ql/fs.py
@@ -240,6 +240,10 @@ class Operations(llfuse.Operations):
self.inodes.flush()
self.cache.flush()
+ elif name == b's3ql_dropcache!':
+ self.inodes.drop()
+ self.cache.drop()
+
elif name == b'copy':
try:
tup = parse_literal(value, (int, int))
diff --git a/src/s3ql/inode_cache.py b/src/s3ql/inode_cache.py
index 77c8985..d82c08b 100644
--- a/src/s3ql/inode_cache.py
+++ b/src/s3ql/inode_cache.py
@@ -219,6 +219,8 @@ class InodeCache(object):
def destroy(self):
'''Flush all entries and empty cache'''
+ # Note: this method is currently also used for dropping the cache
+
for i in range(len(self.cached_rows)):
id_ = self.cached_rows[i]
self.cached_rows[i] = None
@@ -250,6 +252,11 @@ class InodeCache(object):
else:
self.setattr(inode)
+ def drop(self):
+ '''Drop cache (after flushing)'''
+
+ self.destroy()
+
def __del__(self):
if len(self.attrs) == 0:
return
diff --git a/src/s3ql/mount.py b/src/s3ql/mount.py
index 8dfd0da..7bbcc0e 100644
--- a/src/s3ql/mount.py
+++ b/src/s3ql/mount.py
@@ -740,11 +740,8 @@ class CommitThread(Thread):
break
if stamp - el.last_access < 10:
break
- if not el.dirty or el in self.block_cache.in_transit:
- continue
-
- self.block_cache.upload(el)
- did_sth = True
+ if self.block_cache.upload_if_dirty(el):
+ did_sth = True
if not did_sth:
self.stop_event.wait(5)
diff --git a/tests/pytest.ini b/tests/pytest.ini
index a87c1f8..7be3c8d 100644
--- a/tests/pytest.ini
+++ b/tests/pytest.ini
@@ -1,3 +1,6 @@
[pytest]
addopts = --verbose --assert=rewrite --exitfirst --tb=native
python_files = t?_*.py
+log_cli_level = 100
+log_format = %(asctime)s.%(msecs)03d %(threadName)s %(name)s.%(funcName)s: %(message)s
+log_date_format = %H:%M:%S
diff --git a/tests/t2_block_cache.py b/tests/t2_block_cache.py
index 87224c7..ba52b1b 100755
--- a/tests/t2_block_cache.py
+++ b/tests/t2_block_cache.py
@@ -21,9 +21,10 @@ from s3ql.mkfs import init_tables
from s3ql.metadata import create_tables
from s3ql.database import Connection
from s3ql.common import AsyncFn, time_ns
+import s3ql.block_cache
from common import safe_sleep
from pytest_checklogs import assert_logs
-import llfuse
+from unittest.mock import patch
import errno
import os
import logging
@@ -35,6 +36,8 @@ import unittest
import queue
import pytest
+log = logging.getLogger(__name__)
+
# A dummy removal queue to monkeypatch around the need for removal and upload
# threads
class DummyQueue:
@@ -107,10 +110,10 @@ class cache_tests(unittest.TestCase):
# Tested methods assume that they are called from
# file system request handler
- llfuse.lock.acquire()
+ s3ql.block_cache.lock = MockLock()
+ s3ql.block_cache.lock_released = MockLock()
def tearDown(self):
- llfuse.lock.release()
self.cache.backend_pool = self.backend_pool
self.cache.destroy()
shutil.rmtree(self.cachedir)
@@ -163,8 +166,7 @@ class cache_tests(unittest.TestCase):
try:
# Try to clean-up (implicitly calls expire)
- with llfuse.lock_released, \
- assert_logs('Unable to drop cache, no upload threads left alive',
+ with assert_logs('Unable to drop cache, no upload threads left alive',
level=logging.ERROR, count=1):
with pytest.raises(OSError) as exc_info:
self.cache.destroy()
@@ -259,7 +261,7 @@ class cache_tests(unittest.TestCase):
fh.seek(0)
fh.write(data1)
el1 = fh
- self.cache.upload(el1)
+ assert self.cache.upload_if_dirty(el1)
self.cache.backend_pool.verify()
# Case 2: Link new object
@@ -268,7 +270,7 @@ class cache_tests(unittest.TestCase):
fh.seek(0)
fh.write(data1)
el2 = fh
- self.cache.upload(el2)
+ assert not self.cache.upload_if_dirty(el2)
self.cache.backend_pool.verify()
# Case 3: Upload old object, still has references
@@ -276,7 +278,7 @@ class cache_tests(unittest.TestCase):
with self.cache.get(inode, blockno1) as fh:
fh.seek(0)
fh.write(data2)
- self.cache.upload(el1)
+ assert self.cache.upload_if_dirty(el1)
self.cache.backend_pool.verify()
# Case 4: Upload old object, no references left
@@ -284,7 +286,7 @@ class cache_tests(unittest.TestCase):
with self.cache.get(inode, blockno2) as fh:
fh.seek(0)
fh.write(data3)
- self.cache.upload(el2)
+ assert self.cache.upload_if_dirty(el2)
self.cache.backend_pool.verify()
# Case 5: Link old object, no references left
@@ -292,7 +294,7 @@ class cache_tests(unittest.TestCase):
with self.cache.get(inode, blockno2) as fh:
fh.seek(0)
fh.write(data2)
- self.cache.upload(el2)
+ assert not self.cache.upload_if_dirty(el2)
self.cache.backend_pool.verify()
# Case 6: Link old object, still has references
@@ -302,14 +304,14 @@ class cache_tests(unittest.TestCase):
fh.seek(0)
fh.write(data1)
el3 = fh
- self.cache.upload(el3)
+ assert self.cache.upload_if_dirty(el3)
self.cache.backend_pool.verify()
self.cache.backend_pool = MockBackendPool(self.backend_pool)
with self.cache.get(inode, blockno1) as fh:
fh.seek(0)
fh.write(data1)
- self.cache.upload(el1)
+ assert not self.cache.upload_if_dirty(el1)
self.cache.drop()
self.cache.backend_pool.verify()
@@ -359,7 +361,7 @@ class cache_tests(unittest.TestCase):
self.cache.remove(inode, blockno)
# Try to upload it, may happen if CommitThread is interrupted
- self.cache.upload(fh)
+ self.cache.upload_if_dirty(fh)
def test_expire_race(self):
# Create element
@@ -369,32 +371,27 @@ class cache_tests(unittest.TestCase):
with self.cache.get(inode, blockno) as fh:
fh.seek(0)
fh.write(data1)
- self.cache.upload(fh)
+ assert self.cache.upload_if_dirty(fh)
# Make sure entry will be expired
self.cache.cache.max_entries = 0
- def e_w_l():
- with llfuse.lock:
- self.cache.expire()
# Lock it
self.cache._lock_entry(inode, blockno, release_global=True)
try:
# Start expiration, will block on lock
- t1 = AsyncFn(e_w_l)
+ t1 = AsyncFn(self.cache.expire)
t1.start()
# Start second expiration, will block
- t2 = AsyncFn(e_w_l)
+ t2 = AsyncFn(self.cache.expire)
t2.start()
# Release lock
- with llfuse.lock_released:
- safe_sleep(0.1)
- self.cache._unlock_entry(inode, blockno)
- t1.join_and_raise()
- t2.join_and_raise()
+ self.cache._unlock_entry(inode, blockno)
+ t1.join_and_raise()
+ t2.join_and_raise()
assert len(self.cache.cache) == 0
finally:
@@ -413,28 +410,23 @@ class cache_tests(unittest.TestCase):
# We want to expire just one element, but have
# several threads running expire() simultaneously
self.cache.cache.max_entries = 4
- def e_w_l():
- with llfuse.lock:
- self.cache.expire()
# Lock first element so that we have time to start threads
self.cache._lock_entry(inode, 0, release_global=True)
try:
# Start expiration, will block on lock
- t1 = AsyncFn(e_w_l)
+ t1 = AsyncFn(self.cache.expire)
t1.start()
# Start second expiration, will block
- t2 = AsyncFn(e_w_l)
+ t2 = AsyncFn(self.cache.expire)
t2.start()
# Release lock
- with llfuse.lock_released:
- safe_sleep(0.1)
- self.cache._unlock_entry(inode, 0)
- t1.join_and_raise()
- t2.join_and_raise()
+ self.cache._unlock_entry(inode, 0)
+ t1.join_and_raise()
+ t2.join_and_raise()
assert len(self.cache.cache) == 4
finally:
@@ -480,6 +472,94 @@ class cache_tests(unittest.TestCase):
fh.seek(0)
self.assertEqual(fh.read(42), b'')
+ def test_issue_241(self):
+
+ inode = self.inode
+
+ # Create block
+ with self.cache.get(inode, 0) as fh:
+ fh.write(self.random_data(500))
+
+ # "Fill" cache
+ self.cache.cache.max_entries = 0
+
+ # Mock locking to reproduce race condition
+ mlock = MockMultiLock(self.cache.mlock)
+ with patch.object(self.cache, 'mlock', mlock):
+ # Start first expiration run, will block in upload
+ thread1 = AsyncFn(self.cache.expire)
+ thread1.start()
+
+ # Remove the object while the expiration thread waits
+ # for it to become available.
+ thread2 = AsyncFn(self.cache.remove, inode, 0, 1)
+ thread2.start()
+ mlock.yield_to(thread2)
+ thread2.join_and_raise(timeout=10)
+ assert not thread2.is_alive()
+
+ # Create a new object for the same block
+ with self.cache.get(inode, 0) as fh:
+ fh.write(self.random_data(500))
+
+ # Continue first expiration run
+ mlock.yield_to(thread1, block=False)
+ thread1.join_and_raise(timeout=10)
+ assert not thread1.is_alive()
+
+
+class MockMultiLock:
+ def __init__(self, real_mlock):
+ self.cond = real_mlock.cond
+ self.cleared = set()
+ self.real_mlock = real_mlock
+
+ def yield_to(self, thread, block=True):
+ '''Allow *thread* to proceed'''
+
+ me = threading.current_thread()
+ log.debug('%s blocked in yield_to(), phase 1', me.name)
+ with self.cond:
+ self.cleared.add(thread)
+ self.cond.notify_all()
+
+ if not block:
+ return
+ log.debug('%s blocked in yield_to(), phase 2', me.name)
+ with self.cond:
+ if not self.cond.wait_for(lambda: thread not in self.cleared, 10):
+ pytest.fail('Timeout waiting for lock')
+ log.debug('%s completed yield_to()', me.name)
+
+ @contextmanager
+ def __call__(self, *key):
+ self.acquire(*key)
+ try:
+ yield
+ finally:
+ self.release(*key)
+
+ def acquire(self, *key, timeout=None):
+ me = threading.current_thread()
+ log.debug('%s blocked in acquire()', me.name)
+ with self.cond:
+ if not self.cond.wait_for(lambda: me in self.cleared, 10):
+ pytest.fail('Timeout waiting for lock')
+ self.real_mlock.locked_keys.add(key)
+ log.debug('%s got lock', me.name)
+
+ def release(self, *key, noerror=False):
+ me = threading.current_thread()
+ log.debug('%s blocked in release()', me.name)
+ with self.cond:
+ self.cleared.remove(me)
+ self.cond.notify_all()
+ if noerror:
+ self.real_mlock.locked_keys.discard(key)
+ else:
+ self.real_mlock.locked_keys.remove(key)
+ log.debug('%s released lock', me.name)
+
class MockBackendPool(AbstractBackend):
has_native_rename = False
@@ -581,4 +661,21 @@ def start_flush(cache, inode, block=None):
if block is not None and el.blockno != block:
continue
- cache.upload(el)
+ cache.upload_if_dirty(el)
+
+
+class MockLock():
+ def __call__(self):
+ return self
+
+ def acquire(self, timeout=None):
+ pass
+
+ def release(self):
+ pass
+
+ def __enter__(self):
+ pass
+
+ def __exit__(self, *args):
+ pass
diff --git a/tests/t5_ctrl.py b/tests/t5_ctrl.py
index 29834ce..48f1cbe 100755
--- a/tests/t5_ctrl.py
+++ b/tests/t5_ctrl.py
@@ -22,6 +22,7 @@ class TestCtrl(t4_fuse.TestFuse):
self.mkfs()
self.mount()
self.tst_ctrl_flush()
+ self.tst_ctrl_drop()
self.tst_ctrl_log()
self.tst_ctrl_cachesize()
self.umount()
@@ -34,6 +35,13 @@ class TestCtrl(t4_fuse.TestFuse):
sys.excepthook(*sys.exc_info())
pytest.fail("s3qlctrl raised exception")
+ def tst_ctrl_drop(self):
+ try:
+ s3ql.ctrl.main(['dropcache', self.mnt_dir])
+ except:
+ sys.excepthook(*sys.exc_info())
+ pytest.fail("s3qlctrl raised exception")
+
def tst_ctrl_log(self):
try:
s3ql.ctrl.main(['log', self.mnt_dir, 'warn'])