diff options
42 files changed, 1024 insertions, 594 deletions
diff --git a/Changes.txt b/Changes.txt index d088740..77871ec 100644 --- a/Changes.txt +++ b/Changes.txt @@ -1,3 +1,18 @@ +2017-11-04, S3QL 2.24 + + * fsck.s3ql now accepts a new --force-remote parameter. It does + dangerous things, and you should probably not use it. + + * When terminated by a signal (rather than by an unmount call), + mount.s3ql now terminates as quickly as possible (i.e., without + cleaning the cache), resulting in an unclean unmount. + + * The Swift backend feature detection is no longer affected by the + no-ssl backend option (which is intended only for connections to + the authentication server). + + * Accessing the cn-north-1 S3 storage region is now working. + 2017-08-18, S3QL 2.23 * Accessing the us-east-1 S3 storage region is now working again. @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: s3ql -Version: 2.23 +Version: 2.24 Summary: a full-featured file system for online data storage Home-page: https://bitbucket.org/nikratio/s3ql/ Author: Nikolaus Rath @@ -11,9 +11,9 @@ Description: .. NOTE: We cannot use sophisticated ReST syntax here because this file is rendered by Bitbucket. - ============ - About S3QL - ============ + ====== + S3QL + ====== S3QL is a file system that stores all its data online using storage services like `Google Storage`_, `Amazon S3`_, or OpenStack_. S3QL @@ -119,12 +119,12 @@ Description: .. *nikratio-s3ql-bucket* is the S3 bucket in which the file system will be stored. :: - mkfs.s3ql s3://nikratio-s3ql-bucket + mkfs.s3ql s3://ap-south-1/nikratio-s3ql-bucket To mount the S3QL file system stored in the S3 bucket *nikratio_s3ql_bucket* in the directory ``/mnt/s3ql``, enter:: - mount.s3ql s3://nikratio-s3ql-bucket /mnt/s3ql + mount.s3ql s3://ap-south-1/nikratio-s3ql-bucket /mnt/s3ql Now you can instruct your favorite backup program to run a backup into the directory ``/mnt/s3ql`` and the data will be stored an Amazon @@ -2,9 +2,9 @@ NOTE: We cannot use sophisticated ReST syntax here because this file is rendered by Bitbucket. -============ - About S3QL -============ +====== + S3QL +====== S3QL is a file system that stores all its data online using storage services like `Google Storage`_, `Amazon S3`_, or OpenStack_. S3QL @@ -110,12 +110,12 @@ command. Here we are using the Amazon S3 backend, and *nikratio-s3ql-bucket* is the S3 bucket in which the file system will be stored. :: - mkfs.s3ql s3://nikratio-s3ql-bucket + mkfs.s3ql s3://ap-south-1/nikratio-s3ql-bucket To mount the S3QL file system stored in the S3 bucket *nikratio_s3ql_bucket* in the directory ``/mnt/s3ql``, enter:: - mount.s3ql s3://nikratio-s3ql-bucket /mnt/s3ql + mount.s3ql s3://ap-south-1/nikratio-s3ql-bucket /mnt/s3ql Now you can instruct your favorite backup program to run a backup into the directory ``/mnt/s3ql`` and the data will be stored an Amazon diff --git a/contrib/expire_backups.1 b/contrib/expire_backups.1 index 29030a6..cb1e7e2 100644 --- a/contrib/expire_backups.1 +++ b/contrib/expire_backups.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "EXPIRE_BACKUPS" "1" "Aug 18, 2017" "2.23" "S3QL" +.TH "EXPIRE_BACKUPS" "1" "Nov 04, 2017" "2.24" "S3QL" .SH NAME expire_backups \- Intelligently expire old backups . diff --git a/contrib/pcp.1 b/contrib/pcp.1 index 4205e13..c95f74c 100644 --- a/contrib/pcp.1 +++ b/contrib/pcp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "PCP" "1" "Aug 18, 2017" "2.23" "S3QL" +.TH "PCP" "1" "Nov 04, 2017" "2.24" "S3QL" .SH NAME pcp \- Recursive, parallel copy of directory trees . diff --git a/doc/latex/manual.aux b/doc/latex/manual.aux index 3f61a3f..ee2490b 100644 --- a/doc/latex/manual.aux +++ b/doc/latex/manual.aux @@ -21,16 +21,16 @@ \@writefile{lof}{\select@language{english}} \@writefile{lot}{\select@language{english}} \newlabel{index::doc}{{}{1}{}{section*.2}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {1}About S3QL}{1}{chapter.1}} +\@writefile{toc}{\contentsline {chapter}{\numberline {1}S3QL}{1}{chapter.1}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{about:about-s3ql}{{1}{1}{About S3QL}{chapter.1}{}} -\newlabel{about:s3ql-user-s-guide}{{1}{1}{About S3QL}{chapter.1}{}} -\newlabel{about::doc}{{1}{1}{About S3QL}{chapter.1}{}} +\newlabel{about::doc}{{1}{1}{S3QL}{chapter.1}{}} +\newlabel{about:s3ql-user-s-guide}{{1}{1}{S3QL}{chapter.1}{}} +\newlabel{about:s3ql}{{1}{1}{S3QL}{chapter.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {1.1}Features}{1}{section.1.1}} -\newlabel{about:features}{{1.1}{1}{Features}{section.1.1}{}} \newlabel{about:openstack}{{1.1}{1}{Features}{section.1.1}{}} +\newlabel{about:features}{{1.1}{1}{Features}{section.1.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {1.2}Development Status}{2}{section.1.2}} \newlabel{about:development-status}{{1.2}{2}{Development Status}{section.1.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {1.3}Supported Platforms}{2}{section.1.3}} @@ -41,14 +41,14 @@ \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{installation:github}{{2}{3}{Installation}{chapter.2}{}} \newlabel{installation::doc}{{2}{3}{Installation}{chapter.2}{}} \newlabel{installation:installation}{{2}{3}{Installation}{chapter.2}{}} +\newlabel{installation:github}{{2}{3}{Installation}{chapter.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.1}Dependencies}{3}{section.2.1}} \newlabel{installation:dependencies}{{2.1}{3}{Dependencies}{section.2.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.2}Installing S3QL}{4}{section.2.2}} -\newlabel{installation:installing-s3ql}{{2.2}{4}{Installing S3QL}{section.2.2}{}} \newlabel{installation:inst-s3ql}{{2.2}{4}{Installing S3QL}{section.2.2}{}} +\newlabel{installation:installing-s3ql}{{2.2}{4}{Installing S3QL}{section.2.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.3}Development Version}{4}{section.2.3}} \newlabel{installation:development-version}{{2.3}{4}{Development Version}{section.2.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.4}Running tests requiring remote servers}{4}{section.2.4}} @@ -57,18 +57,18 @@ \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{backends:sphinx}{{3}{7}{Storage Backends}{chapter.3}{}} -\newlabel{backends::doc}{{3}{7}{Storage Backends}{chapter.3}{}} \newlabel{backends:id1}{{3}{7}{Storage Backends}{chapter.3}{}} +\newlabel{backends::doc}{{3}{7}{Storage Backends}{chapter.3}{}} \newlabel{backends:storage-backends}{{3}{7}{Storage Backends}{chapter.3}{}} +\newlabel{backends:sphinx}{{3}{7}{Storage Backends}{chapter.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.1}Google Storage}{7}{section.3.1}} \newlabel{backends:google-storage}{{3.1}{7}{Google Storage}{section.3.1}{}} \newlabel{backends:cmdoption-gs_backend-arg-no-ssl}{{3.1}{7}{Google Storage}{section*.3}{}} \newlabel{backends:cmdoption-gs_backend-arg-ssl-ca-path}{{3.1}{8}{Google Storage}{section*.4}{}} \newlabel{backends:cmdoption-gs_backend-arg-tcp-timeout}{{3.1}{8}{Google Storage}{section*.5}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.2}Amazon S3}{8}{section.3.2}} -\newlabel{backends:google-storage-manager}{{3.2}{8}{Amazon S3}{section.3.2}{}} \newlabel{backends:amazon-s3}{{3.2}{8}{Amazon S3}{section.3.2}{}} +\newlabel{backends:google-storage-manager}{{3.2}{8}{Amazon S3}{section.3.2}{}} \newlabel{backends:cmdoption-s3_backend-arg-no-ssl}{{3.2}{8}{Amazon S3}{section*.6}{}} \newlabel{backends:cmdoption-s3_backend-arg-ssl-ca-path}{{3.2}{8}{Amazon S3}{section*.7}{}} \newlabel{backends:cmdoption-s3_backend-arg-tcp-timeout}{{3.2}{8}{Amazon S3}{section*.8}{}} @@ -117,14 +117,14 @@ \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{mkfs:file-system-creation}{{5}{17}{File System Creation}{chapter.5}{}} \newlabel{mkfs::doc}{{5}{17}{File System Creation}{chapter.5}{}} +\newlabel{mkfs:file-system-creation}{{5}{17}{File System Creation}{chapter.5}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {6}Managing File Systems}{19}{chapter.6}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{adm:managing-file-systems}{{6}{19}{Managing File Systems}{chapter.6}{}} \newlabel{adm::doc}{{6}{19}{Managing File Systems}{chapter.6}{}} +\newlabel{adm:managing-file-systems}{{6}{19}{Managing File Systems}{chapter.6}{}} \@writefile{toc}{\contentsline {section}{\numberline {6.1}Changing the Passphrase}{19}{section.6.1}} \newlabel{adm:changing-the-passphrase}{{6.1}{19}{Changing the Passphrase}{section.6.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {6.2}Upgrading the file system}{20}{section.6.2}} @@ -137,8 +137,8 @@ \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{mount:mounting}{{7}{21}{Mounting}{chapter.7}{}} \newlabel{mount::doc}{{7}{21}{Mounting}{chapter.7}{}} +\newlabel{mount:mounting}{{7}{21}{Mounting}{chapter.7}{}} \@writefile{toc}{\contentsline {section}{\numberline {7.1}Permission Checking}{22}{section.7.1}} \newlabel{mount:permission-checking}{{7.1}{22}{Permission Checking}{section.7.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {7.2}Compression Algorithms}{22}{section.7.2}} @@ -169,11 +169,11 @@ \newlabel{special:s3qlstat}{{8.2}{26}{Getting Statistics}{section.8.2}{}} \newlabel{special:getting-statistics}{{8.2}{26}{Getting Statistics}{section.8.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.3}Immutable Trees}{26}{section.8.3}} -\newlabel{special:immutable-trees}{{8.3}{26}{Immutable Trees}{section.8.3}{}} \newlabel{special:s3qllock}{{8.3}{26}{Immutable Trees}{section.8.3}{}} +\newlabel{special:immutable-trees}{{8.3}{26}{Immutable Trees}{section.8.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.4}Fast Recursive Removal}{27}{section.8.4}} -\newlabel{special:fast-recursive-removal}{{8.4}{27}{Fast Recursive Removal}{section.8.4}{}} \newlabel{special:s3qlrm}{{8.4}{27}{Fast Recursive Removal}{section.8.4}{}} +\newlabel{special:fast-recursive-removal}{{8.4}{27}{Fast Recursive Removal}{section.8.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.5}Runtime Configuration}{27}{section.8.5}} \newlabel{special:s3qlctrl}{{8.5}{27}{Runtime Configuration}{section.8.5}{}} \newlabel{special:runtime-configuration}{{8.5}{27}{Runtime Configuration}{section.8.5}{}} @@ -187,278 +187,278 @@ \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{fsck:checking-for-errors}{{10}{31}{Checking for Errors}{chapter.10}{}} \newlabel{fsck::doc}{{10}{31}{Checking for Errors}{chapter.10}{}} +\newlabel{fsck:checking-for-errors}{{10}{31}{Checking for Errors}{chapter.10}{}} \@writefile{toc}{\contentsline {section}{\numberline {10.1}Checking and repairing internal file system errors}{31}{section.10.1}} \newlabel{fsck:checking-and-repairing-internal-file-system-errors}{{10.1}{31}{Checking and repairing internal file system errors}{section.10.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {10.2}Detecting and handling backend data corruption}{32}{section.10.2}} \newlabel{fsck:s3ql-verify}{{10.2}{32}{Detecting and handling backend data corruption}{section.10.2}{}} \newlabel{fsck:detecting-and-handling-backend-data-corruption}{{10.2}{32}{Detecting and handling backend data corruption}{section.10.2}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {11}Storing Authentication Information}{33}{chapter.11}} +\@writefile{toc}{\contentsline {chapter}{\numberline {11}Storing Authentication Information}{35}{chapter.11}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{authinfo:storing-authentication-information}{{11}{33}{Storing Authentication Information}{chapter.11}{}} -\newlabel{authinfo:authinfo}{{11}{33}{Storing Authentication Information}{chapter.11}{}} -\newlabel{authinfo::doc}{{11}{33}{Storing Authentication Information}{chapter.11}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {12}Contributed Programs}{35}{chapter.12}} +\newlabel{authinfo:authinfo}{{11}{35}{Storing Authentication Information}{chapter.11}{}} +\newlabel{authinfo::doc}{{11}{35}{Storing Authentication Information}{chapter.11}{}} +\newlabel{authinfo:storing-authentication-information}{{11}{35}{Storing Authentication Information}{chapter.11}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {12}Contributed Programs}{37}{chapter.12}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{contrib:contributed-programs}{{12}{35}{Contributed Programs}{chapter.12}{}} -\newlabel{contrib::doc}{{12}{35}{Contributed Programs}{chapter.12}{}} -\@writefile{toc}{\contentsline {section}{\numberline {12.1}benchmark.py}{35}{section.12.1}} -\newlabel{contrib:benchmark-py}{{12.1}{35}{benchmark.py}{section.12.1}{}} -\@writefile{toc}{\contentsline {section}{\numberline {12.2}clone\_fs.py}{35}{section.12.2}} -\newlabel{contrib:clone-fs-py}{{12.2}{35}{clone\_fs.py}{section.12.2}{}} -\@writefile{toc}{\contentsline {section}{\numberline {12.3}pcp.py}{35}{section.12.3}} -\newlabel{contrib:pcp-py}{{12.3}{35}{pcp.py}{section.12.3}{}} -\newlabel{contrib:pcp}{{12.3}{35}{pcp.py}{section.12.3}{}} -\@writefile{toc}{\contentsline {section}{\numberline {12.4}s3ql\_backup.sh}{35}{section.12.4}} -\newlabel{contrib:s3ql-backup-sh}{{12.4}{35}{s3ql\_backup.sh}{section.12.4}{}} -\@writefile{toc}{\contentsline {section}{\numberline {12.5}expire\_backups.py}{36}{section.12.5}} -\newlabel{contrib:expire-backups-py}{{12.5}{36}{expire\_backups.py}{section.12.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {12.6}remove\_objects.py}{37}{section.12.6}} -\newlabel{contrib:remove-objects-py}{{12.6}{37}{remove\_objects.py}{section.12.6}{}} -\newlabel{contrib:remove-objects}{{12.6}{37}{remove\_objects.py}{section.12.6}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {13}Tips \& Tricks}{39}{chapter.13}} +\newlabel{contrib::doc}{{12}{37}{Contributed Programs}{chapter.12}{}} +\newlabel{contrib:contributed-programs}{{12}{37}{Contributed Programs}{chapter.12}{}} +\@writefile{toc}{\contentsline {section}{\numberline {12.1}benchmark.py}{37}{section.12.1}} +\newlabel{contrib:benchmark-py}{{12.1}{37}{benchmark.py}{section.12.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {12.2}clone\_fs.py}{37}{section.12.2}} +\newlabel{contrib:clone-fs-py}{{12.2}{37}{clone\_fs.py}{section.12.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {12.3}pcp.py}{37}{section.12.3}} +\newlabel{contrib:pcp}{{12.3}{37}{pcp.py}{section.12.3}{}} +\newlabel{contrib:pcp-py}{{12.3}{37}{pcp.py}{section.12.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {12.4}s3ql\_backup.sh}{37}{section.12.4}} +\newlabel{contrib:s3ql-backup-sh}{{12.4}{37}{s3ql\_backup.sh}{section.12.4}{}} +\@writefile{toc}{\contentsline {section}{\numberline {12.5}expire\_backups.py}{38}{section.12.5}} +\newlabel{contrib:expire-backups-py}{{12.5}{38}{expire\_backups.py}{section.12.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {12.6}remove\_objects.py}{39}{section.12.6}} +\newlabel{contrib:remove-objects}{{12.6}{39}{remove\_objects.py}{section.12.6}{}} +\newlabel{contrib:remove-objects-py}{{12.6}{39}{remove\_objects.py}{section.12.6}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {13}Tips \& Tricks}{41}{chapter.13}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{tips:tips-tricks}{{13}{39}{Tips \& Tricks}{chapter.13}{}} -\newlabel{tips::doc}{{13}{39}{Tips \& Tricks}{chapter.13}{}} -\@writefile{toc}{\contentsline {section}{\numberline {13.1}SSH Backend}{39}{section.13.1}} -\newlabel{tips:ssh-backend}{{13.1}{39}{SSH Backend}{section.13.1}{}} -\newlabel{tips:ssh-tipp}{{13.1}{39}{SSH Backend}{section.13.1}{}} -\@writefile{toc}{\contentsline {section}{\numberline {13.2}Permanently mounted backup file system}{39}{section.13.2}} -\newlabel{tips:permanently-mounted-backup-file-system}{{13.2}{39}{Permanently mounted backup file system}{section.13.2}{}} -\@writefile{toc}{\contentsline {section}{\numberline {13.3}Improving copy performance}{39}{section.13.3}} -\newlabel{tips:improving-copy-performance}{{13.3}{39}{Improving copy performance}{section.13.3}{}} -\newlabel{tips:copy-performance}{{13.3}{39}{Improving copy performance}{section.13.3}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {14}Known Issues}{41}{chapter.14}} +\newlabel{tips::doc}{{13}{41}{Tips \& Tricks}{chapter.13}{}} +\newlabel{tips:tips-tricks}{{13}{41}{Tips \& Tricks}{chapter.13}{}} +\@writefile{toc}{\contentsline {section}{\numberline {13.1}SSH Backend}{41}{section.13.1}} +\newlabel{tips:ssh-backend}{{13.1}{41}{SSH Backend}{section.13.1}{}} +\newlabel{tips:ssh-tipp}{{13.1}{41}{SSH Backend}{section.13.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {13.2}Permanently mounted backup file system}{41}{section.13.2}} +\newlabel{tips:permanently-mounted-backup-file-system}{{13.2}{41}{Permanently mounted backup file system}{section.13.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {13.3}Improving copy performance}{41}{section.13.3}} +\newlabel{tips:copy-performance}{{13.3}{41}{Improving copy performance}{section.13.3}{}} +\newlabel{tips:improving-copy-performance}{{13.3}{41}{Improving copy performance}{section.13.3}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {14}Known Issues}{43}{chapter.14}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{issues::doc}{{14}{41}{Known Issues}{chapter.14}{}} -\newlabel{issues:known-issues}{{14}{41}{Known Issues}{chapter.14}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {15}Manpages}{43}{chapter.15}} +\newlabel{issues::doc}{{14}{43}{Known Issues}{chapter.14}{}} +\newlabel{issues:known-issues}{{14}{43}{Known Issues}{chapter.14}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {15}Manpages}{45}{chapter.15}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{man/index::doc}{{15}{43}{Manpages}{chapter.15}{}} -\newlabel{man/index:manpages}{{15}{43}{Manpages}{chapter.15}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.1}The \textbf {\texttt {mkfs.s3ql}} command}{43}{section.15.1}} -\newlabel{man/mkfs:the-command-command}{{15.1}{43}{The \textbf {\texttt {mkfs.s3ql}} command}{section.15.1}{}} -\newlabel{man/mkfs::doc}{{15.1}{43}{The \textbf {\texttt {mkfs.s3ql}} command}{section.15.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.1.1}Synopsis}{43}{subsection.15.1.1}} -\newlabel{man/mkfs:synopsis}{{15.1.1}{43}{Synopsis}{subsection.15.1.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.1.2}Description}{43}{subsection.15.1.2}} -\newlabel{man/mkfs:description}{{15.1.2}{43}{Description}{subsection.15.1.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.1.3}Options}{43}{subsection.15.1.3}} -\newlabel{man/mkfs:options}{{15.1.3}{43}{Options}{subsection.15.1.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.1.4}Exit Codes}{44}{subsection.15.1.4}} -\newlabel{man/mkfs:exit-codes}{{15.1.4}{44}{Exit Codes}{subsection.15.1.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.1.5}See Also}{44}{subsection.15.1.5}} -\newlabel{man/mkfs:see-also}{{15.1.5}{44}{See Also}{subsection.15.1.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.2}The \textbf {\texttt {s3qladm}} command}{44}{section.15.2}} -\newlabel{man/adm:the-command-command}{{15.2}{44}{The \textbf {\texttt {s3qladm}} command}{section.15.2}{}} -\newlabel{man/adm::doc}{{15.2}{44}{The \textbf {\texttt {s3qladm}} command}{section.15.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.1}Synopsis}{44}{subsection.15.2.1}} -\newlabel{man/adm:synopsis}{{15.2.1}{44}{Synopsis}{subsection.15.2.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.2}Description}{45}{subsection.15.2.2}} -\newlabel{man/adm:description}{{15.2.2}{45}{Description}{subsection.15.2.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.3}Options}{45}{subsection.15.2.3}} -\newlabel{man/adm:options}{{15.2.3}{45}{Options}{subsection.15.2.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.4}Actions}{45}{subsection.15.2.4}} -\newlabel{man/adm:actions}{{15.2.4}{45}{Actions}{subsection.15.2.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.5}Exit Codes}{45}{subsection.15.2.5}} -\newlabel{man/adm:exit-codes}{{15.2.5}{45}{Exit Codes}{subsection.15.2.5}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.6}See Also}{46}{subsection.15.2.6}} -\newlabel{man/adm:see-also}{{15.2.6}{46}{See Also}{subsection.15.2.6}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.3}The \textbf {\texttt {mount.s3ql}} command}{46}{section.15.3}} -\newlabel{man/mount:the-command-command}{{15.3}{46}{The \textbf {\texttt {mount.s3ql}} command}{section.15.3}{}} -\newlabel{man/mount::doc}{{15.3}{46}{The \textbf {\texttt {mount.s3ql}} command}{section.15.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.3.1}Synopsis}{46}{subsection.15.3.1}} -\newlabel{man/mount:synopsis}{{15.3.1}{46}{Synopsis}{subsection.15.3.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.3.2}Description}{46}{subsection.15.3.2}} -\newlabel{man/mount:description}{{15.3.2}{46}{Description}{subsection.15.3.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.3.3}Options}{46}{subsection.15.3.3}} -\newlabel{man/mount:options}{{15.3.3}{46}{Options}{subsection.15.3.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.3.4}Exit Codes}{47}{subsection.15.3.4}} -\newlabel{man/mount:exit-codes}{{15.3.4}{47}{Exit Codes}{subsection.15.3.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.3.5}See Also}{48}{subsection.15.3.5}} -\newlabel{man/mount:see-also}{{15.3.5}{48}{See Also}{subsection.15.3.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.4}The \textbf {\texttt {s3qlstat}} command}{48}{section.15.4}} -\newlabel{man/stat:the-command-command}{{15.4}{48}{The \textbf {\texttt {s3qlstat}} command}{section.15.4}{}} -\newlabel{man/stat::doc}{{15.4}{48}{The \textbf {\texttt {s3qlstat}} command}{section.15.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.4.1}Synopsis}{48}{subsection.15.4.1}} -\newlabel{man/stat:synopsis}{{15.4.1}{48}{Synopsis}{subsection.15.4.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.4.2}Description}{48}{subsection.15.4.2}} -\newlabel{man/stat:description}{{15.4.2}{48}{Description}{subsection.15.4.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.4.3}Options}{49}{subsection.15.4.3}} -\newlabel{man/stat:options}{{15.4.3}{49}{Options}{subsection.15.4.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.4.4}Exit Codes}{49}{subsection.15.4.4}} -\newlabel{man/stat:exit-codes}{{15.4.4}{49}{Exit Codes}{subsection.15.4.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.4.5}See Also}{49}{subsection.15.4.5}} -\newlabel{man/stat:see-also}{{15.4.5}{49}{See Also}{subsection.15.4.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.5}The \textbf {\texttt {s3qlctrl}} command}{49}{section.15.5}} -\newlabel{man/ctrl:the-command-command}{{15.5}{49}{The \textbf {\texttt {s3qlctrl}} command}{section.15.5}{}} -\newlabel{man/ctrl::doc}{{15.5}{49}{The \textbf {\texttt {s3qlctrl}} command}{section.15.5}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.5.1}Synopsis}{49}{subsection.15.5.1}} -\newlabel{man/ctrl:synopsis}{{15.5.1}{49}{Synopsis}{subsection.15.5.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.5.2}Description}{49}{subsection.15.5.2}} -\newlabel{man/ctrl:description}{{15.5.2}{49}{Description}{subsection.15.5.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.5.3}Options}{50}{subsection.15.5.3}} -\newlabel{man/ctrl:options}{{15.5.3}{50}{Options}{subsection.15.5.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.5.4}Exit Codes}{50}{subsection.15.5.4}} -\newlabel{man/ctrl:exit-codes}{{15.5.4}{50}{Exit Codes}{subsection.15.5.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.5.5}See Also}{50}{subsection.15.5.5}} -\newlabel{man/ctrl:see-also}{{15.5.5}{50}{See Also}{subsection.15.5.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.6}The \textbf {\texttt {s3qlcp}} command}{50}{section.15.6}} -\newlabel{man/cp:the-command-command}{{15.6}{50}{The \textbf {\texttt {s3qlcp}} command}{section.15.6}{}} -\newlabel{man/cp::doc}{{15.6}{50}{The \textbf {\texttt {s3qlcp}} command}{section.15.6}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.6.1}Synopsis}{50}{subsection.15.6.1}} -\newlabel{man/cp:synopsis}{{15.6.1}{50}{Synopsis}{subsection.15.6.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.6.2}Description}{51}{subsection.15.6.2}} -\newlabel{man/cp:description}{{15.6.2}{51}{Description}{subsection.15.6.2}{}} -\@writefile{toc}{\contentsline {subsubsection}{Snapshotting vs Hardlinking}{51}{subsubsection*.22}} -\newlabel{man/cp:snapshotting-vs-hardlinking}{{15.6.2}{51}{Snapshotting vs Hardlinking}{subsubsection*.22}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.6.3}Options}{51}{subsection.15.6.3}} -\newlabel{man/cp:options}{{15.6.3}{51}{Options}{subsection.15.6.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.6.4}Exit Codes}{51}{subsection.15.6.4}} -\newlabel{man/cp:exit-codes}{{15.6.4}{51}{Exit Codes}{subsection.15.6.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.6.5}See Also}{52}{subsection.15.6.5}} -\newlabel{man/cp:see-also}{{15.6.5}{52}{See Also}{subsection.15.6.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.7}The \textbf {\texttt {s3qlrm}} command}{52}{section.15.7}} -\newlabel{man/rm:the-command-command}{{15.7}{52}{The \textbf {\texttt {s3qlrm}} command}{section.15.7}{}} -\newlabel{man/rm::doc}{{15.7}{52}{The \textbf {\texttt {s3qlrm}} command}{section.15.7}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.7.1}Synopsis}{52}{subsection.15.7.1}} -\newlabel{man/rm:synopsis}{{15.7.1}{52}{Synopsis}{subsection.15.7.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.7.2}Description}{52}{subsection.15.7.2}} -\newlabel{man/rm:description}{{15.7.2}{52}{Description}{subsection.15.7.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.7.3}Options}{52}{subsection.15.7.3}} -\newlabel{man/rm:options}{{15.7.3}{52}{Options}{subsection.15.7.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.7.4}Exit Codes}{52}{subsection.15.7.4}} -\newlabel{man/rm:exit-codes}{{15.7.4}{52}{Exit Codes}{subsection.15.7.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.7.5}See Also}{53}{subsection.15.7.5}} -\newlabel{man/rm:see-also}{{15.7.5}{53}{See Also}{subsection.15.7.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.8}The \textbf {\texttt {s3qllock}} command}{53}{section.15.8}} -\newlabel{man/lock:the-command-command}{{15.8}{53}{The \textbf {\texttt {s3qllock}} command}{section.15.8}{}} -\newlabel{man/lock::doc}{{15.8}{53}{The \textbf {\texttt {s3qllock}} command}{section.15.8}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.1}Synopsis}{53}{subsection.15.8.1}} -\newlabel{man/lock:synopsis}{{15.8.1}{53}{Synopsis}{subsection.15.8.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.2}Description}{53}{subsection.15.8.2}} -\newlabel{man/lock:description}{{15.8.2}{53}{Description}{subsection.15.8.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.3}Rationale}{53}{subsection.15.8.3}} -\newlabel{man/lock:rationale}{{15.8.3}{53}{Rationale}{subsection.15.8.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.4}Options}{53}{subsection.15.8.4}} -\newlabel{man/lock:options}{{15.8.4}{53}{Options}{subsection.15.8.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.5}Exit Codes}{54}{subsection.15.8.5}} -\newlabel{man/lock:exit-codes}{{15.8.5}{54}{Exit Codes}{subsection.15.8.5}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.6}See Also}{54}{subsection.15.8.6}} -\newlabel{man/lock:see-also}{{15.8.6}{54}{See Also}{subsection.15.8.6}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.9}The \textbf {\texttt {umount.s3ql}} command}{54}{section.15.9}} -\newlabel{man/umount:the-command-command}{{15.9}{54}{The \textbf {\texttt {umount.s3ql}} command}{section.15.9}{}} -\newlabel{man/umount::doc}{{15.9}{54}{The \textbf {\texttt {umount.s3ql}} command}{section.15.9}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.9.1}Synopsis}{54}{subsection.15.9.1}} -\newlabel{man/umount:synopsis}{{15.9.1}{54}{Synopsis}{subsection.15.9.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.9.2}Description}{54}{subsection.15.9.2}} -\newlabel{man/umount:description}{{15.9.2}{54}{Description}{subsection.15.9.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.9.3}Options}{54}{subsection.15.9.3}} -\newlabel{man/umount:options}{{15.9.3}{54}{Options}{subsection.15.9.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.9.4}Exit Codes}{55}{subsection.15.9.4}} -\newlabel{man/umount:exit-codes}{{15.9.4}{55}{Exit Codes}{subsection.15.9.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.9.5}See Also}{55}{subsection.15.9.5}} -\newlabel{man/umount:see-also}{{15.9.5}{55}{See Also}{subsection.15.9.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.10}The \textbf {\texttt {fsck.s3ql}} command}{55}{section.15.10}} -\newlabel{man/fsck:the-command-command}{{15.10}{55}{The \textbf {\texttt {fsck.s3ql}} command}{section.15.10}{}} -\newlabel{man/fsck::doc}{{15.10}{55}{The \textbf {\texttt {fsck.s3ql}} command}{section.15.10}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.1}Synopsis}{55}{subsection.15.10.1}} -\newlabel{man/fsck:synopsis}{{15.10.1}{55}{Synopsis}{subsection.15.10.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.2}Description}{55}{subsection.15.10.2}} -\newlabel{man/fsck:description}{{15.10.2}{55}{Description}{subsection.15.10.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.3}Options}{55}{subsection.15.10.3}} -\newlabel{man/fsck:options}{{15.10.3}{55}{Options}{subsection.15.10.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.4}Exit Codes}{56}{subsection.15.10.4}} -\newlabel{man/fsck:exit-codes}{{15.10.4}{56}{Exit Codes}{subsection.15.10.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.5}See Also}{57}{subsection.15.10.5}} -\newlabel{man/fsck:see-also}{{15.10.5}{57}{See Also}{subsection.15.10.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.11}The \textbf {\texttt {s3ql\_oauth\_client}} command}{57}{section.15.11}} -\newlabel{man/oauth_client:the-command-command}{{15.11}{57}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}} -\newlabel{man/oauth_client::doc}{{15.11}{57}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}} -\newlabel{man/oauth_client:oauth-client}{{15.11}{57}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.1}Synopsis}{57}{subsection.15.11.1}} -\newlabel{man/oauth_client:synopsis}{{15.11.1}{57}{Synopsis}{subsection.15.11.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.2}Description}{57}{subsection.15.11.2}} -\newlabel{man/oauth_client:description}{{15.11.2}{57}{Description}{subsection.15.11.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.3}Options}{57}{subsection.15.11.3}} -\newlabel{man/oauth_client:options}{{15.11.3}{57}{Options}{subsection.15.11.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.4}Exit Codes}{57}{subsection.15.11.4}} -\newlabel{man/oauth_client:exit-codes}{{15.11.4}{57}{Exit Codes}{subsection.15.11.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.5}See Also}{58}{subsection.15.11.5}} -\newlabel{man/oauth_client:see-also}{{15.11.5}{58}{See Also}{subsection.15.11.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.12}The \textbf {\texttt {s3ql\_verify}} command}{58}{section.15.12}} -\newlabel{man/verify:the-command-command}{{15.12}{58}{The \textbf {\texttt {s3ql\_verify}} command}{section.15.12}{}} -\newlabel{man/verify::doc}{{15.12}{58}{The \textbf {\texttt {s3ql\_verify}} command}{section.15.12}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.12.1}Synopsis}{58}{subsection.15.12.1}} -\newlabel{man/verify:synopsis}{{15.12.1}{58}{Synopsis}{subsection.15.12.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.12.2}Description}{58}{subsection.15.12.2}} -\newlabel{man/verify:description}{{15.12.2}{58}{Description}{subsection.15.12.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.12.3}Options}{58}{subsection.15.12.3}} -\newlabel{man/verify:options}{{15.12.3}{58}{Options}{subsection.15.12.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.12.4}Exit Codes}{59}{subsection.15.12.4}} -\newlabel{man/verify:exit-codes}{{15.12.4}{59}{Exit Codes}{subsection.15.12.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.12.5}See Also}{59}{subsection.15.12.5}} -\newlabel{man/verify:see-also}{{15.12.5}{59}{See Also}{subsection.15.12.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.13}The \textbf {\texttt {pcp}} command}{59}{section.15.13}} -\newlabel{man/pcp:the-command-command}{{15.13}{59}{The \textbf {\texttt {pcp}} command}{section.15.13}{}} -\newlabel{man/pcp::doc}{{15.13}{59}{The \textbf {\texttt {pcp}} command}{section.15.13}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.13.1}Synopsis}{59}{subsection.15.13.1}} -\newlabel{man/pcp:synopsis}{{15.13.1}{59}{Synopsis}{subsection.15.13.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.13.2}Description}{59}{subsection.15.13.2}} -\newlabel{man/pcp:description}{{15.13.2}{59}{Description}{subsection.15.13.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.13.3}Options}{60}{subsection.15.13.3}} -\newlabel{man/pcp:options}{{15.13.3}{60}{Options}{subsection.15.13.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.13.4}Exit Codes}{60}{subsection.15.13.4}} -\newlabel{man/pcp:exit-codes}{{15.13.4}{60}{Exit Codes}{subsection.15.13.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.13.5}See Also}{60}{subsection.15.13.5}} -\newlabel{man/pcp:see-also}{{15.13.5}{60}{See Also}{subsection.15.13.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.14}The \textbf {\texttt {expire\_backups}} command}{60}{section.15.14}} -\newlabel{man/expire_backups:the-command-command}{{15.14}{60}{The \textbf {\texttt {expire\_backups}} command}{section.15.14}{}} -\newlabel{man/expire_backups::doc}{{15.14}{60}{The \textbf {\texttt {expire\_backups}} command}{section.15.14}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.14.1}Synopsis}{60}{subsection.15.14.1}} -\newlabel{man/expire_backups:synopsis}{{15.14.1}{60}{Synopsis}{subsection.15.14.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.14.2}Description}{60}{subsection.15.14.2}} -\newlabel{man/expire_backups:description}{{15.14.2}{60}{Description}{subsection.15.14.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.14.3}Options}{61}{subsection.15.14.3}} -\newlabel{man/expire_backups:options}{{15.14.3}{61}{Options}{subsection.15.14.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.14.4}Exit Codes}{62}{subsection.15.14.4}} -\newlabel{man/expire_backups:exit-codes}{{15.14.4}{62}{Exit Codes}{subsection.15.14.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {15.14.5}See Also}{62}{subsection.15.14.5}} -\newlabel{man/expire_backups:see-also}{{15.14.5}{62}{See Also}{subsection.15.14.5}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {16}Further Resources / Getting Help}{63}{chapter.16}} +\newlabel{man/index:manpages}{{15}{45}{Manpages}{chapter.15}{}} +\newlabel{man/index::doc}{{15}{45}{Manpages}{chapter.15}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.1}The \textbf {\texttt {mkfs.s3ql}} command}{45}{section.15.1}} +\newlabel{man/mkfs:the-command-command}{{15.1}{45}{The \textbf {\texttt {mkfs.s3ql}} command}{section.15.1}{}} +\newlabel{man/mkfs::doc}{{15.1}{45}{The \textbf {\texttt {mkfs.s3ql}} command}{section.15.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.1.1}Synopsis}{45}{subsection.15.1.1}} +\newlabel{man/mkfs:synopsis}{{15.1.1}{45}{Synopsis}{subsection.15.1.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.1.2}Description}{45}{subsection.15.1.2}} +\newlabel{man/mkfs:description}{{15.1.2}{45}{Description}{subsection.15.1.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.1.3}Options}{45}{subsection.15.1.3}} +\newlabel{man/mkfs:options}{{15.1.3}{45}{Options}{subsection.15.1.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.1.4}Exit Codes}{46}{subsection.15.1.4}} +\newlabel{man/mkfs:exit-codes}{{15.1.4}{46}{Exit Codes}{subsection.15.1.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.1.5}See Also}{46}{subsection.15.1.5}} +\newlabel{man/mkfs:see-also}{{15.1.5}{46}{See Also}{subsection.15.1.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.2}The \textbf {\texttt {s3qladm}} command}{46}{section.15.2}} +\newlabel{man/adm:the-command-command}{{15.2}{46}{The \textbf {\texttt {s3qladm}} command}{section.15.2}{}} +\newlabel{man/adm::doc}{{15.2}{46}{The \textbf {\texttt {s3qladm}} command}{section.15.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.1}Synopsis}{46}{subsection.15.2.1}} +\newlabel{man/adm:synopsis}{{15.2.1}{46}{Synopsis}{subsection.15.2.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.2}Description}{47}{subsection.15.2.2}} +\newlabel{man/adm:description}{{15.2.2}{47}{Description}{subsection.15.2.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.3}Options}{47}{subsection.15.2.3}} +\newlabel{man/adm:options}{{15.2.3}{47}{Options}{subsection.15.2.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.4}Actions}{47}{subsection.15.2.4}} +\newlabel{man/adm:actions}{{15.2.4}{47}{Actions}{subsection.15.2.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.5}Exit Codes}{47}{subsection.15.2.5}} +\newlabel{man/adm:exit-codes}{{15.2.5}{47}{Exit Codes}{subsection.15.2.5}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.6}See Also}{48}{subsection.15.2.6}} +\newlabel{man/adm:see-also}{{15.2.6}{48}{See Also}{subsection.15.2.6}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.3}The \textbf {\texttt {mount.s3ql}} command}{48}{section.15.3}} +\newlabel{man/mount:the-command-command}{{15.3}{48}{The \textbf {\texttt {mount.s3ql}} command}{section.15.3}{}} +\newlabel{man/mount::doc}{{15.3}{48}{The \textbf {\texttt {mount.s3ql}} command}{section.15.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.3.1}Synopsis}{48}{subsection.15.3.1}} +\newlabel{man/mount:synopsis}{{15.3.1}{48}{Synopsis}{subsection.15.3.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.3.2}Description}{48}{subsection.15.3.2}} +\newlabel{man/mount:description}{{15.3.2}{48}{Description}{subsection.15.3.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.3.3}Options}{48}{subsection.15.3.3}} +\newlabel{man/mount:options}{{15.3.3}{48}{Options}{subsection.15.3.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.3.4}Exit Codes}{49}{subsection.15.3.4}} +\newlabel{man/mount:exit-codes}{{15.3.4}{49}{Exit Codes}{subsection.15.3.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.3.5}See Also}{50}{subsection.15.3.5}} +\newlabel{man/mount:see-also}{{15.3.5}{50}{See Also}{subsection.15.3.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.4}The \textbf {\texttt {s3qlstat}} command}{50}{section.15.4}} +\newlabel{man/stat:the-command-command}{{15.4}{50}{The \textbf {\texttt {s3qlstat}} command}{section.15.4}{}} +\newlabel{man/stat::doc}{{15.4}{50}{The \textbf {\texttt {s3qlstat}} command}{section.15.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.4.1}Synopsis}{50}{subsection.15.4.1}} +\newlabel{man/stat:synopsis}{{15.4.1}{50}{Synopsis}{subsection.15.4.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.4.2}Description}{50}{subsection.15.4.2}} +\newlabel{man/stat:description}{{15.4.2}{50}{Description}{subsection.15.4.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.4.3}Options}{51}{subsection.15.4.3}} +\newlabel{man/stat:options}{{15.4.3}{51}{Options}{subsection.15.4.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.4.4}Exit Codes}{51}{subsection.15.4.4}} +\newlabel{man/stat:exit-codes}{{15.4.4}{51}{Exit Codes}{subsection.15.4.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.4.5}See Also}{51}{subsection.15.4.5}} +\newlabel{man/stat:see-also}{{15.4.5}{51}{See Also}{subsection.15.4.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.5}The \textbf {\texttt {s3qlctrl}} command}{51}{section.15.5}} +\newlabel{man/ctrl:the-command-command}{{15.5}{51}{The \textbf {\texttt {s3qlctrl}} command}{section.15.5}{}} +\newlabel{man/ctrl::doc}{{15.5}{51}{The \textbf {\texttt {s3qlctrl}} command}{section.15.5}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.5.1}Synopsis}{51}{subsection.15.5.1}} +\newlabel{man/ctrl:synopsis}{{15.5.1}{51}{Synopsis}{subsection.15.5.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.5.2}Description}{51}{subsection.15.5.2}} +\newlabel{man/ctrl:description}{{15.5.2}{51}{Description}{subsection.15.5.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.5.3}Options}{52}{subsection.15.5.3}} +\newlabel{man/ctrl:options}{{15.5.3}{52}{Options}{subsection.15.5.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.5.4}Exit Codes}{52}{subsection.15.5.4}} +\newlabel{man/ctrl:exit-codes}{{15.5.4}{52}{Exit Codes}{subsection.15.5.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.5.5}See Also}{52}{subsection.15.5.5}} +\newlabel{man/ctrl:see-also}{{15.5.5}{52}{See Also}{subsection.15.5.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.6}The \textbf {\texttt {s3qlcp}} command}{52}{section.15.6}} +\newlabel{man/cp:the-command-command}{{15.6}{52}{The \textbf {\texttt {s3qlcp}} command}{section.15.6}{}} +\newlabel{man/cp::doc}{{15.6}{52}{The \textbf {\texttt {s3qlcp}} command}{section.15.6}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.6.1}Synopsis}{52}{subsection.15.6.1}} +\newlabel{man/cp:synopsis}{{15.6.1}{52}{Synopsis}{subsection.15.6.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.6.2}Description}{53}{subsection.15.6.2}} +\newlabel{man/cp:description}{{15.6.2}{53}{Description}{subsection.15.6.2}{}} +\@writefile{toc}{\contentsline {subsubsection}{Snapshotting vs Hardlinking}{53}{subsubsection*.22}} +\newlabel{man/cp:snapshotting-vs-hardlinking}{{15.6.2}{53}{Snapshotting vs Hardlinking}{subsubsection*.22}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.6.3}Options}{53}{subsection.15.6.3}} +\newlabel{man/cp:options}{{15.6.3}{53}{Options}{subsection.15.6.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.6.4}Exit Codes}{53}{subsection.15.6.4}} +\newlabel{man/cp:exit-codes}{{15.6.4}{53}{Exit Codes}{subsection.15.6.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.6.5}See Also}{54}{subsection.15.6.5}} +\newlabel{man/cp:see-also}{{15.6.5}{54}{See Also}{subsection.15.6.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.7}The \textbf {\texttt {s3qlrm}} command}{54}{section.15.7}} +\newlabel{man/rm:the-command-command}{{15.7}{54}{The \textbf {\texttt {s3qlrm}} command}{section.15.7}{}} +\newlabel{man/rm::doc}{{15.7}{54}{The \textbf {\texttt {s3qlrm}} command}{section.15.7}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.7.1}Synopsis}{54}{subsection.15.7.1}} +\newlabel{man/rm:synopsis}{{15.7.1}{54}{Synopsis}{subsection.15.7.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.7.2}Description}{54}{subsection.15.7.2}} +\newlabel{man/rm:description}{{15.7.2}{54}{Description}{subsection.15.7.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.7.3}Options}{54}{subsection.15.7.3}} +\newlabel{man/rm:options}{{15.7.3}{54}{Options}{subsection.15.7.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.7.4}Exit Codes}{54}{subsection.15.7.4}} +\newlabel{man/rm:exit-codes}{{15.7.4}{54}{Exit Codes}{subsection.15.7.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.7.5}See Also}{55}{subsection.15.7.5}} +\newlabel{man/rm:see-also}{{15.7.5}{55}{See Also}{subsection.15.7.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.8}The \textbf {\texttt {s3qllock}} command}{55}{section.15.8}} +\newlabel{man/lock:the-command-command}{{15.8}{55}{The \textbf {\texttt {s3qllock}} command}{section.15.8}{}} +\newlabel{man/lock::doc}{{15.8}{55}{The \textbf {\texttt {s3qllock}} command}{section.15.8}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.1}Synopsis}{55}{subsection.15.8.1}} +\newlabel{man/lock:synopsis}{{15.8.1}{55}{Synopsis}{subsection.15.8.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.2}Description}{55}{subsection.15.8.2}} +\newlabel{man/lock:description}{{15.8.2}{55}{Description}{subsection.15.8.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.3}Rationale}{55}{subsection.15.8.3}} +\newlabel{man/lock:rationale}{{15.8.3}{55}{Rationale}{subsection.15.8.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.4}Options}{55}{subsection.15.8.4}} +\newlabel{man/lock:options}{{15.8.4}{55}{Options}{subsection.15.8.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.5}Exit Codes}{56}{subsection.15.8.5}} +\newlabel{man/lock:exit-codes}{{15.8.5}{56}{Exit Codes}{subsection.15.8.5}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.6}See Also}{56}{subsection.15.8.6}} +\newlabel{man/lock:see-also}{{15.8.6}{56}{See Also}{subsection.15.8.6}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.9}The \textbf {\texttt {umount.s3ql}} command}{56}{section.15.9}} +\newlabel{man/umount:the-command-command}{{15.9}{56}{The \textbf {\texttt {umount.s3ql}} command}{section.15.9}{}} +\newlabel{man/umount::doc}{{15.9}{56}{The \textbf {\texttt {umount.s3ql}} command}{section.15.9}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.9.1}Synopsis}{56}{subsection.15.9.1}} +\newlabel{man/umount:synopsis}{{15.9.1}{56}{Synopsis}{subsection.15.9.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.9.2}Description}{56}{subsection.15.9.2}} +\newlabel{man/umount:description}{{15.9.2}{56}{Description}{subsection.15.9.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.9.3}Options}{56}{subsection.15.9.3}} +\newlabel{man/umount:options}{{15.9.3}{56}{Options}{subsection.15.9.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.9.4}Exit Codes}{57}{subsection.15.9.4}} +\newlabel{man/umount:exit-codes}{{15.9.4}{57}{Exit Codes}{subsection.15.9.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.9.5}See Also}{57}{subsection.15.9.5}} +\newlabel{man/umount:see-also}{{15.9.5}{57}{See Also}{subsection.15.9.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.10}The \textbf {\texttt {fsck.s3ql}} command}{57}{section.15.10}} +\newlabel{man/fsck:the-command-command}{{15.10}{57}{The \textbf {\texttt {fsck.s3ql}} command}{section.15.10}{}} +\newlabel{man/fsck::doc}{{15.10}{57}{The \textbf {\texttt {fsck.s3ql}} command}{section.15.10}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.1}Synopsis}{57}{subsection.15.10.1}} +\newlabel{man/fsck:synopsis}{{15.10.1}{57}{Synopsis}{subsection.15.10.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.2}Description}{57}{subsection.15.10.2}} +\newlabel{man/fsck:description}{{15.10.2}{57}{Description}{subsection.15.10.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.3}Options}{57}{subsection.15.10.3}} +\newlabel{man/fsck:options}{{15.10.3}{57}{Options}{subsection.15.10.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.4}Exit Codes}{58}{subsection.15.10.4}} +\newlabel{man/fsck:exit-codes}{{15.10.4}{58}{Exit Codes}{subsection.15.10.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.5}See Also}{59}{subsection.15.10.5}} +\newlabel{man/fsck:see-also}{{15.10.5}{59}{See Also}{subsection.15.10.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.11}The \textbf {\texttt {s3ql\_oauth\_client}} command}{59}{section.15.11}} +\newlabel{man/oauth_client:the-command-command}{{15.11}{59}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}} +\newlabel{man/oauth_client::doc}{{15.11}{59}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}} +\newlabel{man/oauth_client:oauth-client}{{15.11}{59}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.1}Synopsis}{59}{subsection.15.11.1}} +\newlabel{man/oauth_client:synopsis}{{15.11.1}{59}{Synopsis}{subsection.15.11.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.2}Description}{59}{subsection.15.11.2}} +\newlabel{man/oauth_client:description}{{15.11.2}{59}{Description}{subsection.15.11.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.3}Options}{59}{subsection.15.11.3}} +\newlabel{man/oauth_client:options}{{15.11.3}{59}{Options}{subsection.15.11.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.4}Exit Codes}{59}{subsection.15.11.4}} +\newlabel{man/oauth_client:exit-codes}{{15.11.4}{59}{Exit Codes}{subsection.15.11.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.5}See Also}{60}{subsection.15.11.5}} +\newlabel{man/oauth_client:see-also}{{15.11.5}{60}{See Also}{subsection.15.11.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.12}The \textbf {\texttt {s3ql\_verify}} command}{60}{section.15.12}} +\newlabel{man/verify:the-command-command}{{15.12}{60}{The \textbf {\texttt {s3ql\_verify}} command}{section.15.12}{}} +\newlabel{man/verify::doc}{{15.12}{60}{The \textbf {\texttt {s3ql\_verify}} command}{section.15.12}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.12.1}Synopsis}{60}{subsection.15.12.1}} +\newlabel{man/verify:synopsis}{{15.12.1}{60}{Synopsis}{subsection.15.12.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.12.2}Description}{60}{subsection.15.12.2}} +\newlabel{man/verify:description}{{15.12.2}{60}{Description}{subsection.15.12.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.12.3}Options}{60}{subsection.15.12.3}} +\newlabel{man/verify:options}{{15.12.3}{60}{Options}{subsection.15.12.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.12.4}Exit Codes}{61}{subsection.15.12.4}} +\newlabel{man/verify:exit-codes}{{15.12.4}{61}{Exit Codes}{subsection.15.12.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.12.5}See Also}{61}{subsection.15.12.5}} +\newlabel{man/verify:see-also}{{15.12.5}{61}{See Also}{subsection.15.12.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.13}The \textbf {\texttt {pcp}} command}{61}{section.15.13}} +\newlabel{man/pcp:the-command-command}{{15.13}{61}{The \textbf {\texttt {pcp}} command}{section.15.13}{}} +\newlabel{man/pcp::doc}{{15.13}{61}{The \textbf {\texttt {pcp}} command}{section.15.13}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.13.1}Synopsis}{61}{subsection.15.13.1}} +\newlabel{man/pcp:synopsis}{{15.13.1}{61}{Synopsis}{subsection.15.13.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.13.2}Description}{61}{subsection.15.13.2}} +\newlabel{man/pcp:description}{{15.13.2}{61}{Description}{subsection.15.13.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.13.3}Options}{62}{subsection.15.13.3}} +\newlabel{man/pcp:options}{{15.13.3}{62}{Options}{subsection.15.13.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.13.4}Exit Codes}{62}{subsection.15.13.4}} +\newlabel{man/pcp:exit-codes}{{15.13.4}{62}{Exit Codes}{subsection.15.13.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.13.5}See Also}{62}{subsection.15.13.5}} +\newlabel{man/pcp:see-also}{{15.13.5}{62}{See Also}{subsection.15.13.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.14}The \textbf {\texttt {expire\_backups}} command}{62}{section.15.14}} +\newlabel{man/expire_backups:the-command-command}{{15.14}{62}{The \textbf {\texttt {expire\_backups}} command}{section.15.14}{}} +\newlabel{man/expire_backups::doc}{{15.14}{62}{The \textbf {\texttt {expire\_backups}} command}{section.15.14}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.14.1}Synopsis}{62}{subsection.15.14.1}} +\newlabel{man/expire_backups:synopsis}{{15.14.1}{62}{Synopsis}{subsection.15.14.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.14.2}Description}{62}{subsection.15.14.2}} +\newlabel{man/expire_backups:description}{{15.14.2}{62}{Description}{subsection.15.14.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.14.3}Options}{63}{subsection.15.14.3}} +\newlabel{man/expire_backups:options}{{15.14.3}{63}{Options}{subsection.15.14.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.14.4}Exit Codes}{64}{subsection.15.14.4}} +\newlabel{man/expire_backups:exit-codes}{{15.14.4}{64}{Exit Codes}{subsection.15.14.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.14.5}See Also}{64}{subsection.15.14.5}} +\newlabel{man/expire_backups:see-also}{{15.14.5}{64}{See Also}{subsection.15.14.5}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {16}Further Resources / Getting Help}{65}{chapter.16}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{resources:resources}{{16}{63}{Further Resources / Getting Help}{chapter.16}{}} -\newlabel{resources::doc}{{16}{63}{Further Resources / Getting Help}{chapter.16}{}} -\newlabel{resources:further-resources-getting-help}{{16}{63}{Further Resources / Getting Help}{chapter.16}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {17}Implementation Details}{65}{chapter.17}} +\newlabel{resources::doc}{{16}{65}{Further Resources / Getting Help}{chapter.16}{}} +\newlabel{resources:further-resources-getting-help}{{16}{65}{Further Resources / Getting Help}{chapter.16}{}} +\newlabel{resources:resources}{{16}{65}{Further Resources / Getting Help}{chapter.16}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {17}Implementation Details}{67}{chapter.17}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \@writefile{loliteral-block}{\addvspace {10\p@ }} -\newlabel{impl_details:impl-details}{{17}{65}{Implementation Details}{chapter.17}{}} -\newlabel{impl_details::doc}{{17}{65}{Implementation Details}{chapter.17}{}} -\newlabel{impl_details:implementation-details}{{17}{65}{Implementation Details}{chapter.17}{}} -\@writefile{toc}{\contentsline {section}{\numberline {17.1}Metadata Storage}{65}{section.17.1}} -\newlabel{impl_details:metadata-storage}{{17.1}{65}{Metadata Storage}{section.17.1}{}} -\@writefile{toc}{\contentsline {section}{\numberline {17.2}Data Storage}{65}{section.17.2}} -\newlabel{impl_details:data-storage}{{17.2}{65}{Data Storage}{section.17.2}{}} -\@writefile{toc}{\contentsline {section}{\numberline {17.3}Data De-Duplication}{66}{section.17.3}} -\newlabel{impl_details:data-de-duplication}{{17.3}{66}{Data De-Duplication}{section.17.3}{}} -\@writefile{toc}{\contentsline {section}{\numberline {17.4}Caching}{66}{section.17.4}} -\newlabel{impl_details:caching}{{17.4}{66}{Caching}{section.17.4}{}} -\@writefile{toc}{\contentsline {section}{\numberline {17.5}Eventual Consistency Handling}{66}{section.17.5}} -\newlabel{impl_details:eventual-consistency-handling}{{17.5}{66}{Eventual Consistency Handling}{section.17.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {17.6}Encryption}{66}{section.17.6}} -\newlabel{impl_details:encryption}{{17.6}{66}{Encryption}{section.17.6}{}} +\newlabel{impl_details::doc}{{17}{67}{Implementation Details}{chapter.17}{}} +\newlabel{impl_details:implementation-details}{{17}{67}{Implementation Details}{chapter.17}{}} +\newlabel{impl_details:impl-details}{{17}{67}{Implementation Details}{chapter.17}{}} +\@writefile{toc}{\contentsline {section}{\numberline {17.1}Metadata Storage}{67}{section.17.1}} +\newlabel{impl_details:metadata-storage}{{17.1}{67}{Metadata Storage}{section.17.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {17.2}Data Storage}{67}{section.17.2}} +\newlabel{impl_details:data-storage}{{17.2}{67}{Data Storage}{section.17.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {17.3}Data De-Duplication}{68}{section.17.3}} +\newlabel{impl_details:data-de-duplication}{{17.3}{68}{Data De-Duplication}{section.17.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {17.4}Caching}{68}{section.17.4}} +\newlabel{impl_details:caching}{{17.4}{68}{Caching}{section.17.4}{}} +\@writefile{toc}{\contentsline {section}{\numberline {17.5}Eventual Consistency Handling}{68}{section.17.5}} +\newlabel{impl_details:eventual-consistency-handling}{{17.5}{68}{Eventual Consistency Handling}{section.17.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {17.6}Encryption}{68}{section.17.6}} +\newlabel{impl_details:encryption}{{17.6}{68}{Encryption}{section.17.6}{}} diff --git a/doc/latex/manual.out b/doc/latex/manual.out index 602bff1..1b3eea9 100644 --- a/doc/latex/manual.out +++ b/doc/latex/manual.out @@ -1,4 +1,4 @@ -\BOOKMARK [0][-]{chapter.1}{\376\377\000A\000b\000o\000u\000t\000\040\000S\0003\000Q\000L}{}% 1 +\BOOKMARK [0][-]{chapter.1}{\376\377\000S\0003\000Q\000L}{}% 1 \BOOKMARK [1][-]{section.1.1}{\376\377\000F\000e\000a\000t\000u\000r\000e\000s}{chapter.1}% 2 \BOOKMARK [1][-]{section.1.2}{\376\377\000D\000e\000v\000e\000l\000o\000p\000m\000e\000n\000t\000\040\000S\000t\000a\000t\000u\000s}{chapter.1}% 3 \BOOKMARK [1][-]{section.1.3}{\376\377\000S\000u\000p\000p\000o\000r\000t\000e\000d\000\040\000P\000l\000a\000t\000f\000o\000r\000m\000s}{chapter.1}% 4 diff --git a/doc/latex/manual.tex b/doc/latex/manual.tex index 5530b0c..d6f917a 100644 --- a/doc/latex/manual.tex +++ b/doc/latex/manual.tex @@ -32,8 +32,8 @@ \title{S3QL Documentation} -\date{Aug 18, 2017} -\release{2.23} +\date{Nov 04, 2017} +\release{2.24} \author{Nikolaus Rath} \newcommand{\sphinxlogo}{} \renewcommand{\releasename}{Release} @@ -50,73 +50,73 @@ \PYG@it{\PYG@bf{\PYG@ff{#1}}}}}}} \def\PYG#1#2{\PYG@reset\PYG@toks#1+\relax+\PYG@do{#2}} -\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} -\expandafter\def\csname PYG@tok@bp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} -\expandafter\def\csname PYG@tok@kt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.60}{##1}}} \expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.20}{##1}}} -\expandafter\def\csname PYG@tok@gu\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}} -\expandafter\def\csname PYG@tok@mb\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@sa\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@no\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.40}{##1}}} -\expandafter\def\csname PYG@tok@mo\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.27,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@cpf\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.73,0.00,0.40}{##1}}} +\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} +\expandafter\def\csname PYG@tok@gp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}} +\expandafter\def\csname PYG@tok@kt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.60}{##1}}} +\expandafter\def\csname PYG@tok@mi\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} +\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} \expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.33,0.47,0.60}{##1}}} +\expandafter\def\csname PYG@tok@dl\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@c1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@vm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} +\expandafter\def\csname PYG@tok@nl\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.60,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@nt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}} +\expandafter\def\csname PYG@tok@cs\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.80,0.00,0.00}{##1}}} \expandafter\def\csname PYG@tok@mf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@se\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@kr\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@sd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.27,0.13}{##1}}} +\expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@mb\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} \expandafter\def\csname PYG@tok@sc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} +\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}} +\expandafter\def\csname PYG@tok@vg\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.87,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@gu\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}} +\expandafter\def\csname PYG@tok@mo\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.27,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.33,0.33,0.33}{##1}}} +\expandafter\def\csname PYG@tok@gs\endcsname{\let\PYG@bf=\textbf} \expandafter\def\csname PYG@tok@kc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@c\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.33,0.53}{##1}}} +\expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}} +\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.73}{##1}}} \expandafter\def\csname PYG@tok@vc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.40,0.60}{##1}}} +\expandafter\def\csname PYG@tok@bp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} +\expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} +\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.67,0.40,0.00}{##1}}} +\expandafter\def\csname PYG@tok@gh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}} \expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.33,0.33,0.33}{##1}}} -\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@ch\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.73}{##1}}} +\expandafter\def\csname PYG@tok@no\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.40}{##1}}} +\expandafter\def\csname PYG@tok@ne\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@sa\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@sh\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@err\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.67,0.67}{\strut ##1}}} +\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.33,0.53}{##1}}} +\expandafter\def\csname PYG@tok@kp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.53}{##1}}} +\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} +\expandafter\def\csname PYG@tok@fm\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} \expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.80}{##1}}} \expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@cs\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.80,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@il\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} -\expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}} -\expandafter\def\csname PYG@tok@sd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.27,0.13}{##1}}} -\expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit} -\expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}} -\expandafter\def\csname PYG@tok@gs\endcsname{\let\PYG@bf=\textbf} \expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@err\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.67,0.67}{\strut ##1}}} -\expandafter\def\csname PYG@tok@vm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} -\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} -\expandafter\def\csname PYG@tok@sh\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@gp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}} -\expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.73,0.00,0.40}{##1}}} -\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} -\expandafter\def\csname PYG@tok@vg\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.87,0.47,0.00}{##1}}} -\expandafter\def\csname PYG@tok@dl\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@fm\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} \expandafter\def\csname PYG@tok@s\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@nt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.47,0.00}{##1}}} -\expandafter\def\csname PYG@tok@kr\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@mi\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} \expandafter\def\csname PYG@tok@si\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.93,0.93,0.93}{\strut ##1}}} -\expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} -\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.67,0.40,0.00}{##1}}} -\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@ne\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@c1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@c\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} \expandafter\def\csname PYG@tok@sr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,1.00}{\strut ##1}}} -\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} \expandafter\def\csname PYG@tok@ow\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@nl\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.60,0.47,0.00}{##1}}} -\expandafter\def\csname PYG@tok@gh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}} -\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}} -\expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@kp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.53}{##1}}} +\expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit} +\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@il\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} +\expandafter\def\csname PYG@tok@ch\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@se\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@cpf\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} \def\PYGZbs{\char`\\} \def\PYGZus{\char`\_} @@ -149,8 +149,8 @@ -\chapter{About S3QL} -\label{about:about-s3ql}\label{about:s3ql-user-s-guide}\label{about::doc} +\chapter{S3QL} +\label{about::doc}\label{about:s3ql-user-s-guide}\label{about:s3ql} S3QL is a file system that stores all its data online using storage services like \href{http://code.google.com/apis/storage/}{Google Storage}, \href{http://aws.amazon.com/s3}{Amazon S3}, or \href{http://openstack.org/projects/storage/}{OpenStack}. S3QL effectively provides a hard disk of dynamic, infinite capacity that @@ -170,7 +170,7 @@ with extensive automated test cases for all its components. \section{Features} -\label{about:features}\label{about:openstack}\begin{itemize} +\label{about:openstack}\label{about:features}\begin{itemize} \item {} \textbf{Transparency.} Conceptually, S3QL is indistinguishable from a local file system. For example, it supports hardlinks, symlinks, @@ -256,12 +256,12 @@ The S3QL source code is available both on \href{https://github.com/s3ql/main}{Gi \chapter{Installation} -\label{installation:github}\label{installation::doc}\label{installation:installation} +\label{installation::doc}\label{installation:installation}\label{installation:github} S3QL depends on several other programs and libraries that have to be installed first. The best method to satisfy these dependencies depends on your distribution. -The following instructions are for S3QL 2.23 and should be +The following instructions are for S3QL 2.24 and should be applicable to any system. The \href{https://bitbucket.org/nikratio/s3ql/wiki/Home}{S3QL Wiki} contains \href{https://bitbucket.org/nikratio/s3ql/wiki/Installation}{additional help} help for specific distributions and operating systems. Note, however, that @@ -347,7 +347,7 @@ installed version if the module is installed. \section{Installing S3QL} -\label{installation:installing-s3ql}\label{installation:inst-s3ql} +\label{installation:inst-s3ql}\label{installation:installing-s3ql} To build and install S3QL itself, proceed as follows: \begin{enumerate} \item {} @@ -467,7 +467,7 @@ being skipped by passing the \sphinxcode{-rs} argument to \chapter{Storage Backends} -\label{backends:sphinx}\label{backends::doc}\label{backends:id1}\label{backends:storage-backends} +\label{backends:id1}\label{backends::doc}\label{backends:storage-backends}\label{backends:sphinx} S3QL supports different \emph{backends} to store data at different service providers and using different protocols. A \emph{storage url} specifies a backend together with some backend-specific information and uniquely @@ -572,7 +572,7 @@ TCP connection is closed and re-established (default: 20 seconds). \section{Amazon S3} -\label{backends:google-storage-manager}\label{backends:amazon-s3} +\label{backends:amazon-s3}\label{backends:google-storage-manager} \href{http://aws.amazon.com/s3}{Amazon S3} is the online storage service offered by \href{http://aws.amazon.com/}{Amazon Web Services (AWS)}. To use the S3 backend, you first need to sign up for an AWS account. The @@ -1183,7 +1183,7 @@ neglected over long periods of time. \chapter{File System Creation} -\label{mkfs:file-system-creation}\label{mkfs::doc} +\label{mkfs::doc}\label{mkfs:file-system-creation} A S3QL file system is created with the \textbf{\texttt{mkfs.s3ql}} command. It has the following syntax: @@ -1260,7 +1260,7 @@ one at \sphinxcode{s3://bucketname/outerprefix/innerprefix}. \chapter{Managing File Systems} -\label{adm:managing-file-systems}\label{adm::doc} +\label{adm::doc}\label{adm:managing-file-systems} The \sphinxcode{s3qladm} command performs various operations on \emph{unmounted} S3QL file systems. The file system \emph{must not be mounted} when using \sphinxcode{s3qladm} or things will go wrong badly. @@ -1379,7 +1379,7 @@ for help on the mailing list first (see {\hyperref[resources:resources]{\sphinxc \chapter{Mounting} -\label{mount:mounting}\label{mount::doc} +\label{mount::doc}\label{mount:mounting} A S3QL file system is mounted with the \textbf{\texttt{mount.s3ql}} command. It has the following syntax: @@ -1734,7 +1734,7 @@ For a full list of available options, run \sphinxcode{s3qlstat -{-}help}. \section{Immutable Trees} -\label{special:immutable-trees}\label{special:s3qllock} +\label{special:s3qllock}\label{special:immutable-trees} The command \textbf{\texttt{s3qllock}} can be used to make a directory tree immutable. Immutable trees can no longer be changed in any way whatsoever. You can not add new files or directories and you can not @@ -1781,7 +1781,7 @@ changed after they have been made immutable. \section{Fast Recursive Removal} -\label{special:fast-recursive-removal}\label{special:s3qlrm} +\label{special:s3qlrm}\label{special:fast-recursive-removal} The \sphinxcode{s3qlrm} command can be used to recursively delete files and directories on an S3QL file system. Although \sphinxcode{s3qlrm} is faster than using e.g. \sphinxcode{rm -r}, the main reason for its existence is that it @@ -1877,7 +1877,7 @@ upload data in the background for a while longer. \chapter{Checking for Errors} -\label{fsck:checking-for-errors}\label{fsck::doc} +\label{fsck::doc}\label{fsck:checking-for-errors} It is recommended to periodically run the \textbf{\texttt{fsck.s3ql}} and \textbf{\texttt{s3ql\_verify}} commands (in this order) to ensure that the file system is consistent, and that there has been no data corruption @@ -1947,6 +1947,9 @@ just print program version and exit If user input is required, exit without prompting. \item [-{-}force] Force checking even if file system is marked clean. +\item [-{-}force-remote] +Force use of remote metadata even when this would +likely result in data loss. \end{optionlist} \end{quote} @@ -2030,7 +2033,7 @@ Skip over first \textless{}n\textgreater{} objects and with verifying object \chapter{Storing Authentication Information} -\label{authinfo:storing-authentication-information}\label{authinfo:authinfo}\label{authinfo::doc} +\label{authinfo:authinfo}\label{authinfo::doc}\label{authinfo:storing-authentication-information} Normally, S3QL reads username and password for the backend as well as an encryption passphrase for the file system from the terminal. Most commands also accept an \sphinxcode{-{-}authfile} parameter that can be @@ -2097,7 +2100,7 @@ module}. \chapter{Contributed Programs} -\label{contrib:contributed-programs}\label{contrib::doc} +\label{contrib::doc}\label{contrib:contributed-programs} S3QL comes with a few contributed programs that are not part of the core distribution (and are therefore not installed automatically by default), but which may nevertheless be useful. These programs are in @@ -2122,7 +2125,7 @@ migrate S3 buckets to a different storage region or storage class \section{pcp.py} -\label{contrib:pcp-py}\label{contrib:pcp} +\label{contrib:pcp}\label{contrib:pcp-py} \sphinxcode{pcp.py} is a wrapper program that starts several rsync processes to copy directory trees in parallel. This is important because transferring files in parallel significantly enhances performance when @@ -2251,14 +2254,14 @@ For a full list of available options, run \textbf{\texttt{expire\_backups.py \section{remove\_objects.py} -\label{contrib:remove-objects-py}\label{contrib:remove-objects} +\label{contrib:remove-objects}\label{contrib:remove-objects-py} \textbf{\texttt{remove\_objects.py}} is a program to remove a list of objects from a storage backend. Since it acts on the backend-level, the backend need not contain an S3QL file system. \chapter{Tips \& Tricks} -\label{tips:tips-tricks}\label{tips::doc} +\label{tips::doc}\label{tips:tips-tricks} \section{SSH Backend} \label{tips:ssh-backend}\label{tips:ssh-tipp} @@ -2302,7 +2305,7 @@ to zero). \section{Improving copy performance} -\label{tips:improving-copy-performance}\label{tips:copy-performance} +\label{tips:copy-performance}\label{tips:improving-copy-performance} \begin{notice}{note}{Note:} The following applies only when copying data \textbf{from} an S3QL file system, \textbf{not} when copying data \textbf{to} an S3QL file system. @@ -2466,7 +2469,7 @@ newer removes this limitation. \chapter{Manpages} -\label{man/index::doc}\label{man/index:manpages} +\label{man/index:manpages}\label{man/index::doc} The man pages are installed with S3QL on your system and can be viewed with the \textbf{\texttt{man}} command. For reference, they are also included here in the User's Guide. @@ -3533,6 +3536,9 @@ just print program version and exit If user input is required, exit without prompting. \item [-{-}force] Force checking even if file system is marked clean. +\item [-{-}force-remote] +Force use of remote metadata even when this would +likely result in data loss. \end{optionlist} \end{quote} @@ -4049,7 +4055,7 @@ Invalid command line argument. \chapter{Further Resources / Getting Help} -\label{resources:resources}\label{resources::doc}\label{resources:further-resources-getting-help} +\label{resources::doc}\label{resources:further-resources-getting-help}\label{resources:resources} If you have questions or problems with S3QL that you weren't able to resolve with this manual, you might want to consider the following other resources: \begin{itemize} @@ -4070,7 +4076,7 @@ Please report any bugs you may encounter in the \href{https://bitbucket.org/nikr \chapter{Implementation Details} -\label{impl_details:impl-details}\label{impl_details::doc}\label{impl_details:implementation-details} +\label{impl_details::doc}\label{impl_details:implementation-details}\label{impl_details:impl-details} This section provides some background information on how S3QL works internally. Reading this section is not necessary to use S3QL. diff --git a/doc/latex/manual.toc b/doc/latex/manual.toc index 0fb732d..8173897 100644 --- a/doc/latex/manual.toc +++ b/doc/latex/manual.toc @@ -1,5 +1,5 @@ \select@language {english} -\contentsline {chapter}{\numberline {1}About S3QL}{1}{chapter.1} +\contentsline {chapter}{\numberline {1}S3QL}{1}{chapter.1} \contentsline {section}{\numberline {1.1}Features}{1}{section.1.1} \contentsline {section}{\numberline {1.2}Development Status}{2}{section.1.2} \contentsline {section}{\numberline {1.3}Supported Platforms}{2}{section.1.3} @@ -46,112 +46,112 @@ \contentsline {chapter}{\numberline {10}Checking for Errors}{31}{chapter.10} \contentsline {section}{\numberline {10.1}Checking and repairing internal file system errors}{31}{section.10.1} \contentsline {section}{\numberline {10.2}Detecting and handling backend data corruption}{32}{section.10.2} -\contentsline {chapter}{\numberline {11}Storing Authentication Information}{33}{chapter.11} -\contentsline {chapter}{\numberline {12}Contributed Programs}{35}{chapter.12} -\contentsline {section}{\numberline {12.1}benchmark.py}{35}{section.12.1} -\contentsline {section}{\numberline {12.2}clone\_fs.py}{35}{section.12.2} -\contentsline {section}{\numberline {12.3}pcp.py}{35}{section.12.3} -\contentsline {section}{\numberline {12.4}s3ql\_backup.sh}{35}{section.12.4} -\contentsline {section}{\numberline {12.5}expire\_backups.py}{36}{section.12.5} -\contentsline {section}{\numberline {12.6}remove\_objects.py}{37}{section.12.6} -\contentsline {chapter}{\numberline {13}Tips \& Tricks}{39}{chapter.13} -\contentsline {section}{\numberline {13.1}SSH Backend}{39}{section.13.1} -\contentsline {section}{\numberline {13.2}Permanently mounted backup file system}{39}{section.13.2} -\contentsline {section}{\numberline {13.3}Improving copy performance}{39}{section.13.3} -\contentsline {chapter}{\numberline {14}Known Issues}{41}{chapter.14} -\contentsline {chapter}{\numberline {15}Manpages}{43}{chapter.15} -\contentsline {section}{\numberline {15.1}The \textbf {\texttt {mkfs.s3ql}} command}{43}{section.15.1} -\contentsline {subsection}{\numberline {15.1.1}Synopsis}{43}{subsection.15.1.1} -\contentsline {subsection}{\numberline {15.1.2}Description}{43}{subsection.15.1.2} -\contentsline {subsection}{\numberline {15.1.3}Options}{43}{subsection.15.1.3} -\contentsline {subsection}{\numberline {15.1.4}Exit Codes}{44}{subsection.15.1.4} -\contentsline {subsection}{\numberline {15.1.5}See Also}{44}{subsection.15.1.5} -\contentsline {section}{\numberline {15.2}The \textbf {\texttt {s3qladm}} command}{44}{section.15.2} -\contentsline {subsection}{\numberline {15.2.1}Synopsis}{44}{subsection.15.2.1} -\contentsline {subsection}{\numberline {15.2.2}Description}{45}{subsection.15.2.2} -\contentsline {subsection}{\numberline {15.2.3}Options}{45}{subsection.15.2.3} -\contentsline {subsection}{\numberline {15.2.4}Actions}{45}{subsection.15.2.4} -\contentsline {subsection}{\numberline {15.2.5}Exit Codes}{45}{subsection.15.2.5} -\contentsline {subsection}{\numberline {15.2.6}See Also}{46}{subsection.15.2.6} -\contentsline {section}{\numberline {15.3}The \textbf {\texttt {mount.s3ql}} command}{46}{section.15.3} -\contentsline {subsection}{\numberline {15.3.1}Synopsis}{46}{subsection.15.3.1} -\contentsline {subsection}{\numberline {15.3.2}Description}{46}{subsection.15.3.2} -\contentsline {subsection}{\numberline {15.3.3}Options}{46}{subsection.15.3.3} -\contentsline {subsection}{\numberline {15.3.4}Exit Codes}{47}{subsection.15.3.4} -\contentsline {subsection}{\numberline {15.3.5}See Also}{48}{subsection.15.3.5} -\contentsline {section}{\numberline {15.4}The \textbf {\texttt {s3qlstat}} command}{48}{section.15.4} -\contentsline {subsection}{\numberline {15.4.1}Synopsis}{48}{subsection.15.4.1} -\contentsline {subsection}{\numberline {15.4.2}Description}{48}{subsection.15.4.2} -\contentsline {subsection}{\numberline {15.4.3}Options}{49}{subsection.15.4.3} -\contentsline {subsection}{\numberline {15.4.4}Exit Codes}{49}{subsection.15.4.4} -\contentsline {subsection}{\numberline {15.4.5}See Also}{49}{subsection.15.4.5} -\contentsline {section}{\numberline {15.5}The \textbf {\texttt {s3qlctrl}} command}{49}{section.15.5} -\contentsline {subsection}{\numberline {15.5.1}Synopsis}{49}{subsection.15.5.1} -\contentsline {subsection}{\numberline {15.5.2}Description}{49}{subsection.15.5.2} -\contentsline {subsection}{\numberline {15.5.3}Options}{50}{subsection.15.5.3} -\contentsline {subsection}{\numberline {15.5.4}Exit Codes}{50}{subsection.15.5.4} -\contentsline {subsection}{\numberline {15.5.5}See Also}{50}{subsection.15.5.5} -\contentsline {section}{\numberline {15.6}The \textbf {\texttt {s3qlcp}} command}{50}{section.15.6} -\contentsline {subsection}{\numberline {15.6.1}Synopsis}{50}{subsection.15.6.1} -\contentsline {subsection}{\numberline {15.6.2}Description}{51}{subsection.15.6.2} -\contentsline {subsubsection}{Snapshotting vs Hardlinking}{51}{subsubsection*.22} -\contentsline {subsection}{\numberline {15.6.3}Options}{51}{subsection.15.6.3} -\contentsline {subsection}{\numberline {15.6.4}Exit Codes}{51}{subsection.15.6.4} -\contentsline {subsection}{\numberline {15.6.5}See Also}{52}{subsection.15.6.5} -\contentsline {section}{\numberline {15.7}The \textbf {\texttt {s3qlrm}} command}{52}{section.15.7} -\contentsline {subsection}{\numberline {15.7.1}Synopsis}{52}{subsection.15.7.1} -\contentsline {subsection}{\numberline {15.7.2}Description}{52}{subsection.15.7.2} -\contentsline {subsection}{\numberline {15.7.3}Options}{52}{subsection.15.7.3} -\contentsline {subsection}{\numberline {15.7.4}Exit Codes}{52}{subsection.15.7.4} -\contentsline {subsection}{\numberline {15.7.5}See Also}{53}{subsection.15.7.5} -\contentsline {section}{\numberline {15.8}The \textbf {\texttt {s3qllock}} command}{53}{section.15.8} -\contentsline {subsection}{\numberline {15.8.1}Synopsis}{53}{subsection.15.8.1} -\contentsline {subsection}{\numberline {15.8.2}Description}{53}{subsection.15.8.2} -\contentsline {subsection}{\numberline {15.8.3}Rationale}{53}{subsection.15.8.3} -\contentsline {subsection}{\numberline {15.8.4}Options}{53}{subsection.15.8.4} -\contentsline {subsection}{\numberline {15.8.5}Exit Codes}{54}{subsection.15.8.5} -\contentsline {subsection}{\numberline {15.8.6}See Also}{54}{subsection.15.8.6} -\contentsline {section}{\numberline {15.9}The \textbf {\texttt {umount.s3ql}} command}{54}{section.15.9} -\contentsline {subsection}{\numberline {15.9.1}Synopsis}{54}{subsection.15.9.1} -\contentsline {subsection}{\numberline {15.9.2}Description}{54}{subsection.15.9.2} -\contentsline {subsection}{\numberline {15.9.3}Options}{54}{subsection.15.9.3} -\contentsline {subsection}{\numberline {15.9.4}Exit Codes}{55}{subsection.15.9.4} -\contentsline {subsection}{\numberline {15.9.5}See Also}{55}{subsection.15.9.5} -\contentsline {section}{\numberline {15.10}The \textbf {\texttt {fsck.s3ql}} command}{55}{section.15.10} -\contentsline {subsection}{\numberline {15.10.1}Synopsis}{55}{subsection.15.10.1} -\contentsline {subsection}{\numberline {15.10.2}Description}{55}{subsection.15.10.2} -\contentsline {subsection}{\numberline {15.10.3}Options}{55}{subsection.15.10.3} -\contentsline {subsection}{\numberline {15.10.4}Exit Codes}{56}{subsection.15.10.4} -\contentsline {subsection}{\numberline {15.10.5}See Also}{57}{subsection.15.10.5} -\contentsline {section}{\numberline {15.11}The \textbf {\texttt {s3ql\_oauth\_client}} command}{57}{section.15.11} -\contentsline {subsection}{\numberline {15.11.1}Synopsis}{57}{subsection.15.11.1} -\contentsline {subsection}{\numberline {15.11.2}Description}{57}{subsection.15.11.2} -\contentsline {subsection}{\numberline {15.11.3}Options}{57}{subsection.15.11.3} -\contentsline {subsection}{\numberline {15.11.4}Exit Codes}{57}{subsection.15.11.4} -\contentsline {subsection}{\numberline {15.11.5}See Also}{58}{subsection.15.11.5} -\contentsline {section}{\numberline {15.12}The \textbf {\texttt {s3ql\_verify}} command}{58}{section.15.12} -\contentsline {subsection}{\numberline {15.12.1}Synopsis}{58}{subsection.15.12.1} -\contentsline {subsection}{\numberline {15.12.2}Description}{58}{subsection.15.12.2} -\contentsline {subsection}{\numberline {15.12.3}Options}{58}{subsection.15.12.3} -\contentsline {subsection}{\numberline {15.12.4}Exit Codes}{59}{subsection.15.12.4} -\contentsline {subsection}{\numberline {15.12.5}See Also}{59}{subsection.15.12.5} -\contentsline {section}{\numberline {15.13}The \textbf {\texttt {pcp}} command}{59}{section.15.13} -\contentsline {subsection}{\numberline {15.13.1}Synopsis}{59}{subsection.15.13.1} -\contentsline {subsection}{\numberline {15.13.2}Description}{59}{subsection.15.13.2} -\contentsline {subsection}{\numberline {15.13.3}Options}{60}{subsection.15.13.3} -\contentsline {subsection}{\numberline {15.13.4}Exit Codes}{60}{subsection.15.13.4} -\contentsline {subsection}{\numberline {15.13.5}See Also}{60}{subsection.15.13.5} -\contentsline {section}{\numberline {15.14}The \textbf {\texttt {expire\_backups}} command}{60}{section.15.14} -\contentsline {subsection}{\numberline {15.14.1}Synopsis}{60}{subsection.15.14.1} -\contentsline {subsection}{\numberline {15.14.2}Description}{60}{subsection.15.14.2} -\contentsline {subsection}{\numberline {15.14.3}Options}{61}{subsection.15.14.3} -\contentsline {subsection}{\numberline {15.14.4}Exit Codes}{62}{subsection.15.14.4} -\contentsline {subsection}{\numberline {15.14.5}See Also}{62}{subsection.15.14.5} -\contentsline {chapter}{\numberline {16}Further Resources / Getting Help}{63}{chapter.16} -\contentsline {chapter}{\numberline {17}Implementation Details}{65}{chapter.17} -\contentsline {section}{\numberline {17.1}Metadata Storage}{65}{section.17.1} -\contentsline {section}{\numberline {17.2}Data Storage}{65}{section.17.2} -\contentsline {section}{\numberline {17.3}Data De-Duplication}{66}{section.17.3} -\contentsline {section}{\numberline {17.4}Caching}{66}{section.17.4} -\contentsline {section}{\numberline {17.5}Eventual Consistency Handling}{66}{section.17.5} -\contentsline {section}{\numberline {17.6}Encryption}{66}{section.17.6} +\contentsline {chapter}{\numberline {11}Storing Authentication Information}{35}{chapter.11} +\contentsline {chapter}{\numberline {12}Contributed Programs}{37}{chapter.12} +\contentsline {section}{\numberline {12.1}benchmark.py}{37}{section.12.1} +\contentsline {section}{\numberline {12.2}clone\_fs.py}{37}{section.12.2} +\contentsline {section}{\numberline {12.3}pcp.py}{37}{section.12.3} +\contentsline {section}{\numberline {12.4}s3ql\_backup.sh}{37}{section.12.4} +\contentsline {section}{\numberline {12.5}expire\_backups.py}{38}{section.12.5} +\contentsline {section}{\numberline {12.6}remove\_objects.py}{39}{section.12.6} +\contentsline {chapter}{\numberline {13}Tips \& Tricks}{41}{chapter.13} +\contentsline {section}{\numberline {13.1}SSH Backend}{41}{section.13.1} +\contentsline {section}{\numberline {13.2}Permanently mounted backup file system}{41}{section.13.2} +\contentsline {section}{\numberline {13.3}Improving copy performance}{41}{section.13.3} +\contentsline {chapter}{\numberline {14}Known Issues}{43}{chapter.14} +\contentsline {chapter}{\numberline {15}Manpages}{45}{chapter.15} +\contentsline {section}{\numberline {15.1}The \textbf {\texttt {mkfs.s3ql}} command}{45}{section.15.1} +\contentsline {subsection}{\numberline {15.1.1}Synopsis}{45}{subsection.15.1.1} +\contentsline {subsection}{\numberline {15.1.2}Description}{45}{subsection.15.1.2} +\contentsline {subsection}{\numberline {15.1.3}Options}{45}{subsection.15.1.3} +\contentsline {subsection}{\numberline {15.1.4}Exit Codes}{46}{subsection.15.1.4} +\contentsline {subsection}{\numberline {15.1.5}See Also}{46}{subsection.15.1.5} +\contentsline {section}{\numberline {15.2}The \textbf {\texttt {s3qladm}} command}{46}{section.15.2} +\contentsline {subsection}{\numberline {15.2.1}Synopsis}{46}{subsection.15.2.1} +\contentsline {subsection}{\numberline {15.2.2}Description}{47}{subsection.15.2.2} +\contentsline {subsection}{\numberline {15.2.3}Options}{47}{subsection.15.2.3} +\contentsline {subsection}{\numberline {15.2.4}Actions}{47}{subsection.15.2.4} +\contentsline {subsection}{\numberline {15.2.5}Exit Codes}{47}{subsection.15.2.5} +\contentsline {subsection}{\numberline {15.2.6}See Also}{48}{subsection.15.2.6} +\contentsline {section}{\numberline {15.3}The \textbf {\texttt {mount.s3ql}} command}{48}{section.15.3} +\contentsline {subsection}{\numberline {15.3.1}Synopsis}{48}{subsection.15.3.1} +\contentsline {subsection}{\numberline {15.3.2}Description}{48}{subsection.15.3.2} +\contentsline {subsection}{\numberline {15.3.3}Options}{48}{subsection.15.3.3} +\contentsline {subsection}{\numberline {15.3.4}Exit Codes}{49}{subsection.15.3.4} +\contentsline {subsection}{\numberline {15.3.5}See Also}{50}{subsection.15.3.5} +\contentsline {section}{\numberline {15.4}The \textbf {\texttt {s3qlstat}} command}{50}{section.15.4} +\contentsline {subsection}{\numberline {15.4.1}Synopsis}{50}{subsection.15.4.1} +\contentsline {subsection}{\numberline {15.4.2}Description}{50}{subsection.15.4.2} +\contentsline {subsection}{\numberline {15.4.3}Options}{51}{subsection.15.4.3} +\contentsline {subsection}{\numberline {15.4.4}Exit Codes}{51}{subsection.15.4.4} +\contentsline {subsection}{\numberline {15.4.5}See Also}{51}{subsection.15.4.5} +\contentsline {section}{\numberline {15.5}The \textbf {\texttt {s3qlctrl}} command}{51}{section.15.5} +\contentsline {subsection}{\numberline {15.5.1}Synopsis}{51}{subsection.15.5.1} +\contentsline {subsection}{\numberline {15.5.2}Description}{51}{subsection.15.5.2} +\contentsline {subsection}{\numberline {15.5.3}Options}{52}{subsection.15.5.3} +\contentsline {subsection}{\numberline {15.5.4}Exit Codes}{52}{subsection.15.5.4} +\contentsline {subsection}{\numberline {15.5.5}See Also}{52}{subsection.15.5.5} +\contentsline {section}{\numberline {15.6}The \textbf {\texttt {s3qlcp}} command}{52}{section.15.6} +\contentsline {subsection}{\numberline {15.6.1}Synopsis}{52}{subsection.15.6.1} +\contentsline {subsection}{\numberline {15.6.2}Description}{53}{subsection.15.6.2} +\contentsline {subsubsection}{Snapshotting vs Hardlinking}{53}{subsubsection*.22} +\contentsline {subsection}{\numberline {15.6.3}Options}{53}{subsection.15.6.3} +\contentsline {subsection}{\numberline {15.6.4}Exit Codes}{53}{subsection.15.6.4} +\contentsline {subsection}{\numberline {15.6.5}See Also}{54}{subsection.15.6.5} +\contentsline {section}{\numberline {15.7}The \textbf {\texttt {s3qlrm}} command}{54}{section.15.7} +\contentsline {subsection}{\numberline {15.7.1}Synopsis}{54}{subsection.15.7.1} +\contentsline {subsection}{\numberline {15.7.2}Description}{54}{subsection.15.7.2} +\contentsline {subsection}{\numberline {15.7.3}Options}{54}{subsection.15.7.3} +\contentsline {subsection}{\numberline {15.7.4}Exit Codes}{54}{subsection.15.7.4} +\contentsline {subsection}{\numberline {15.7.5}See Also}{55}{subsection.15.7.5} +\contentsline {section}{\numberline {15.8}The \textbf {\texttt {s3qllock}} command}{55}{section.15.8} +\contentsline {subsection}{\numberline {15.8.1}Synopsis}{55}{subsection.15.8.1} +\contentsline {subsection}{\numberline {15.8.2}Description}{55}{subsection.15.8.2} +\contentsline {subsection}{\numberline {15.8.3}Rationale}{55}{subsection.15.8.3} +\contentsline {subsection}{\numberline {15.8.4}Options}{55}{subsection.15.8.4} +\contentsline {subsection}{\numberline {15.8.5}Exit Codes}{56}{subsection.15.8.5} +\contentsline {subsection}{\numberline {15.8.6}See Also}{56}{subsection.15.8.6} +\contentsline {section}{\numberline {15.9}The \textbf {\texttt {umount.s3ql}} command}{56}{section.15.9} +\contentsline {subsection}{\numberline {15.9.1}Synopsis}{56}{subsection.15.9.1} +\contentsline {subsection}{\numberline {15.9.2}Description}{56}{subsection.15.9.2} +\contentsline {subsection}{\numberline {15.9.3}Options}{56}{subsection.15.9.3} +\contentsline {subsection}{\numberline {15.9.4}Exit Codes}{57}{subsection.15.9.4} +\contentsline {subsection}{\numberline {15.9.5}See Also}{57}{subsection.15.9.5} +\contentsline {section}{\numberline {15.10}The \textbf {\texttt {fsck.s3ql}} command}{57}{section.15.10} +\contentsline {subsection}{\numberline {15.10.1}Synopsis}{57}{subsection.15.10.1} +\contentsline {subsection}{\numberline {15.10.2}Description}{57}{subsection.15.10.2} +\contentsline {subsection}{\numberline {15.10.3}Options}{57}{subsection.15.10.3} +\contentsline {subsection}{\numberline {15.10.4}Exit Codes}{58}{subsection.15.10.4} +\contentsline {subsection}{\numberline {15.10.5}See Also}{59}{subsection.15.10.5} +\contentsline {section}{\numberline {15.11}The \textbf {\texttt {s3ql\_oauth\_client}} command}{59}{section.15.11} +\contentsline {subsection}{\numberline {15.11.1}Synopsis}{59}{subsection.15.11.1} +\contentsline {subsection}{\numberline {15.11.2}Description}{59}{subsection.15.11.2} +\contentsline {subsection}{\numberline {15.11.3}Options}{59}{subsection.15.11.3} +\contentsline {subsection}{\numberline {15.11.4}Exit Codes}{59}{subsection.15.11.4} +\contentsline {subsection}{\numberline {15.11.5}See Also}{60}{subsection.15.11.5} +\contentsline {section}{\numberline {15.12}The \textbf {\texttt {s3ql\_verify}} command}{60}{section.15.12} +\contentsline {subsection}{\numberline {15.12.1}Synopsis}{60}{subsection.15.12.1} +\contentsline {subsection}{\numberline {15.12.2}Description}{60}{subsection.15.12.2} +\contentsline {subsection}{\numberline {15.12.3}Options}{60}{subsection.15.12.3} +\contentsline {subsection}{\numberline {15.12.4}Exit Codes}{61}{subsection.15.12.4} +\contentsline {subsection}{\numberline {15.12.5}See Also}{61}{subsection.15.12.5} +\contentsline {section}{\numberline {15.13}The \textbf {\texttt {pcp}} command}{61}{section.15.13} +\contentsline {subsection}{\numberline {15.13.1}Synopsis}{61}{subsection.15.13.1} +\contentsline {subsection}{\numberline {15.13.2}Description}{61}{subsection.15.13.2} +\contentsline {subsection}{\numberline {15.13.3}Options}{62}{subsection.15.13.3} +\contentsline {subsection}{\numberline {15.13.4}Exit Codes}{62}{subsection.15.13.4} +\contentsline {subsection}{\numberline {15.13.5}See Also}{62}{subsection.15.13.5} +\contentsline {section}{\numberline {15.14}The \textbf {\texttt {expire\_backups}} command}{62}{section.15.14} +\contentsline {subsection}{\numberline {15.14.1}Synopsis}{62}{subsection.15.14.1} +\contentsline {subsection}{\numberline {15.14.2}Description}{62}{subsection.15.14.2} +\contentsline {subsection}{\numberline {15.14.3}Options}{63}{subsection.15.14.3} +\contentsline {subsection}{\numberline {15.14.4}Exit Codes}{64}{subsection.15.14.4} +\contentsline {subsection}{\numberline {15.14.5}See Also}{64}{subsection.15.14.5} +\contentsline {chapter}{\numberline {16}Further Resources / Getting Help}{65}{chapter.16} +\contentsline {chapter}{\numberline {17}Implementation Details}{67}{chapter.17} +\contentsline {section}{\numberline {17.1}Metadata Storage}{67}{section.17.1} +\contentsline {section}{\numberline {17.2}Data Storage}{67}{section.17.2} +\contentsline {section}{\numberline {17.3}Data De-Duplication}{68}{section.17.3} +\contentsline {section}{\numberline {17.4}Caching}{68}{section.17.4} +\contentsline {section}{\numberline {17.5}Eventual Consistency Handling}{68}{section.17.5} +\contentsline {section}{\numberline {17.6}Encryption}{68}{section.17.6} diff --git a/doc/man/fsck.s3ql.1 b/doc/man/fsck.s3ql.1 index e75e344..afa100d 100644 --- a/doc/man/fsck.s3ql.1 +++ b/doc/man/fsck.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "FSCK.S3QL" "1" "Aug 18, 2017" "2.23" "S3QL" +.TH "FSCK.S3QL" "1" "Nov 04, 2017" "2.24" "S3QL" .SH NAME fsck.s3ql \- Check an S3QL file system for errors . @@ -100,6 +100,10 @@ If user input is required, exit without prompting. .TP .B \-\-force Force checking even if file system is marked clean. +.TP +.B \-\-force\-remote +Force use of remote metadata even when this would +likely result in data loss. .UNINDENT .UNINDENT .UNINDENT diff --git a/doc/man/mkfs.s3ql.1 b/doc/man/mkfs.s3ql.1 index 6f5969e..4c2b38b 100644 --- a/doc/man/mkfs.s3ql.1 +++ b/doc/man/mkfs.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "MKFS.S3QL" "1" "Aug 18, 2017" "2.23" "S3QL" +.TH "MKFS.S3QL" "1" "Nov 04, 2017" "2.24" "S3QL" .SH NAME mkfs.s3ql \- Create an S3QL file system . diff --git a/doc/man/mount.s3ql.1 b/doc/man/mount.s3ql.1 index 09351f3..ba28133 100644 --- a/doc/man/mount.s3ql.1 +++ b/doc/man/mount.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "MOUNT.S3QL" "1" "Aug 18, 2017" "2.23" "S3QL" +.TH "MOUNT.S3QL" "1" "Nov 04, 2017" "2.24" "S3QL" .SH NAME mount.s3ql \- Mount an S3QL file system . diff --git a/doc/man/s3ql_oauth_client.1 b/doc/man/s3ql_oauth_client.1 index b647f73..1556ae6 100644 --- a/doc/man/s3ql_oauth_client.1 +++ b/doc/man/s3ql_oauth_client.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QL_OAUTH_CLIENT" "1" "Aug 18, 2017" "2.23" "S3QL" +.TH "S3QL_OAUTH_CLIENT" "1" "Nov 04, 2017" "2.24" "S3QL" .SH NAME s3ql_oauth_client \- Obtain Google Storage OAuth2 tokens . diff --git a/doc/man/s3ql_verify.1 b/doc/man/s3ql_verify.1 index 3f55c42..38ebddf 100644 --- a/doc/man/s3ql_verify.1 +++ b/doc/man/s3ql_verify.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QL_VERIFY" "1" "Aug 18, 2017" "2.23" "S3QL" +.TH "S3QL_VERIFY" "1" "Nov 04, 2017" "2.24" "S3QL" .SH NAME s3ql_verify \- Verify data in an S3QL file system . diff --git a/doc/man/s3qladm.1 b/doc/man/s3qladm.1 index 72c84df..5ac553c 100644 --- a/doc/man/s3qladm.1 +++ b/doc/man/s3qladm.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLADM" "1" "Aug 18, 2017" "2.23" "S3QL" +.TH "S3QLADM" "1" "Nov 04, 2017" "2.24" "S3QL" .SH NAME s3qladm \- Manage S3QL file systems . diff --git a/doc/man/s3qlcp.1 b/doc/man/s3qlcp.1 index 1f876db..dc12671 100644 --- a/doc/man/s3qlcp.1 +++ b/doc/man/s3qlcp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLCP" "1" "Aug 18, 2017" "2.23" "S3QL" +.TH "S3QLCP" "1" "Nov 04, 2017" "2.24" "S3QL" .SH NAME s3qlcp \- Copy-on-write replication on S3QL file systems . diff --git a/doc/man/s3qlctrl.1 b/doc/man/s3qlctrl.1 index fd2232f..a764420 100644 --- a/doc/man/s3qlctrl.1 +++ b/doc/man/s3qlctrl.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLCTRL" "1" "Aug 18, 2017" "2.23" "S3QL" +.TH "S3QLCTRL" "1" "Nov 04, 2017" "2.24" "S3QL" .SH NAME s3qlctrl \- Control a mounted S3QL file system . diff --git a/doc/man/s3qllock.1 b/doc/man/s3qllock.1 index a0b4cac..a789752 100644 --- a/doc/man/s3qllock.1 +++ b/doc/man/s3qllock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLLOCK" "1" "Aug 18, 2017" "2.23" "S3QL" +.TH "S3QLLOCK" "1" "Nov 04, 2017" "2.24" "S3QL" .SH NAME s3qllock \- Make trees on an S3QL file system immutable . diff --git a/doc/man/s3qlrm.1 b/doc/man/s3qlrm.1 index a4fa71a..61bfd1d 100644 --- a/doc/man/s3qlrm.1 +++ b/doc/man/s3qlrm.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLRM" "1" "Aug 18, 2017" "2.23" "S3QL" +.TH "S3QLRM" "1" "Nov 04, 2017" "2.24" "S3QL" .SH NAME s3qlrm \- Fast tree removal on S3QL file systems . diff --git a/doc/man/s3qlstat.1 b/doc/man/s3qlstat.1 index 074621f..53629ad 100644 --- a/doc/man/s3qlstat.1 +++ b/doc/man/s3qlstat.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLSTAT" "1" "Aug 18, 2017" "2.23" "S3QL" +.TH "S3QLSTAT" "1" "Nov 04, 2017" "2.24" "S3QL" .SH NAME s3qlstat \- Gather S3QL file system statistics . diff --git a/doc/man/umount.s3ql.1 b/doc/man/umount.s3ql.1 index da1f68e..e1e6fbe 100644 --- a/doc/man/umount.s3ql.1 +++ b/doc/man/umount.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "UMOUNT.S3QL" "1" "Aug 18, 2017" "2.23" "S3QL" +.TH "UMOUNT.S3QL" "1" "Nov 04, 2017" "2.24" "S3QL" .SH NAME umount.s3ql \- Unmount an S3QL file system . diff --git a/doc/manual.pdf b/doc/manual.pdf Binary files differindex c8c85ff..1363116 100644 --- a/doc/manual.pdf +++ b/doc/manual.pdf @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# -*- coding: utf-8 -*- ''' setup.py - this file is part of S3QL. @@ -123,7 +124,8 @@ def main(): # (otherwise we break forward compatibility because compilation with newer # compiler may fail if additional warnings are added) if DEVELOPER_MODE: - compile_args.append('-Werror') + if os.environ.get('CI') != 'true': + compile_args.append('-Werror') compile_args.append('-Wfatal-errors') compile_args.append('-Wno-unused-function') diff --git a/src/s3ql.egg-info/PKG-INFO b/src/s3ql.egg-info/PKG-INFO index 44641a4..71674df 100644 --- a/src/s3ql.egg-info/PKG-INFO +++ b/src/s3ql.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: s3ql -Version: 2.23 +Version: 2.24 Summary: a full-featured file system for online data storage Home-page: https://bitbucket.org/nikratio/s3ql/ Author: Nikolaus Rath @@ -11,9 +11,9 @@ Description: .. NOTE: We cannot use sophisticated ReST syntax here because this file is rendered by Bitbucket. - ============ - About S3QL - ============ + ====== + S3QL + ====== S3QL is a file system that stores all its data online using storage services like `Google Storage`_, `Amazon S3`_, or OpenStack_. S3QL @@ -119,12 +119,12 @@ Description: .. *nikratio-s3ql-bucket* is the S3 bucket in which the file system will be stored. :: - mkfs.s3ql s3://nikratio-s3ql-bucket + mkfs.s3ql s3://ap-south-1/nikratio-s3ql-bucket To mount the S3QL file system stored in the S3 bucket *nikratio_s3ql_bucket* in the directory ``/mnt/s3ql``, enter:: - mount.s3ql s3://nikratio-s3ql-bucket /mnt/s3ql + mount.s3ql s3://ap-south-1/nikratio-s3ql-bucket /mnt/s3ql Now you can instruct your favorite backup program to run a backup into the directory ``/mnt/s3ql`` and the data will be stored an Amazon diff --git a/src/s3ql.egg-info/requires.txt b/src/s3ql.egg-info/requires.txt index c4213dc..6495f17 100644 --- a/src/s3ql.egg-info/requires.txt +++ b/src/s3ql.egg-info/requires.txt @@ -1,6 +1,6 @@ -apsw >= 3.7.0 -defusedxml -dugong >= 3.4, < 4.0 -llfuse >= 1.0, < 2.0 +apsw>=3.7.0 pycrypto requests +defusedxml +dugong<4.0,>=3.4 +llfuse<2.0,>=1.0 diff --git a/src/s3ql.egg-info/top_level.txt b/src/s3ql.egg-info/top_level.txt index fdb8283..9d43dc9 100644 --- a/src/s3ql.egg-info/top_level.txt +++ b/src/s3ql.egg-info/top_level.txt @@ -1,2 +1 @@ -dugong s3ql diff --git a/src/s3ql/__init__.py b/src/s3ql/__init__.py index 5e9bca8..3a1a67d 100644 --- a/src/s3ql/__init__.py +++ b/src/s3ql/__init__.py @@ -22,7 +22,7 @@ __all__ = [ 'adm', 'backends', 'block_cache', 'common', 'calc_mro', 'REV_VER_MAP', 'RELEASE', 'BUFSIZE', 'CTRL_NAME', 'CTRL_INODE' ] -VERSION = '2.23' +VERSION = '2.24' RELEASE = '%s' % VERSION # TODO: On next revision bump, consider removing support for TIME diff --git a/src/s3ql/backends/s3.py b/src/s3ql/backends/s3.py index 30b4916..67e579e 100644 --- a/src/s3ql/backends/s3.py +++ b/src/s3ql/backends/s3.py @@ -58,6 +58,8 @@ class Backend(s3c.Backend): if self.region == 'us-east-1': hostname = 's3.amazonaws.com' + elif self.region.startswith('cn-'): + hostname = 's3.%s.amazonaws.com.cn' % self.region else: hostname = 's3-%s.amazonaws.com' % self.region diff --git a/src/s3ql/backends/swift.py b/src/s3ql/backends/swift.py index 2774fa3..fa65c14 100644 --- a/src/s3ql/backends/swift.py +++ b/src/s3ql/backends/swift.py @@ -195,10 +195,16 @@ class Backend(AbstractBackend, metaclass=ABCDocstMeta): # fall through to scheme used for authentication pass - self._detect_features(o.hostname, o.port) + # mock server can only handle one connection at a time + # so we explicitly disconnect this connection before + # opening the feature detection connection + # (mock server handles both - storage and authentication) + conn.disconnect() - conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy, - ssl_context=ssl_context) + self._detect_features(o.hostname, o.port, ssl_context) + + conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy, + ssl_context=ssl_context) conn.timeout = int(self.options.get('tcp-timeout', 20)) return conn @@ -250,12 +256,8 @@ class Backend(AbstractBackend, metaclass=ABCDocstMeta): # Expired auth token if resp.status == 401: - log.info('OpenStack auth token seems to have expired, requesting new one.') - self.conn.disconnect() - # Force constructing a new connection with a new token, otherwise - # the connection will be reestablished with the same token. - self.conn = None - raise AuthenticationExpired(resp.reason) + self._do_authentication_expired(resp.reason) + # raises AuthenticationExpired # If method == HEAD, server must not return response body # even in case of errors @@ -477,7 +479,7 @@ class Backend(AbstractBackend, metaclass=ABCDocstMeta): if resp.status is not 200: raise HTTPError(resp.status, resp.reason, resp.headers) - hit = re.match('^application/json(;\s*charset="?(.+?)"?)?$', + hit = re.match(r'^application/json(;\s*charset="?(.+?)"?)?$', resp.headers['content-type']) if not hit: log.error('Unexpected server response. Expected json, got:\n%s', @@ -489,14 +491,14 @@ class Backend(AbstractBackend, metaclass=ABCDocstMeta): # but json.loads discards these whitespace characters automatically resp_dict = json.loads(self.conn.readall().decode(hit.group(2) or 'utf-8')) - hit = re.match('^([0-9]{3})', resp_dict['Response Status']) - if not hit: - log.error('Unexpected server response. Expected valid Response Status, got:\n%s', - resp_dict) + log.debug('Response %s', resp_dict) + + try: + resp_status_code, resp_status_text = _split_response_status(resp_dict['Response Status']) + except ValueError: raise RuntimeError('Unexpected server reply') - resp_status = int(hit.group(1)) - if resp_status is 200: + if resp_status_code is 200: # No errors occured, everything has been deleted del keys[:] return @@ -507,7 +509,7 @@ class Backend(AbstractBackend, metaclass=ABCDocstMeta): offset = len(esc_prefix) for error in resp_dict['Errors']: fullkey = error[0] - # stangely the name is url encoded in JSON + # strangely the name is url encoded in JSON assert fullkey.startswith(esc_prefix) key = urllib.parse.unquote(fullkey[offset:]) failed_keys.append(key) @@ -516,27 +518,52 @@ class Backend(AbstractBackend, metaclass=ABCDocstMeta): if key not in failed_keys: keys.remove(key) - # If *force*, just modify the passed list and return without - # raising an exception, otherwise raise exception for the first error - if force: - return - - if resp_status in (400, 404) and len(resp_dict['Errors']) == 0: + if resp_status_code in (400, 404) and len(resp_dict['Errors']) == 0: # Swift returns 400 instead of 404 when files were not found. # (but we also accept the correct status code 404 if Swift # decides to correct this in the future) # ensure that we actually have objects that were not found # (otherwise there is a logic error that we need to know about) - assert(resp_dict['Number Not Found'] > 0) + assert resp_dict['Number Not Found'] > 0 - # Unfortunately we cannot find out from the response which object - # was actually not found. # Since AbstractBackend.delete_multi allows this, we just # swallow this error even when *force* is False. + # N.B.: We removed even the keys from *keys* that are not found. + # This is because Swift only returns a counter of deleted + # objects, not the list of deleted objects (as S3 does). return - raise HTTPError(resp_status, resp_dict['Response Body'], {}) + # At this point it is clear that the server has sent some kind of error response. + # Swift is not very consistent in returning errors. + # We need to jump through these hoops to get something meaningful. + + error_msg = resp_dict['Response Body'] + error_code = resp_status_code + if not error_msg: + if len(resp_dict['Errors']) > 0: + error_code, error_msg = _split_response_status(resp_dict['Errors'][0][1]) + else: + error_msg = resp_status_text + + if error_code == 401: + # Expired auth token + self._do_authentication_expired(error_msg) + # raises AuthenticationExpired + if 'Invalid bulk delete.' in error_msg: + error_code = 400 + # change error message to something more meaningful + error_msg = 'Sent a bulk delete with an empty list of keys to delete' + elif 'Max delete failures exceeded' in error_msg: + error_code = 502 + elif 'Maximum Bulk Deletes: ' in error_msg: + # Sent more keys in one bulk delete than allowed + error_code = 413 + elif 'Invalid File Name' in error_msg: + # get returned when file name is too long + error_code = 422 + + raise HTTPError(error_code, error_msg, {}) @copy_ancestor_docstring def delete_multi(self, keys, force=False): @@ -687,7 +714,7 @@ class Backend(AbstractBackend, metaclass=ABCDocstMeta): def close(self): self.conn.disconnect() - def _detect_features(self, hostname, port): + def _detect_features(self, hostname, port, ssl_context): '''Try to figure out the Swift version and supported features by examining the /info endpoint of the storage server. @@ -698,10 +725,6 @@ class Backend(AbstractBackend, metaclass=ABCDocstMeta): log.debug('Skip feature detection') return - ssl_context = self.ssl_context - if 'no-ssl' in self.options: - ssl_context = None - if not port: port = 443 if ssl_context else 80 @@ -724,7 +747,7 @@ class Backend(AbstractBackend, metaclass=ABCDocstMeta): raise HTTPError(resp.status, resp.reason, resp.headers) if resp.status is 200: - hit = re.match('^application/json(;\s*charset="?(.+?)"?)?$', + hit = re.match(r'^application/json(;\s*charset="?(.+?)"?)?$', resp.headers['content-type']) if not hit: log.error("Wrong server response. Expected json. Got: \n%s", @@ -750,10 +773,21 @@ class Backend(AbstractBackend, metaclass=ABCDocstMeta): if info.get('bulk_delete', False): detected_features.has_bulk_delete = True + bulk_delete = info['bulk_delete'] + assert bulk_delete.get('max_failed_deletes', 1000) <= \ + bulk_delete.get('max_deletes_per_request', 10000) + assert bulk_delete.get('max_failed_deletes', 1000) > 0 # The block cache removal queue has a capacity of 1000. - # We do not need bigger values than that (the default maximum is 10000). - detected_features.max_deletes = max(1, min(1000, - int(info['bulk_delete'].get('max_deletes_per_request', 1000)))) + # We do not need bigger values than that. + # We use max_failed_deletes instead of max_deletes_per_request + # because then we can be sure even when all our delete requests + # get rejected we get a complete error list back from the server. + # If we would set the value higher, _delete_multi() would maybe + # delete some entries from the *keys* list that did not get + # deleted and would miss them in a retry. + detected_features.max_deletes = min(1000, + int(bulk_delete.get('max_failed_deletes', 1000))) + log.info('Detected Swift features for %s:%s: %s', hostname, port, detected_features, extra=LOG_ONCE) @@ -763,6 +797,29 @@ class Backend(AbstractBackend, metaclass=ABCDocstMeta): self.features = detected_features + def _do_authentication_expired(self, reason): + '''Closes the current connection and raises AuthenticationExpired''' + log.info('OpenStack auth token seems to have expired, requesting new one.') + self.conn.disconnect() + # Force constructing a new connection with a new token, otherwise + # the connection will be reestablished with the same token. + self.conn = None + raise AuthenticationExpired(reason) + +def _split_response_status(line): + '''Splits a HTTP response line into status code (integer) + and status text. + + Returns 2-tuple (int, string) + + Raises ValueError when line is not parsable''' + hit = re.match('^([0-9]{3})\s+(.*)$', line) + if not hit: + log.error('Expected valid Response Status, got: %s', + line) + raise ValueError('Expected valid Response Status, got: %s' % line) + return (int(hit.group(1)), hit.group(2)) + class AuthenticationExpired(Exception): '''Raised if the provided Authentication Token has expired''' diff --git a/src/s3ql/backends/swiftks.py b/src/s3ql/backends/swiftks.py index 3a41587..e5cf9a6 100644 --- a/src/s3ql/backends/swiftks.py +++ b/src/s3ql/backends/swiftks.py @@ -120,7 +120,7 @@ class Backend(swift.Backend): # fall through to scheme used for authentication pass - self._detect_features(o.hostname, o.port) + self._detect_features(o.hostname, o.port, ssl_context) conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy, ssl_context=ssl_context) diff --git a/src/s3ql/block_cache.py b/src/s3ql/block_cache.py index 6b5e368..e8a02b4 100644 --- a/src/s3ql/block_cache.py +++ b/src/s3ql/block_cache.py @@ -346,12 +346,12 @@ class BlockCache(object): This method should be called without the global lock held. ''' - log.debug('clearing cache...') + log.debug('Dropping cache...') try: with lock: - self.clear() + self.drop() except NoWorkerThreads: - log.error('Unable to flush cache, no upload threads left alive') + log.error('Unable to drop cache, no upload threads left alive') # Signal termination to worker threads. If some of them # terminated prematurely, continue gracefully. @@ -898,7 +898,7 @@ class BlockCache(object): log.debug('finished') - def flush(self, inode, blockno): + def flush_local(self, inode, blockno): """Flush buffers for given block""" try: @@ -908,7 +908,7 @@ class BlockCache(object): el.flush() - def commit(self): + def start_flush(self): """Initiate upload of all dirty blocks When the method returns, all blocks have been registered @@ -927,8 +927,34 @@ class BlockCache(object): self.upload(el) # Releases global lock - def clear(self): - """Clear cache + def flush(self): + """Upload all dirty blocks + + This method releases the global lock. + """ + + log.debug('started') + + while True: + sth_in_transit = False + for el in self.cache.values(): + if not el.dirty: + continue + if el not in self.in_transit: + log.debug('uploading %s..', el) + self.upload(el) # Releases global lock + sth_in_transit = True + + if not sth_in_transit: + break + + log.debug('waiting for transfer threads..') + self.wait() # Releases global lock + + log.debug('finished') + + def drop(self): + """Drop cache This method releases the global lock. """ diff --git a/src/s3ql/common.py b/src/s3ql/common.py index 09216cb..6a7825d 100644 --- a/src/s3ql/common.py +++ b/src/s3ql/common.py @@ -172,7 +172,7 @@ def get_backend_cachedir(storage_url, cachedir): raise QuietError('No permission to access cache directory (%s)' % cachedir, exitcode=45) - return os.path.join(cachedir, _escape(storage_url)) + return os.path.abspath(os.path.join(cachedir, _escape(storage_url))) def sha256_fh(fh): diff --git a/src/s3ql/fs.py b/src/s3ql/fs.py index 1fddd94..dff2b59 100644 --- a/src/s3ql/fs.py +++ b/src/s3ql/fs.py @@ -237,7 +237,8 @@ class Operations(llfuse.Operations): # Handle S3QL commands if id_ == CTRL_INODE: if name == b's3ql_flushcache!': - self.cache.clear() + self.inodes.flush() + self.cache.flush() elif name == b'copy': try: @@ -249,6 +250,7 @@ class Operations(llfuse.Operations): elif name == b'upload-meta': if self.upload_event is not None: + self.inodes.flush() self.upload_event.set() else: raise llfuse.FUSEError(errno.ENOTTY) @@ -440,8 +442,7 @@ class Operations(llfuse.Operations): db = self.db # First we make sure that all blocks are in the database - self.cache.commit() - log.debug('committed cache') + self.cache.start_flush() # Copy target attributes # These come from setxattr, so they may have been deleted @@ -1152,7 +1153,7 @@ class Operations(llfuse.Operations): self.inodes.flush_id(fh) for blockno in range(0, self.inodes[fh].size // self.max_obj_size + 1): - self.cache.flush(fh, blockno) + self.cache.flush_local(fh, blockno) def forget(self, forget_list): log.debug('started with %s', forget_list) diff --git a/src/s3ql/fsck.py b/src/s3ql/fsck.py index 3b03414..61fadba 100644 --- a/src/s3ql/fsck.py +++ b/src/s3ql/fsck.py @@ -155,12 +155,23 @@ class Fsck(object): def check_cache(self): """Commit uncommitted cache files""" - log.info("Checking cached objects...") + log.info("Checking for dirty cache objects...") if not os.path.exists(self.cachedir): return + candidates = os.listdir(self.cachedir) - for filename in os.listdir(self.cachedir): - self.found_errors = True + if sys.stdout.isatty(): + stamp1 = 0 + else: + stamp1 = float('inf') + + for (i, filename) in enumerate(candidates): + stamp2 = time.time() + if stamp2 - stamp1 > 1: + sys.stdout.write('\r..processed %d/%d files (%d%%)..' + % (i, len(candidates), i/len(candidates)*100)) + sys.stdout.flush() + stamp1 = stamp2 match = re.match('^(\\d+)-(\\d+)$', filename) if match: @@ -169,11 +180,30 @@ class Fsck(object): else: raise RuntimeError('Strange file in cache directory: %s' % filename) - self.log_error("Committing block %d of inode %d to backend", blockno, inode) - + # Calculate block checksum with open(os.path.join(self.cachedir, filename), "rb") as fh: size = os.fstat(fh.fileno()).st_size - hash_ = sha256_fh(fh) + hash_should = sha256_fh(fh) + log.debug('%s has checksum %s', filename, hash_should) + + # Check if stored block has same checksum + try: + block_id = self.conn.get_val('SELECT block_id FROM inode_blocks ' + 'WHERE inode=? AND blockno=?', + (inode, blockno,)) + hash_is = self.conn.get_val('SELECT hash FROM blocks WHERE id=?', + (block_id,)) + except NoSuchRowError: + hash_is = None + log.debug('Inode %d, block %d has checksum %s', inode, blockno, + hash_is) + if hash_should == hash_is: + os.unlink(os.path.join(self.cachedir, filename)) + continue + + self.found_errors = True + self.log_error("Writing dirty block %d of inode %d to backend", blockno, inode) + hash_ = hash_should try: (block_id, obj_id) = self.conn.get_row('SELECT id, obj_id FROM blocks WHERE hash=?', (hash_,)) @@ -1135,6 +1165,9 @@ def parse_args(args): help="If user input is required, exit without prompting.") parser.add_argument("--force", action="store_true", default=False, help="Force checking even if file system is marked clean.") + parser.add_argument("--force-remote", action="store_true", default=False, + help="Force use of remote metadata even when this would " + "likely result in data loss.") options = parser.parse_args(args) return options @@ -1210,9 +1243,11 @@ def main(args=None): print('Enter "continue, I know what I am doing" to use the outdated data anyway:', '> ', sep='\n', end='') - if options.batch: + if options.force_remote: + print('(--force-remote specified, continuing anyway)') + elif options.batch: raise QuietError('(in batch mode, exiting)', exitcode=41) - if sys.stdin.readline().strip() != 'continue, I know what I am doing': + elif sys.stdin.readline().strip() != 'continue, I know what I am doing': raise QuietError(exitcode=42) param['seq_no'] = seq_no diff --git a/src/s3ql/metadata.py b/src/s3ql/metadata.py index 7e0d2f2..a6b45a2 100644 --- a/src/s3ql/metadata.py +++ b/src/s3ql/metadata.py @@ -133,7 +133,9 @@ def cycle_metadata(backend, keep=10): pass cycle_fn("s3ql_metadata_new", "s3ql_metadata") - if cycle_fn is backend.copy: + # Note that we can't compare with "is" (maybe because the bound-method + # is re-created on the fly on access?) + if cycle_fn == backend.copy: backend.delete('s3ql_metadata_new') def dump_metadata(db, fh): diff --git a/src/s3ql/mkfs.py b/src/s3ql/mkfs.py index 310a78a..af96612 100644 --- a/src/s3ql/mkfs.py +++ b/src/s3ql/mkfs.py @@ -156,7 +156,7 @@ def main(args=None): param = dict() param['revision'] = CURRENT_FS_REV - param['seq_no'] = 1 + param['seq_no'] = int(time.time()) param['label'] = options.label param['max_obj_size'] = options.max_obj_size * 1024 param['needs_fsck'] = False diff --git a/src/s3ql/mount.py b/src/s3ql/mount.py index 1a559b6..8dfd0da 100644 --- a/src/s3ql/mount.py +++ b/src/s3ql/mount.py @@ -209,9 +209,12 @@ def main(args=None): workers = 1 if options.single else None # use default if options.profile: - prof.runcall(llfuse.main, workers) + ret = prof.runcall(llfuse.main, workers) else: - llfuse.main(workers) + ret = llfuse.main(workers) + + if ret is not None: + raise RuntimeError('Received signal %d, terminating' % (ret,)) # Allow operations to terminate while block_cache is still available # (destroy() will be called again when from llfuse.close(), but at that diff --git a/tests/common.py b/tests/common.py index 9373654..2b1e8b3 100644 --- a/tests/common.py +++ b/tests/common.py @@ -9,16 +9,12 @@ This work can be distributed under the terms of the GNU GPLv3. This module contains common functions used by multiple unit tests. ''' -from contextlib import contextmanager -from functools import wraps -import re import time import os import subprocess import stat import random import configparser -import logging import pytest import functools diff --git a/tests/mock_server.py b/tests/mock_server.py index 135cb64..1371b28 100644 --- a/tests/mock_server.py +++ b/tests/mock_server.py @@ -13,8 +13,8 @@ import socketserver import logging import hashlib import urllib.parse -import traceback from xml.sax.saxutils import escape as xml_escape +import json log = logging.getLogger(__name__) @@ -48,20 +48,6 @@ class StorageServer(socketserver.TCPServer): class ParsedURL: __slots__ = [ 'bucket', 'key', 'params', 'fragment' ] -def parse_url(path): - p = ParsedURL() - q = urllib.parse.urlsplit(path) - - path = urllib.parse.unquote(q.path) - - assert path[0] == '/' - (p.bucket, p.key) = path[1:].split('/', maxsplit=1) - - p.params = urllib.parse.parse_qs(q.query) - p.fragment = q.fragment - - return p - class S3CRequestHandler(BaseHTTPRequestHandler): '''A request handler implementing a subset of the AWS S3 Interface @@ -76,6 +62,20 @@ class S3CRequestHandler(BaseHTTPRequestHandler): hdr_prefix = 'X-AMZ-' xml_ns = 'http://s3.amazonaws.com/doc/2006-03-01/' + def parse_url(self, path): + p = ParsedURL() + q = urllib.parse.urlsplit(path) + + path = urllib.parse.unquote(q.path) + + assert path[0] == '/' + (p.bucket, p.key) = path[1:].split('/', maxsplit=1) + + p.params = urllib.parse.parse_qs(q.query) + p.fragment = q.fragment + + return p + def log_message(self, format, *args): log.debug(format, *args) @@ -93,7 +93,7 @@ class S3CRequestHandler(BaseHTTPRequestHandler): pass def do_DELETE(self): - q = parse_url(self.path) + q = self.parse_url(self.path) try: del self.server.data[q.key] del self.server.metadata[q.key] @@ -117,14 +117,18 @@ class S3CRequestHandler(BaseHTTPRequestHandler): return int(self.headers['Content-Length']) - def do_PUT(self): - len_ = self._check_encoding() - q = parse_url(self.path) + def _get_meta(self): meta = dict() for (name, value) in self.headers.items(): hit = self.meta_header_re.search(name) if hit: meta[hit.group(1)] = value + return meta + + def do_PUT(self): + len_ = self._check_encoding() + q = self.parse_url(self.path) + meta = self._get_meta() src = self.headers.get(self.hdr_prefix + 'copy-source') if src and len_: @@ -189,7 +193,7 @@ class S3CRequestHandler(BaseHTTPRequestHandler): return True def do_GET(self): - q = parse_url(self.path) + q = self.parse_url(self.path) if not q.key: return self.do_list(q) @@ -246,7 +250,7 @@ class S3CRequestHandler(BaseHTTPRequestHandler): self.wfile.write(body) def do_HEAD(self): - q = parse_url(self.path) + q = self.parse_url(self.path) try: meta = self.server.metadata[q.key] data = self.server.data[q.key] @@ -298,9 +302,283 @@ class GSRequestHandler(S3CRequestHandler): hdr_prefix = 'x-goog-' xml_ns = 'http://doc.s3.amazonaws.com/2006-03-01' +class BasicSwiftRequestHandler(S3CRequestHandler): + '''A request handler implementing a subset of the OpenStack Swift Interface + + Container and AUTH_* prefix are ignored, all keys share the same global + namespace. + + To keep it simple, this handler is both storage server and authentication + server in one. + ''' + + meta_header_re = re.compile(r'X-Object-Meta-([a-z0-9_.-]+)$', + re.IGNORECASE) + hdr_prefix = 'X-Object-' + + SWIFT_INFO = { + "swift": { + "max_meta_count": 90, + "max_meta_value_length": 256, + "container_listing_limit": 10000, + "extra_header_count": 0, + "max_meta_overall_size": 4096, + "version": "2.0.0", # < 2.8 + "max_meta_name_length": 128, + "max_header_size": 16384 + } + } + + def parse_url(self, path): + p = ParsedURL() + q = urllib.parse.urlsplit(path) + + path = urllib.parse.unquote(q.path) + + assert path[0:4] == '/v1/' + (_, p.bucket, p.key) = path[4:].split('/', maxsplit=2) + + p.params = urllib.parse.parse_qs(q.query, True) + p.fragment = q.fragment + + return p + + def do_PUT(self): + len_ = self._check_encoding() + q = self.parse_url(self.path) + meta = self._get_meta() + + src = self.headers.get('x-copy-from') + if src and len_: + self.send_error(400, message='Upload and copy are mutually exclusive', + code='UnexpectedContent') + return + elif src: + src = urllib.parse.unquote(src) + hit = re.match('^/([a-z0-9._-]+)/(.+)$', src) + if not hit: + self.send_error(400, message='Cannot parse x-copy-from', + code='InvalidArgument') + return + + src = hit.group(2) + try: + data = self.server.data[src] + self.server.data[q.key] = data + if 'x-fresh-metadata' in self.headers: + self.server.metadata[q.key] = meta + else: + self.server.metadata[q.key] = self.server.metadata[src].copy() + self.server.metadata[q.key].update(meta) + except KeyError: + self.send_error(404, code='NoSuchKey', resource=src) + return + else: + data = self.rfile.read(len_) + self.server.metadata[q.key] = meta + self.server.data[q.key] = data + + md5 = hashlib.md5() + md5.update(data) + + if src: + self.send_response(202) + self.send_header('X-Copied-From', self.headers['x-copy-from']) + self.send_header('Content-Length', '0') + self.end_headers() + else: + self.send_response(201) + self.send_header('ETag', '"%s"' % md5.hexdigest()) + self.send_header('Content-Length', '0') + self.end_headers() + + def do_POST(self): + q = self.parse_url(self.path) + meta = self._get_meta() + + if q.key not in self.server.metadata: + self.send_error(404, code='NoSuchKey', resource=q.key) + return + + self.server.metadata[q.key] = meta + + self.send_response(204) + self.send_header('Content-Length', '0') + self.end_headers() + + def do_GET(self): + if self.path in ('/v1.0', '/auth/v1.0'): + self.send_response(200) + self.send_header('X-Storage-Url', + 'http://%s:%d/v1/AUTH_xyz' % (self.server.hostname, self.server.port)) + self.send_header('X-Auth-Token', 'static') + self.send_header('Content-Length', '0') + self.end_headers() + elif self.path == '/info': + content = json.dumps(self.SWIFT_INFO).encode('utf-8') + self.send_response(200) + self.send_header('Content-Length', str(len(content))) + self.send_header("Content-Type", 'application/json; charset="utf-8"') + self.end_headers() + self.wfile.write(content) + else: + return super().do_GET() + + def do_list(self, q): + marker = q.params['marker'][0] if 'marker' in q.params else None + max_keys = int(q.params['limit'][0]) if 'limit' in q.params else 10000 + prefix = q.params['prefix'][0] if 'prefix' in q.params else '' + + resp = [] + + count = 0 + for key in sorted(self.server.data): + if not key.startswith(prefix): + continue + if marker and key <= marker: + continue + resp.append({'name': key}) + count += 1 + if count == max_keys: + break + + body = json.dumps(resp).encode('utf-8') + + self.send_response(200) + self.send_header("Content-Type", 'application/json; charset="utf-8"') + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + +class CopySwiftRequestHandler(BasicSwiftRequestHandler): + '''OpenStack Swift handler that emulates Copy middleware.''' + + SWIFT_INFO = { + "swift": { + "max_meta_count": 90, + "max_meta_value_length": 256, + "container_listing_limit": 10000, + "extra_header_count": 0, + "max_meta_overall_size": 4096, + "version": "2.9.0", # >= 2.8 + "max_meta_name_length": 128, + "max_header_size": 16384 + } + } + + def do_COPY(self): + src = self.parse_url(self.path) + meta = self._get_meta() + + try: + dst = self.headers['destination'] + assert dst[0] == '/' + (_, dst) = dst[1:].split('/', maxsplit=1) + except KeyError: + self.send_error(400, message='No Destination provided', + code='InvalidArgument') + return + + if src.key not in self.server.metadata: + self.send_error(404, code='NoSuchKey', resource=src) + return + + if 'x-fresh-metadata' in self.headers: + self.server.metadata[dst] = meta + else: + self.server.metadata[dst] = self.server.metadata[src.key].copy() + self.server.metadata[dst].update(meta) + + self.server.data[dst] = self.server.data[src.key] + + self.send_response(202) + self.send_header('X-Copied-From', '%s/%s' % (src.bucket, src.key)) + self.send_header('Content-Length', '0') + self.end_headers() + +class BulkDeleteSwiftRequestHandler(BasicSwiftRequestHandler): + '''OpenStack Swift handler that emulates bulk middleware (the delete part).''' + + MAX_DELETES = 8 # test deletes 16 objects, so needs two requests + SWIFT_INFO = { + "bulk_delete": { + "max_failed_deletes": MAX_DELETES, + "max_deletes_per_request": MAX_DELETES + }, + "swift": { + "max_meta_count": 90, + "max_meta_value_length": 256, + "container_listing_limit": 10000, + "extra_header_count": 0, + "max_meta_overall_size": 4096, + "version": "2.0.0", # < 2.8 + "max_meta_name_length": 128, + "max_header_size": 16384 + } + } + + def do_POST(self): + q = self.parse_url(self.path) + if not 'bulk-delete' in q.params: + return super().do_POST() + + response = { 'Response Status': '200 OK', + 'Response Body': '', + 'Number Deleted': 0, + 'Number Not Found': 0, + 'Errors': [] } + + def send_response(status_int): + content = json.dumps(response).encode('utf-8') + self.send_response(status_int) + self.send_header('Content-Length', str(len(content))) + self.send_header("Content-Type", 'application/json; charset="utf-8"') + self.end_headers() + self.wfile.write(content) + + def error(reason): + response['Response Status'] = '502 Internal Server Error' + response['Response Body'] = reason + send_response(502) + + def inline_error(http_status, body): + '''bail out when processing begun. Always HTTP 200 Ok.''' + response['Response Status'] = http_status + response['Response Body'] = body + send_response(200) + + len_ = self._check_encoding() + lines = self.rfile.read(len_).decode('utf-8').split("\n") + for index, to_delete in enumerate(lines): + if index >= self.MAX_DELETES: + return inline_error('413 Request entity too large', + 'Maximum Bulk Deletes: %d per request' % + self.MAX_DELETES) + to_delete = urllib.parse.unquote(to_delete.strip()) + assert to_delete[0] == '/' + to_delete = to_delete[1:].split('/', maxsplit=1) + if len(to_delete) < 2: + return error("deleting containers is not supported") + to_delete = to_delete[1] + try: + del self.server.data[to_delete] + del self.server.metadata[to_delete] + except KeyError: + response['Number Not Found'] += 1 + else: + response['Number Deleted'] += 1 + + if not (response['Number Deleted'] or + response['Number Not Found']): + return inline_error('400 Bad Request', 'Invalid bulk delete.') + send_response(200) + #: A list of the available mock request handlers with #: corresponding storage urls handler_list = [ (S3CRequestHandler, 's3c://%(host)s:%(port)d/s3ql_test'), # Special syntax only for testing against mock server - (GSRequestHandler, 'gs://!unittest!%(host)s:%(port)d/s3ql_test') ] + (GSRequestHandler, 'gs://!unittest!%(host)s:%(port)d/s3ql_test'), + (BasicSwiftRequestHandler, 'swift://%(host)s:%(port)d/s3ql_test'), + (CopySwiftRequestHandler, 'swift://%(host)s:%(port)d/s3ql_test'), + (BulkDeleteSwiftRequestHandler, 'swift://%(host)s:%(port)d/s3ql_test') ] diff --git a/tests/t1_backends.py b/tests/t1_backends.py index 290446b..3b5e583 100755 --- a/tests/t1_backends.py +++ b/tests/t1_backends.py @@ -21,6 +21,7 @@ from s3ql.backends.gs import Backend as GSBackend from s3ql.backends.common import (NoSuchObject, CorruptedObjectError) from s3ql.backends.comprenc import ComprencBackend, ObjectNotEncrypted from s3ql.backends.s3c import BadDigestError, OperationAbortedError, HTTPError, S3Error +from s3ql.backends.swift import Backend as SwiftBackend from argparse import Namespace from common import get_remote_test_info, NoTestSection, CLOCK_GRANULARITY from pytest_checklogs import assert_logs @@ -39,7 +40,7 @@ log = logging.getLogger(__name__) empty_set = set() def brace_expand(s): - hit = re.search('^(.*)\{(.+)\}(.*)$', s) + hit = re.search(r'^(.*)\{(.+)\}(.*)$', s) if not hit: return [s] (p, e, s) = hit.groups() @@ -473,7 +474,7 @@ def test_delete_multi(backend): fetch_object(backend, key) # Delete half of them - # We don't use force=True but catch the exemption to increase the + # We don't use force=True but catch the exception to increase the # chance that some existing objects are not deleted because of the # error. to_delete = keys[::2] @@ -485,7 +486,10 @@ def test_delete_multi(backend): # Without full consistency, deleting an non-existing object # may not give an error - assert backend.unittest_info.retry_time or len(to_delete) > 0 + # Swift backend does not return a list of actually deleted objects + # so to_delete wil always be empty for Swift and this assertion fails + if not isinstance(backend.backend, SwiftBackend): + assert backend.unittest_info.retry_time or len(to_delete) > 0 deleted = set(keys[::2]) - set(to_delete) assert len(deleted) > 0 diff --git a/tests/t2_block_cache.py b/tests/t2_block_cache.py index d521725..87224c7 100755 --- a/tests/t2_block_cache.py +++ b/tests/t2_block_cache.py @@ -147,7 +147,7 @@ class cache_tests(unittest.TestCase): # Create first object (we'll try to remove that) with self.cache.get(self.inode, 0) as fh: fh.write(b'bar wurfz!') - self.cache.commit() + self.cache.start_flush() self.cache.wait() # Make sure that upload and removal will fail @@ -164,7 +164,7 @@ class cache_tests(unittest.TestCase): try: # Try to clean-up (implicitly calls expire) with llfuse.lock_released, \ - assert_logs('Unable to flush cache, no upload threads left alive', + assert_logs('Unable to drop cache, no upload threads left alive', level=logging.ERROR, count=1): with pytest.raises(OSError) as exc_info: self.cache.destroy() @@ -203,7 +203,7 @@ class cache_tests(unittest.TestCase): self.assertEqual(data, fh.read(len(data))) # Case 3: Object needs to be downloaded - self.cache.clear() + self.cache.drop() with self.cache.get(inode, blockno) as fh: fh.seek(0) self.assertEqual(data, fh.read(len(data))) @@ -226,8 +226,8 @@ class cache_tests(unittest.TestCase): fh.write(('%d' % i).encode()) # Flush the 2 most recently accessed ones - commit(self.cache, inode, most_recent[-2]) - commit(self.cache, inode, most_recent[-3]) + start_flush(self.cache, inode, most_recent[-2]) + start_flush(self.cache, inode, most_recent[-3]) # We want to expire 4 entries, 2 of which are already flushed self.cache.cache.max_entries = 16 @@ -310,7 +310,7 @@ class cache_tests(unittest.TestCase): fh.seek(0) fh.write(data1) self.cache.upload(el1) - self.cache.clear() + self.cache.drop() self.cache.backend_pool.verify() def test_remove_referenced(self): @@ -327,7 +327,7 @@ class cache_tests(unittest.TestCase): with self.cache.get(inode, blockno2) as fh: fh.seek(0) fh.write(data) - self.cache.clear() + self.cache.drop() self.cache.backend_pool.verify() self.cache.backend_pool = MockBackendPool(self.backend_pool) @@ -451,7 +451,7 @@ class cache_tests(unittest.TestCase): fh.seek(0) fh.write(data1) self.cache.backend_pool = MockBackendPool(self.backend_pool, no_write=1) - commit(self.cache, inode) + start_flush(self.cache, inode) self.cache.backend_pool.verify() self.cache.backend_pool = MockBackendPool(self.backend_pool, no_del=1) self.cache.remove(inode, 1) @@ -471,7 +471,7 @@ class cache_tests(unittest.TestCase): fh.seek(0) fh.write(data1) self.cache.backend_pool = MockBackendPool(self.backend_pool, no_write=1) - self.cache.clear() + self.cache.drop() self.cache.backend_pool.verify() self.cache.backend_pool = MockBackendPool(self.backend_pool, no_del=1) self.cache.remove(inode, 1) @@ -565,7 +565,7 @@ class MockBackendPool(AbstractBackend): return self.backend.get_size(key) -def commit(cache, inode, block=None): +def start_flush(cache, inode, block=None): """Upload data for `inode` This is only for testing purposes, since the method blocks until all current diff --git a/tests/t3_fs_api.py b/tests/t3_fs_api.py index 1218d82..5e401f7 100755 --- a/tests/t3_fs_api.py +++ b/tests/t3_fs_api.py @@ -119,7 +119,7 @@ class fs_api_tests(unittest.TestCase): return fd.read(len_) def fsck(self): - self.block_cache.clear() + self.block_cache.drop() self.server.inodes.flush() fsck = Fsck(self.cachedir + '/cache', self.backend, { 'max_obj_size': self.max_obj_size }, self.db) @@ -715,7 +715,7 @@ class fs_api_tests(unittest.TestCase): (fh, inode) = self.server.create(ROOT_INODE, self.newname(), self.file_mode(), os.O_RDWR, some_ctx) self.server.write(fh, 0, data) - self.server.cache.clear() + self.server.cache.drop() self.assertTrue(self.server.failsafe is False) datafile = os.path.join(self.backend_dir, 's3ql_data_', 's3ql_data_1') @@ -754,7 +754,7 @@ class fs_api_tests(unittest.TestCase): # Remove completely, should give error after cache flush os.unlink(datafile) self.server.read(fh, 3, len_//2) - self.server.cache.clear() + self.server.cache.drop() with self.assertRaises(FUSEError) as cm: with assert_logs('^Backend lost block', count=1, level=logging.ERROR): @@ -788,7 +788,7 @@ class fs_api_tests(unittest.TestCase): self.server.write(fh, 0, data) self.server.release(fh) - self.block_cache.clear() + self.block_cache.drop() fh = self.server.open(inode.st_ino, os.O_RDWR, some_ctx) attr = self.server.getattr(inode.st_ino, some_ctx) |