diff options
47 files changed, 413 insertions, 538 deletions
diff --git a/Changes.txt b/Changes.txt index 1909c68..92a8182 100644 --- a/Changes.txt +++ b/Changes.txt @@ -1,3 +1,14 @@ +UNRELEASED, S3QL tip + + * The `runtests.py` file has been dropped in favor of requiring + installation of py.test. + + * S3QL now uses semantic versioning. This means that + backwards-incompatible versions (i.e., versions that require an + upgrade of the file system revision) will be reflected in an + increase of the major version number, i.e. the next + backwards-incompatible version will have version 3.0. + 2016-03-08, S3QL 2.17.1 * Fixed a bug in the upgrade procedure that prevented file system @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: s3ql -Version: 2.17.1 +Version: 2.17.1-hg2 Summary: a full-featured file system for online data storage Home-page: https://bitbucket.org/nikratio/s3ql/ Author: Nikolaus Rath @@ -92,13 +92,12 @@ Description: .. Development Status ================== - S3QL is considered stable and suitable for production use. However, - upgrades from one minor version to the next (e.g. *2.x* to *2.x+1*) - may change the public interface (e.g. different command line options), - or require the file system structure to be upgraded (so that the file - system can no longer be accessed by older releases). Therefore, it is - strongly recommended to read the changelog (`Changes.txt` in the S3QL - tarball) before upgrading. + S3QL is considered stable and suitable for production use. Starting + with version 2.17.1, S3QL uses semantic versioning. This means that + backwards-incompatible versions (e.g., versions that require an + upgrade of the file system revision) will be reflected in an increase + of the major version number. + Supported Platforms =================== @@ -111,22 +110,6 @@ Description: .. try to fix them. - Which Version Should I Download? - ================================ - - Short answer: if your system supports Python 3.3 or newer, download - the most recent *2.x* version. - - Long answer: there are two supported branches of S3QL. Both branches - are suitable for production use. The *maint-1.x* branch (version - numbers *1.x*) is no longer actively developed and receives only - selected high-impact bugfixes. It is provided for systems without - Python 3 support. For systems with Python 3.3 or newer, it is - recommended run the *default* S3QL branch (with version numbers - *2.x*). This branch is actively developed and has a number of new - features that are not available in the *1.x* versions. - - Typical Usage ============= @@ -83,13 +83,12 @@ Features Development Status ================== -S3QL is considered stable and suitable for production use. However, -upgrades from one minor version to the next (e.g. *2.x* to *2.x+1*) -may change the public interface (e.g. different command line options), -or require the file system structure to be upgraded (so that the file -system can no longer be accessed by older releases). Therefore, it is -strongly recommended to read the changelog (`Changes.txt` in the S3QL -tarball) before upgrading. +S3QL is considered stable and suitable for production use. Starting +with version 2.17.1, S3QL uses semantic versioning. This means that +backwards-incompatible versions (e.g., versions that require an +upgrade of the file system revision) will be reflected in an increase +of the major version number. + Supported Platforms =================== @@ -102,22 +101,6 @@ on all non-Linux systems. Please report any bugs you find, and we will try to fix them. -Which Version Should I Download? -================================ - -Short answer: if your system supports Python 3.3 or newer, download -the most recent *2.x* version. - -Long answer: there are two supported branches of S3QL. Both branches -are suitable for production use. The *maint-1.x* branch (version -numbers *1.x*) is no longer actively developed and receives only -selected high-impact bugfixes. It is provided for systems without -Python 3 support. For systems with Python 3.3 or newer, it is -recommended run the *default* S3QL branch (with version numbers -*2.x*). This branch is actively developed and has a number of new -features that are not available in the *1.x* versions. - - Typical Usage ============= diff --git a/contrib/expire_backups.1 b/contrib/expire_backups.1 index 1ba0f6c..eaaeab0 100644 --- a/contrib/expire_backups.1 +++ b/contrib/expire_backups.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "EXPIRE_BACKUPS" "1" "March 09, 2016" "2.17.1" "S3QL" +.TH "EXPIRE_BACKUPS" "1" "March 10, 2016" "2.17.1" "S3QL" .SH NAME expire_backups \- Intelligently expire old backups . diff --git a/contrib/expire_backups.py b/contrib/expire_backups.py index a20a0d5..34efc02 100755 --- a/contrib/expire_backups.py +++ b/contrib/expire_backups.py @@ -183,7 +183,7 @@ def process_backups(backup_list, state, cycles): # Missing backups missing_backups = set(state) - backup_list for x in missing_backups: - log.warning('Warning: backup %s is missing. Did you delete it manually?', x) + log.warning('backup %s is missing. Did you delete it manually?', x) del state[x] # Ranges diff --git a/contrib/pcp.1 b/contrib/pcp.1 index 56cbb13..abc7724 100644 --- a/contrib/pcp.1 +++ b/contrib/pcp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "PCP" "1" "March 09, 2016" "2.17.1" "S3QL" +.TH "PCP" "1" "March 10, 2016" "2.17.1" "S3QL" .SH NAME pcp \- Recursive, parallel copy of directory trees . diff --git a/doc/latex/manual.aux b/doc/latex/manual.aux index 417333c..816b015 100644 --- a/doc/latex/manual.aux +++ b/doc/latex/manual.aux @@ -41,9 +41,9 @@ \@writefile{toc}{\contentsline {chapter}{\numberline {2}Installation}{3}{chapter.2}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} +\newlabel{installation::doc}{{2}{3}{Installation}{chapter.2}{}} \newlabel{installation:installation}{{2}{3}{Installation}{chapter.2}{}} \newlabel{installation:github}{{2}{3}{Installation}{chapter.2}{}} -\newlabel{installation::doc}{{2}{3}{Installation}{chapter.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.1}Dependencies}{3}{section.2.1}} \newlabel{installation:dependencies}{{2.1}{3}{Dependencies}{section.2.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.2}Installing S3QL}{4}{section.2.2}} @@ -56,8 +56,8 @@ \@writefile{toc}{\contentsline {chapter}{\numberline {3}Storage Backends}{7}{chapter.3}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} +\newlabel{backends:sphinx}{{3}{7}{Storage Backends}{chapter.3}{}} \newlabel{backends:id1}{{3}{7}{Storage Backends}{chapter.3}{}} -\newlabel{backends:py-test}{{3}{7}{Storage Backends}{chapter.3}{}} \newlabel{backends::doc}{{3}{7}{Storage Backends}{chapter.3}{}} \newlabel{backends:storage-backends}{{3}{7}{Storage Backends}{chapter.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.1}Google Storage}{7}{section.3.1}} @@ -66,8 +66,8 @@ \newlabel{backends:cmdoption-gs_backend-arg-ssl-ca-path}{{3.1}{8}{Google Storage}{section*.4}{}} \newlabel{backends:cmdoption-gs_backend-arg-tcp-timeout}{{3.1}{8}{Google Storage}{section*.5}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.2}Amazon S3}{8}{section.3.2}} -\newlabel{backends:amazon-s3}{{3.2}{8}{Amazon S3}{section.3.2}{}} \newlabel{backends:google-storage-manager}{{3.2}{8}{Amazon S3}{section.3.2}{}} +\newlabel{backends:amazon-s3}{{3.2}{8}{Amazon S3}{section.3.2}{}} \newlabel{backends:cmdoption-s3_backend-arg-no-ssl}{{3.2}{8}{Amazon S3}{section*.6}{}} \newlabel{backends:cmdoption-s3_backend-arg-ssl-ca-path}{{3.2}{8}{Amazon S3}{section*.7}{}} \newlabel{backends:cmdoption-s3_backend-arg-tcp-timeout}{{3.2}{8}{Amazon S3}{section*.8}{}} @@ -75,15 +75,15 @@ \newlabel{backends:cmdoption-s3_backend-arg-ia}{{3.2}{8}{Amazon S3}{section*.10}{}} \newlabel{backends:cmdoption-s3_backend-arg-rrs}{{3.2}{8}{Amazon S3}{section*.11}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.3}OpenStack/Swift}{9}{section.3.3}} -\newlabel{backends:openstack-swift}{{3.3}{9}{OpenStack/Swift}{section.3.3}{}} \newlabel{backends:openstack-backend}{{3.3}{9}{OpenStack/Swift}{section.3.3}{}} +\newlabel{backends:openstack-swift}{{3.3}{9}{OpenStack/Swift}{section.3.3}{}} \newlabel{backends:cmdoption-swift_backend-arg-no-ssl}{{3.3}{9}{OpenStack/Swift}{section*.12}{}} \newlabel{backends:cmdoption-swift_backend-arg-ssl-ca-path}{{3.3}{9}{OpenStack/Swift}{section*.13}{}} \newlabel{backends:cmdoption-swift_backend-arg-tcp-timeout}{{3.3}{9}{OpenStack/Swift}{section*.14}{}} \newlabel{backends:cmdoption-swift_backend-arg-disable-expect100}{{3.3}{9}{OpenStack/Swift}{section*.15}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.4}Rackspace CloudFiles}{9}{section.3.4}} -\newlabel{backends:swift}{{3.4}{9}{Rackspace CloudFiles}{section.3.4}{}} \newlabel{backends:rackspace-cloudfiles}{{3.4}{9}{Rackspace CloudFiles}{section.3.4}{}} +\newlabel{backends:swift}{{3.4}{9}{Rackspace CloudFiles}{section.3.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.5}S3 compatible}{10}{section.3.5}} \newlabel{backends:rackspace}{{3.5}{10}{S3 compatible}{section.3.5}{}} \newlabel{backends:s3-compatible}{{3.5}{10}{S3 compatible}{section.3.5}{}} @@ -93,15 +93,15 @@ \newlabel{backends:cmdoption-s3c_backend-arg-disable-expect100}{{3.5}{10}{S3 compatible}{section*.19}{}} \newlabel{backends:cmdoption-s3c_backend-arg-dumb-copy}{{3.5}{10}{S3 compatible}{section*.20}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.6}Local}{10}{section.3.6}} -\newlabel{backends:local}{{3.6}{10}{Local}{section.3.6}{}} \newlabel{backends:id6}{{3.6}{10}{Local}{section.3.6}{}} +\newlabel{backends:local}{{3.6}{10}{Local}{section.3.6}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {4}Important Rules to Avoid Losing Data}{13}{chapter.4}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{durability:durability}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} -\newlabel{durability:important-rules-to-avoid-losing-data}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} -\newlabel{durability:sshfs}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} \newlabel{durability::doc}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} +\newlabel{durability:sshfs}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} +\newlabel{durability:important-rules-to-avoid-losing-data}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.1}Rules in a Nutshell}{13}{section.4.1}} \newlabel{durability:rules-in-a-nutshell}{{4.1}{13}{Rules in a Nutshell}{section.4.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.2}Consistency Window List}{14}{section.4.2}} @@ -109,18 +109,18 @@ \@writefile{toc}{\contentsline {section}{\numberline {4.3}Data Consistency}{14}{section.4.3}} \newlabel{durability:data-consistency}{{4.3}{14}{Data Consistency}{section.4.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.4}Data Durability}{15}{section.4.4}} -\newlabel{durability:data-durability}{{4.4}{15}{Data Durability}{section.4.4}{}} \newlabel{durability:backend-reliability}{{4.4}{15}{Data Durability}{section.4.4}{}} +\newlabel{durability:data-durability}{{4.4}{15}{Data Durability}{section.4.4}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {5}File System Creation}{17}{chapter.5}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{mkfs::doc}{{5}{17}{File System Creation}{chapter.5}{}} \newlabel{mkfs:file-system-creation}{{5}{17}{File System Creation}{chapter.5}{}} +\newlabel{mkfs::doc}{{5}{17}{File System Creation}{chapter.5}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {6}Managing File Systems}{19}{chapter.6}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{adm:managing-file-systems}{{6}{19}{Managing File Systems}{chapter.6}{}} \newlabel{adm::doc}{{6}{19}{Managing File Systems}{chapter.6}{}} +\newlabel{adm:managing-file-systems}{{6}{19}{Managing File Systems}{chapter.6}{}} \@writefile{toc}{\contentsline {section}{\numberline {6.1}Changing the Passphrase}{19}{section.6.1}} \newlabel{adm:changing-the-passphrase}{{6.1}{19}{Changing the Passphrase}{section.6.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {6.2}Upgrading the file system}{20}{section.6.2}} @@ -155,8 +155,8 @@ \newlabel{special:advanced-s3ql-features}{{8}{25}{Advanced S3QL Features}{chapter.8}{}} \newlabel{special::doc}{{8}{25}{Advanced S3QL Features}{chapter.8}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.1}Snapshotting and Copy-on-Write}{25}{section.8.1}} -\newlabel{special:snapshotting-and-copy-on-write}{{8.1}{25}{Snapshotting and Copy-on-Write}{section.8.1}{}} \newlabel{special:s3qlcp}{{8.1}{25}{Snapshotting and Copy-on-Write}{section.8.1}{}} +\newlabel{special:snapshotting-and-copy-on-write}{{8.1}{25}{Snapshotting and Copy-on-Write}{section.8.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.1.1}Snapshotting vs Hardlinking}{25}{subsection.8.1.1}} \newlabel{special:snapshotting-vs-hardlinking}{{8.1.1}{25}{Snapshotting vs Hardlinking}{subsection.8.1.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.2}Getting Statistics}{26}{section.8.2}} @@ -174,13 +174,13 @@ \@writefile{toc}{\contentsline {chapter}{\numberline {9}Unmounting}{29}{chapter.9}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{umount:unmounting}{{9}{29}{Unmounting}{chapter.9}{}} \newlabel{umount::doc}{{9}{29}{Unmounting}{chapter.9}{}} +\newlabel{umount:unmounting}{{9}{29}{Unmounting}{chapter.9}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {10}Checking for Errors}{31}{chapter.10}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{fsck:checking-for-errors}{{10}{31}{Checking for Errors}{chapter.10}{}} \newlabel{fsck::doc}{{10}{31}{Checking for Errors}{chapter.10}{}} +\newlabel{fsck:checking-for-errors}{{10}{31}{Checking for Errors}{chapter.10}{}} \@writefile{toc}{\contentsline {section}{\numberline {10.1}Checking and repairing internal file system errors}{31}{section.10.1}} \newlabel{fsck:checking-and-repairing-internal-file-system-errors}{{10.1}{31}{Checking and repairing internal file system errors}{section.10.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {10.2}Detecting and handling backend data corruption}{32}{section.10.2}} @@ -189,21 +189,21 @@ \@writefile{toc}{\contentsline {chapter}{\numberline {11}Storing Authentication Information}{33}{chapter.11}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{authinfo:storing-authentication-information}{{11}{33}{Storing Authentication Information}{chapter.11}{}} \newlabel{authinfo:authinfo}{{11}{33}{Storing Authentication Information}{chapter.11}{}} \newlabel{authinfo::doc}{{11}{33}{Storing Authentication Information}{chapter.11}{}} +\newlabel{authinfo:storing-authentication-information}{{11}{33}{Storing Authentication Information}{chapter.11}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {12}Contributed Programs}{35}{chapter.12}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{contrib:contributed-programs}{{12}{35}{Contributed Programs}{chapter.12}{}} \newlabel{contrib::doc}{{12}{35}{Contributed Programs}{chapter.12}{}} +\newlabel{contrib:contributed-programs}{{12}{35}{Contributed Programs}{chapter.12}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.1}benchmark.py}{35}{section.12.1}} \newlabel{contrib:benchmark-py}{{12.1}{35}{benchmark.py}{section.12.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.2}clone\_fs.py}{35}{section.12.2}} \newlabel{contrib:clone-fs-py}{{12.2}{35}{clone\_fs.py}{section.12.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.3}pcp.py}{35}{section.12.3}} -\newlabel{contrib:pcp-py}{{12.3}{35}{pcp.py}{section.12.3}{}} \newlabel{contrib:pcp}{{12.3}{35}{pcp.py}{section.12.3}{}} +\newlabel{contrib:pcp-py}{{12.3}{35}{pcp.py}{section.12.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.4}s3ql\_backup.sh}{35}{section.12.4}} \newlabel{contrib:s3ql-backup-sh}{{12.4}{35}{s3ql\_backup.sh}{section.12.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.5}expire\_backups.py}{36}{section.12.5}} @@ -222,8 +222,8 @@ \@writefile{toc}{\contentsline {section}{\numberline {13.2}Permanently mounted backup file system}{39}{section.13.2}} \newlabel{tips:permanently-mounted-backup-file-system}{{13.2}{39}{Permanently mounted backup file system}{section.13.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {13.3}Improving copy performance}{39}{section.13.3}} -\newlabel{tips:improving-copy-performance}{{13.3}{39}{Improving copy performance}{section.13.3}{}} \newlabel{tips:copy-performance}{{13.3}{39}{Improving copy performance}{section.13.3}{}} +\newlabel{tips:improving-copy-performance}{{13.3}{39}{Improving copy performance}{section.13.3}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {14}Known Issues}{41}{chapter.14}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} @@ -372,8 +372,8 @@ \newlabel{man/fsck:see-also}{{15.10.5}{57}{See Also}{subsection.15.10.5}{}} \@writefile{toc}{\contentsline {section}{\numberline {15.11}The \textbf {s3ql\_oauth\_client} command}{57}{section.15.11}} \newlabel{man/oauth_client:the-command-command}{{15.11}{57}{The \textbf {s3ql\_oauth\_client} command}{section.15.11}{}} -\newlabel{man/oauth_client:oauth-client}{{15.11}{57}{The \textbf {s3ql\_oauth\_client} command}{section.15.11}{}} \newlabel{man/oauth_client::doc}{{15.11}{57}{The \textbf {s3ql\_oauth\_client} command}{section.15.11}{}} +\newlabel{man/oauth_client:oauth-client}{{15.11}{57}{The \textbf {s3ql\_oauth\_client} command}{section.15.11}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.11.1}Synopsis}{57}{subsection.15.11.1}} \newlabel{man/oauth_client:synopsis}{{15.11.1}{57}{Synopsis}{subsection.15.11.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.11.2}Description}{57}{subsection.15.11.2}} @@ -433,8 +433,8 @@ \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} \newlabel{impl_details:impl-details}{{17}{65}{Implementation Details}{chapter.17}{}} -\newlabel{impl_details:implementation-details}{{17}{65}{Implementation Details}{chapter.17}{}} \newlabel{impl_details::doc}{{17}{65}{Implementation Details}{chapter.17}{}} +\newlabel{impl_details:implementation-details}{{17}{65}{Implementation Details}{chapter.17}{}} \@writefile{toc}{\contentsline {section}{\numberline {17.1}Metadata Storage}{65}{section.17.1}} \newlabel{impl_details:metadata-storage}{{17.1}{65}{Metadata Storage}{section.17.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {17.2}Data Storage}{65}{section.17.2}} diff --git a/doc/latex/manual.tex b/doc/latex/manual.tex index f432edb..719d0f1 100644 --- a/doc/latex/manual.tex +++ b/doc/latex/manual.tex @@ -14,7 +14,7 @@ \title{S3QL Documentation} -\date{March 09, 2016} +\date{March 10, 2016} \release{2.17.1} \author{Nikolaus Rath} \newcommand{\sphinxlogo}{} @@ -32,67 +32,67 @@ \PYG@it{\PYG@bf{\PYG@ff{#1}}}}}}} \def\PYG#1#2{\PYG@reset\PYG@toks#1+\relax+\PYG@do{#2}} -\expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@ne\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@sh\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@mf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@kr\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.20}{##1}}} +\expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}} +\expandafter\def\csname PYG@tok@gu\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}} +\expandafter\def\csname PYG@tok@sd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.27,0.13}{##1}}} +\expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} +\expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.73}{##1}}} \expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.73,0.00,0.40}{##1}}} -\expandafter\def\csname PYG@tok@si\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.93,0.93,0.93}{\strut ##1}}} -\expandafter\def\csname PYG@tok@sr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,1.00}{\strut ##1}}} +\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.67,0.40,0.00}{##1}}} +\expandafter\def\csname PYG@tok@mf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@nt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.33,0.53}{##1}}} +\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} +\expandafter\def\csname PYG@tok@il\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} +\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.33,0.33,0.33}{##1}}} +\expandafter\def\csname PYG@tok@sc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} +\expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}} \expandafter\def\csname PYG@tok@vc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.40,0.60}{##1}}} -\expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} -\expandafter\def\csname PYG@tok@c\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@no\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.40}{##1}}} \expandafter\def\csname PYG@tok@se\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@kp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.53}{##1}}} -\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@s\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.67,0.40,0.00}{##1}}} -\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@sh\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} \expandafter\def\csname PYG@tok@gh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}} -\expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} -\expandafter\def\csname PYG@tok@sc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} -\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}} \expandafter\def\csname PYG@tok@gp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}} -\expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@nl\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.60,0.47,0.00}{##1}}} -\expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}} -\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}} -\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.33,0.53}{##1}}} -\expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}} -\expandafter\def\csname PYG@tok@kc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} \expandafter\def\csname PYG@tok@mo\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.27,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@il\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} -\expandafter\def\csname PYG@tok@cs\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.80,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.80}{##1}}} -\expandafter\def\csname PYG@tok@mb\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@si\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.93,0.93,0.93}{\strut ##1}}} \expandafter\def\csname PYG@tok@err\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.67,0.67}{\strut ##1}}} -\expandafter\def\csname PYG@tok@nt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.47,0.00}{##1}}} \expandafter\def\csname PYG@tok@mi\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} -\expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.33,0.33,0.33}{##1}}} -\expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit} -\expandafter\def\csname PYG@tok@sd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.27,0.13}{##1}}} +\expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.20}{##1}}} +\expandafter\def\csname PYG@tok@kr\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@ne\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@kc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} +\expandafter\def\csname PYG@tok@vg\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.87,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@sr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,1.00}{\strut ##1}}} +\expandafter\def\csname PYG@tok@kp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.53}{##1}}} +\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} +\expandafter\def\csname PYG@tok@mb\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}} +\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@s\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.80}{##1}}} \expandafter\def\csname PYG@tok@bp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} -\expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.73}{##1}}} -\expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.33,0.47,0.60}{##1}}} \expandafter\def\csname PYG@tok@ow\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@gu\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}} +\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}} \expandafter\def\csname PYG@tok@c1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@nl\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.60,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} \expandafter\def\csname PYG@tok@gs\endcsname{\let\PYG@bf=\textbf} -\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} -\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} +\expandafter\def\csname PYG@tok@c\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit} +\expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.33,0.47,0.60}{##1}}} +\expandafter\def\csname PYG@tok@cs\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.80,0.00,0.00}{##1}}} \expandafter\def\csname PYG@tok@kt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.60}{##1}}} -\expandafter\def\csname PYG@tok@vg\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.87,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@no\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.40}{##1}}} \def\PYGZbs{\char`\\} \def\PYGZus{\char`\_} @@ -249,7 +249,7 @@ The S3QL source code is available both on \href{https://github.com/s3ql/main}{Gi \chapter{Installation} -\label{installation:installation}\label{installation:github}\label{installation::doc} +\label{installation::doc}\label{installation:installation}\label{installation:github} S3QL depends on several other programs and libraries that have to be installed first. The best method to satisfy these dependencies depends on your distribution. In some cases S3QL and all its dependencies can @@ -290,73 +290,66 @@ possible}. The \href{http://psmisc.sf.net/}{psmisc} utilities. \item {} +\href{http://www.sqlite.org/}{SQLite} version 3.7.0 or newer. SQLite +has to be installed as a \emph{shared library} with development headers. + +\item {} \href{http://www.python.org/}{Python} 3.3.0 or newer. Make sure to also install the development headers. \item {} -The \href{https://pypi.python.org/pypi/setuptools}{setuptools Python Module}, version 1.0 or newer. -To check if with version (if any) of this module is installed, try -to execute - -\begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{python3 \PYGZhy{}c \PYGZsq{}import setuptools; print(setuptools.\PYGZus{}\PYGZus{}version\PYGZus{}\PYGZus{})\PYGZsq{}} -\end{Verbatim} - +The following Python modules: +\begin{itemize} \item {} -The \href{https://www.dlitz.net/software/pycrypto/}{PyCrypto Python Module}. To check if this -module is installed, try to execute \code{python3 -c 'import Crypto'}. +\href{https://pypi.python.org/pypi/setuptools}{setuptools}, version 1.0 or newer. \item {} -The \href{https://pypi.python.org/pypi/defusedxml/}{Python defusedxml module}. To check if this -module is installed, try to execute \code{python3 -c 'import defusedxml'}. +\href{https://www.dlitz.net/software/pycrypto/}{pycrypto} \item {} -If you want to use OAuth2 authentication with Google Storage, you -need the \href{https://pypi.python.org/pypi/requests/}{Python requests module}. To check if this module -is installed, try to execute \code{python3 -c 'import requests'}. +\href{https://pypi.python.org/pypi/defusedxml/}{defusedxml} \item {} -If you want to use systemd integration, you need the \href{https://github.com/systemd/python-systemd}{Python systemd -module}. To check if -this module is installed, try to execute \code{python3 -c 'import systemd'}. +\href{https://pypi.python.org/pypi/requests/}{requests} (optional, +required for OAuth2 authentication with Google Storage) \item {} -\href{http://www.sqlite.org/}{SQLite} version 3.7.0 or newer. SQLite -has to be installed as a \emph{shared library} with development headers. +\href{https://github.com/systemd/python-systemd}{systemd} (optional, +for enabling systemd support). \item {} -The \href{http://code.google.com/p/apsw/}{APSW Python Module}. To check -which (if any) version of APWS is installed, run the command - -\begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{python3 \PYGZhy{}c \PYGZsq{}import apsw; print(apsw.apswversion())\PYGZsq{}} -\end{Verbatim} +\href{https://github.com/rogerbinns/apsw}{apsw}, version 3.7.0 or +newer. -The printed version number should be at least 3.7.0. +\item {} +\href{https://bitbucket.org/nikratio/python-llfuse/}{llfuse}, any +version between 1.0 (inclusive) and 2.0 (exclusive) \item {} -The \href{https://pypi.python.org/pypi/llfuse/}{Python LLFUSE module}. To check which (if any) -version of if this module is installed, execute \code{python3 -c 'import -llfuse; print(llfuse.\_\_version\_\_)'}. Any version between 1.0 -(inclusive) and 2.0 (exclusive) will do. +\href{https://bitbucket.org/nikratio/python-dugong/}{dugong}, any +version between 3.4 (inclusive) and 4.0 (exclusive) \item {} -The \href{https://bitbucket.org/nikratio/python-dugong/}{Python dugong module}. To check if this -module is installed, try to execute \code{python3 -c 'import dugong; -print(dugong.\_\_version\_\_)'}. This should print a version number. You -need at least version 3.4. +\href{http://pytest.org/}{pytest}, version 2.3.3 or newer (optional, to run unit tests) \item {} -If your system is using \href{http://www.freedesktop.org/wiki/Software/systemd/}{systemd}, make sure that the \code{systemd} -Python module is available for Python 3. To check this, try to run -\code{python3 -c 'import systemd.daemon'}. +\href{https://github.com/eisensheng/pytest-catchlog}{pytest-catchlog} +(optional, to run unit tests) + +\end{itemize} + +To check if a specific module \code{\textless{}module\textgreater{}} is installed, execute +\code{python3 -c 'import \emph{\textless{}module\textgreater{}}; +print(\emph{\textless{}module\textgreater{}}.\_\_version\_\_)'}. This will result in an +\code{ImportError} if the module is not installed, and will print the +installed version if the module is installed. \end{itemize} \section{Installing S3QL} \label{installation:inst-s3ql}\label{installation:installing-s3ql} -To install S3QL itself, proceed as follows: +To build and install S3QL itself, proceed as follows: \begin{enumerate} \item {} Download S3QL from \href{https://bitbucket.org/nikratio/s3ql/downloads}{https://bitbucket.org/nikratio/s3ql/downloads} @@ -368,7 +361,7 @@ Unpack it into a folder of your choice Run \code{python3 setup.py build\_ext -{-}inplace} to build S3QL. \item {} -Run \code{python3 runtests.py tests} to run a self-test. If this fails, ask +Run \code{python3 -m pytest tests/} to run a self-test. If this fails, ask for help on the \href{http://groups.google.com/group/s3ql}{mailing list} or report a bug in the \href{https://bitbucket.org/nikratio/s3ql/issues}{issue tracker}. @@ -402,9 +395,6 @@ Version 0.24 or newer of the \href{http://www.cython.org/}{Cython} compiler. \item {} Version 1.2b1 or newer of the \href{http://sphinx.pocoo.org/}{Sphinx} document processor. -\item {} -The \href{http://pytest.org/}{py.test} testing tool, version 2.3.3 or newer. - \end{itemize} With these additional dependencies installed, S3QL can be build and @@ -413,18 +403,17 @@ tested with \begin{Verbatim}[commandchars=\\\{\}] \PYG{l}{python3 setup.py build\PYGZus{}cython} \PYG{l}{python3 setup.py build\PYGZus{}ext \PYGZhy{}\PYGZhy{}inplace} -\PYG{l}{py.test tests/} +\PYG{l}{python3 \PYGZhy{}m pytest tests/} \end{Verbatim} -Note that when building from the Mercurial repository, building and -testing is done with several additional checks. This may cause +Note that when building from the Mercurial or Git repository, building +and testing is done with several additional checks. This may cause compilation and/or tests to fail even though there are no problems -with functionality. For example, when building from the Mercurial -repository, any use of functions that are scheduled for deprecation in -future Python version will cause tests to fail. If you would rather -just check for functionality, you can delete the \code{MANIFEST.in} -file. In that case, the build system will behave as it does for a -regular release. +with functionality. For example, any use of functions that are +scheduled for deprecation in future Python version will cause tests to +fail. If you would rather just check for functionality, you can delete +the \code{MANIFEST.in} file. In that case, the build system will +behave as it does for a regular release. The HTML and PDF documentation can be generated with @@ -441,12 +430,11 @@ and S3QL can be installed as usual with \section{Running tests requiring remote servers} \label{installation:running-tests-requiring-remote-servers} -By default, the \code{runtest.py} (or \code{py.test}) script skips all tests -that require connection to a remote storage backend. If you would like -to run these tests too (which is always a good idea), you have to -create additional entries in your \code{\textasciitilde{}/.s3ql/authinfo2} file that tell -S3QL what server and credentials to use for these tests. These entries -have the following form: +By default, tests requiring a connection to a remote storage backend +are skipped. If you would like to run these tests too (which is always +a good idea), you have to create additional entries in your +\code{\textasciitilde{}/.s3ql/authinfo2} file that tell S3QL what server and credentials to +use for these tests. These entries have the following form: \begin{Verbatim}[commandchars=\\\{\}] \PYG{g+ge}{[\PYGZlt{}BACKEND\PYGZgt{}\PYGZhy{}test]} @@ -480,7 +468,7 @@ being skipped by passing the \code{-rs} argument to \chapter{Storage Backends} -\label{backends:id1}\label{backends:py-test}\label{backends::doc}\label{backends:storage-backends} +\label{backends:sphinx}\label{backends:id1}\label{backends::doc}\label{backends:storage-backends} S3QL supports different \emph{backends} to store data at different service providers and using different protocols. A \emph{storage url} specifies a backend together with some backend-specific information and uniquely @@ -585,7 +573,7 @@ TCP connection is closed and re-established (default: 20 seconds). \section{Amazon S3} -\label{backends:amazon-s3}\label{backends:google-storage-manager} +\label{backends:google-storage-manager}\label{backends:amazon-s3} \href{http://aws.amazon.com/s3}{Amazon S3} is the online storage service offered by \href{http://aws.amazon.com/}{Amazon Web Services (AWS)}. To use the S3 backend, you first need to sign up for an AWS account. The @@ -680,7 +668,7 @@ even later in time due to the data de-duplication feature of S3QL (see \section{OpenStack/Swift} -\label{backends:openstack-swift}\label{backends:openstack-backend} +\label{backends:openstack-backend}\label{backends:openstack-swift} \href{http://www.openstack.org/}{OpenStack} is an open-source cloud server application suite. \href{http://openstack.org/projects/storage/}{Swift} is the cloud storage module of OpenStack. Swift/OpenStack storage is offered by many different companies. @@ -767,7 +755,7 @@ than once in some circumstances. \section{Rackspace CloudFiles} -\label{backends:swift}\label{backends:rackspace-cloudfiles} +\label{backends:rackspace-cloudfiles}\label{backends:swift} \href{http://www.rackspace.com/}{Rackspace} CloudFiles uses \href{http://www.openstack.org/}{OpenStack} internally, so it is possible to just use the OpenStack/Swift backend (see above) with \code{auth.api.rackspacecloud.com} as the host name. For convenince, @@ -880,7 +868,7 @@ body for a succesfull copy operation. \section{Local} -\label{backends:local}\label{backends:id6} +\label{backends:id6}\label{backends:local} S3QL is also able to store its data on the local file system. This can be used to backup data on external media, or to access external services that S3QL can not talk to directly (e.g., it is possible to @@ -906,7 +894,7 @@ The local backend does not accept any backend options. \chapter{Important Rules to Avoid Losing Data} -\label{durability:durability}\label{durability:important-rules-to-avoid-losing-data}\label{durability:sshfs}\label{durability::doc} +\label{durability:durability}\label{durability::doc}\label{durability:sshfs}\label{durability:important-rules-to-avoid-losing-data} Most S3QL backends store data in distributed storage systems. These systems differ from a traditional, local hard disk in several important ways. In order to avoid losing data, this section should be @@ -1097,7 +1085,7 @@ rise up to hours (\href{http://forums.aws.amazon.com/message.jspa?messageID=3847 \section{Data Durability} -\label{durability:data-durability}\label{durability:backend-reliability} +\label{durability:backend-reliability}\label{durability:data-durability} The durability of a storage service a measure of the average probability of a storage object to become corrupted over time. The lower the chance of data loss, the higher the durability. Storage @@ -1180,7 +1168,7 @@ neglected over long periods of time. \chapter{File System Creation} -\label{mkfs::doc}\label{mkfs:file-system-creation} +\label{mkfs:file-system-creation}\label{mkfs::doc} A S3QL file system is created with the \textbf{mkfs.s3ql} command. It has the following syntax: @@ -1257,7 +1245,7 @@ one at \code{s3://bucketname/outerprefix/innerprefix}. \chapter{Managing File Systems} -\label{adm:managing-file-systems}\label{adm::doc} +\label{adm::doc}\label{adm:managing-file-systems} The \code{s3qladm} command performs various operations on \emph{unmounted} S3QL file systems. The file system \emph{must not be mounted} when using \code{s3qladm} or things will go wrong badly. @@ -1623,7 +1611,7 @@ mounted. \label{special:advanced-s3ql-features}\label{special::doc} \section{Snapshotting and Copy-on-Write} -\label{special:snapshotting-and-copy-on-write}\label{special:s3qlcp} +\label{special:s3qlcp}\label{special:snapshotting-and-copy-on-write} The command \code{s3qlcp} can be used to duplicate a directory tree without physically copying the file contents. This is made possible by the data de-duplication feature of S3QL. @@ -1825,7 +1813,7 @@ Trigger a metadata upload. \chapter{Unmounting} -\label{umount:unmounting}\label{umount::doc} +\label{umount::doc}\label{umount:unmounting} To unmount an S3QL file system, use the command: \begin{Verbatim}[commandchars=\\\{\}] @@ -1874,7 +1862,7 @@ upload data in the background for a while longer. \chapter{Checking for Errors} -\label{fsck:checking-for-errors}\label{fsck::doc} +\label{fsck::doc}\label{fsck:checking-for-errors} It is recommended to periodically run the \textbf{fsck.s3ql} and \textbf{s3ql\_verify} commands (in this order) to ensure that the file system is consistent, and that there has been no data corruption @@ -2027,7 +2015,7 @@ Skip over first \textless{}n\textgreater{} objects and with verifying object \chapter{Storing Authentication Information} -\label{authinfo:storing-authentication-information}\label{authinfo:authinfo}\label{authinfo::doc} +\label{authinfo:authinfo}\label{authinfo::doc}\label{authinfo:storing-authentication-information} Normally, S3QL reads username and password for the backend as well as an encryption passphrase for the file system from the terminal. Most commands also accept an \code{-{-}authfile} parameter that can be @@ -2094,7 +2082,7 @@ module}. \chapter{Contributed Programs} -\label{contrib:contributed-programs}\label{contrib::doc} +\label{contrib::doc}\label{contrib:contributed-programs} S3QL comes with a few contributed programs that are not part of the core distribution (and are therefore not installed automatically by default), but which may nevertheless be useful. These programs are in @@ -2119,7 +2107,7 @@ migrate S3 buckets to a different storage region or storage class \section{pcp.py} -\label{contrib:pcp-py}\label{contrib:pcp} +\label{contrib:pcp}\label{contrib:pcp-py} \code{pcp.py} is a wrapper program that starts several rsync processes to copy directory trees in parallel. This is important because transferring files in parallel significantly enhances performance when @@ -2299,7 +2287,7 @@ to zero). \section{Improving copy performance} -\label{tips:improving-copy-performance}\label{tips:copy-performance} +\label{tips:copy-performance}\label{tips:improving-copy-performance} \begin{notice}{note}{Note:} The following applies only when copying data \textbf{from} an S3QL file system, \textbf{not} when copying data \textbf{to} an S3QL file system. @@ -3624,7 +3612,7 @@ system, common locations are \code{/usr/share/doc/s3ql} or \section{The \textbf{s3ql\_oauth\_client} command} -\label{man/oauth_client:the-command-command}\label{man/oauth_client:oauth-client}\label{man/oauth_client::doc} +\label{man/oauth_client:the-command-command}\label{man/oauth_client::doc}\label{man/oauth_client:oauth-client} \subsection{Synopsis} \label{man/oauth_client:synopsis} @@ -4065,7 +4053,7 @@ Please report any bugs you may encounter in the \href{https://bitbucket.org/nikr \chapter{Implementation Details} -\label{impl_details:impl-details}\label{impl_details:implementation-details}\label{impl_details::doc} +\label{impl_details:impl-details}\label{impl_details::doc}\label{impl_details:implementation-details} This section provides some background information on how S3QL works internally. Reading this section is not necessary to use S3QL. diff --git a/doc/man/fsck.s3ql.1 b/doc/man/fsck.s3ql.1 index 982d475..ef7d3ab 100644 --- a/doc/man/fsck.s3ql.1 +++ b/doc/man/fsck.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "FSCK.S3QL" "1" "March 09, 2016" "2.17.1" "S3QL" +.TH "FSCK.S3QL" "1" "March 10, 2016" "2.17.1" "S3QL" .SH NAME fsck.s3ql \- Check an S3QL file system for errors . diff --git a/doc/man/mkfs.s3ql.1 b/doc/man/mkfs.s3ql.1 index f874483..d67f259 100644 --- a/doc/man/mkfs.s3ql.1 +++ b/doc/man/mkfs.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "MKFS.S3QL" "1" "March 09, 2016" "2.17.1" "S3QL" +.TH "MKFS.S3QL" "1" "March 10, 2016" "2.17.1" "S3QL" .SH NAME mkfs.s3ql \- Create an S3QL file system . diff --git a/doc/man/mount.s3ql.1 b/doc/man/mount.s3ql.1 index 53b53d6..f94639c 100644 --- a/doc/man/mount.s3ql.1 +++ b/doc/man/mount.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "MOUNT.S3QL" "1" "March 09, 2016" "2.17.1" "S3QL" +.TH "MOUNT.S3QL" "1" "March 10, 2016" "2.17.1" "S3QL" .SH NAME mount.s3ql \- Mount an S3QL file system . diff --git a/doc/man/s3ql_oauth_client.1 b/doc/man/s3ql_oauth_client.1 index b913ece..9e6f0f3 100644 --- a/doc/man/s3ql_oauth_client.1 +++ b/doc/man/s3ql_oauth_client.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QL_OAUTH_CLIENT" "1" "March 09, 2016" "2.17.1" "S3QL" +.TH "S3QL_OAUTH_CLIENT" "1" "March 10, 2016" "2.17.1" "S3QL" .SH NAME s3ql_oauth_client \- Obtain Google Storage OAuth2 tokens . diff --git a/doc/man/s3ql_verify.1 b/doc/man/s3ql_verify.1 index a08ea84..6959ef7 100644 --- a/doc/man/s3ql_verify.1 +++ b/doc/man/s3ql_verify.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QL_VERIFY" "1" "March 09, 2016" "2.17.1" "S3QL" +.TH "S3QL_VERIFY" "1" "March 10, 2016" "2.17.1" "S3QL" .SH NAME s3ql_verify \- Verify data in an S3QL file system . diff --git a/doc/man/s3qladm.1 b/doc/man/s3qladm.1 index 150ab3c..900d844 100644 --- a/doc/man/s3qladm.1 +++ b/doc/man/s3qladm.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLADM" "1" "March 09, 2016" "2.17.1" "S3QL" +.TH "S3QLADM" "1" "March 10, 2016" "2.17.1" "S3QL" .SH NAME s3qladm \- Manage S3QL file systems . diff --git a/doc/man/s3qlcp.1 b/doc/man/s3qlcp.1 index 5b7cb8f..fb00b72 100644 --- a/doc/man/s3qlcp.1 +++ b/doc/man/s3qlcp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLCP" "1" "March 09, 2016" "2.17.1" "S3QL" +.TH "S3QLCP" "1" "March 10, 2016" "2.17.1" "S3QL" .SH NAME s3qlcp \- Copy-on-write replication on S3QL file systems . diff --git a/doc/man/s3qlctrl.1 b/doc/man/s3qlctrl.1 index df02fee..59ef133 100644 --- a/doc/man/s3qlctrl.1 +++ b/doc/man/s3qlctrl.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLCTRL" "1" "March 09, 2016" "2.17.1" "S3QL" +.TH "S3QLCTRL" "1" "March 10, 2016" "2.17.1" "S3QL" .SH NAME s3qlctrl \- Control a mounted S3QL file system . diff --git a/doc/man/s3qllock.1 b/doc/man/s3qllock.1 index 1a74981..905c1cf 100644 --- a/doc/man/s3qllock.1 +++ b/doc/man/s3qllock.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLLOCK" "1" "March 09, 2016" "2.17.1" "S3QL" +.TH "S3QLLOCK" "1" "March 10, 2016" "2.17.1" "S3QL" .SH NAME s3qllock \- Make trees on an S3QL file system immutable . diff --git a/doc/man/s3qlrm.1 b/doc/man/s3qlrm.1 index 2b1e366..578b30b 100644 --- a/doc/man/s3qlrm.1 +++ b/doc/man/s3qlrm.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLRM" "1" "March 09, 2016" "2.17.1" "S3QL" +.TH "S3QLRM" "1" "March 10, 2016" "2.17.1" "S3QL" .SH NAME s3qlrm \- Fast tree removal on S3QL file systems . diff --git a/doc/man/s3qlstat.1 b/doc/man/s3qlstat.1 index faf33ff..34f0308 100644 --- a/doc/man/s3qlstat.1 +++ b/doc/man/s3qlstat.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLSTAT" "1" "March 09, 2016" "2.17.1" "S3QL" +.TH "S3QLSTAT" "1" "March 10, 2016" "2.17.1" "S3QL" .SH NAME s3qlstat \- Gather S3QL file system statistics . diff --git a/doc/man/umount.s3ql.1 b/doc/man/umount.s3ql.1 index 94273fb..f9bb7b6 100644 --- a/doc/man/umount.s3ql.1 +++ b/doc/man/umount.s3ql.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "UMOUNT.S3QL" "1" "March 09, 2016" "2.17.1" "S3QL" +.TH "UMOUNT.S3QL" "1" "March 10, 2016" "2.17.1" "S3QL" .SH NAME umount.s3ql \- Unmount an S3QL file system . diff --git a/doc/manual.pdf b/doc/manual.pdf Binary files differindex 00a2d3e..ebfc6ac 100644 --- a/doc/manual.pdf +++ b/doc/manual.pdf diff --git a/rst/installation.rst b/rst/installation.rst index 7c1d79b..a63bdce 100644 --- a/rst/installation.rst +++ b/rst/installation.rst @@ -45,72 +45,49 @@ that is not the case. * The `psmisc <http://psmisc.sf.net/>`_ utilities. -* `Python <http://www.python.org/>`_ 3.3.0 or newer. Make sure to also - install the development headers. - -* The `setuptools Python Module - <https://pypi.python.org/pypi/setuptools>`_, version 1.0 or newer. - To check if with version (if any) of this module is installed, try - to execute :: - - python3 -c 'import setuptools; print(setuptools.__version__)' - -* The `PyCrypto Python Module - <https://www.dlitz.net/software/pycrypto/>`_. To check if this - module is installed, try to execute `python3 -c 'import Crypto'`. - -* The `Python defusedxml module - <https://pypi.python.org/pypi/defusedxml/>`_. To check if this - module is installed, try to execute `python3 -c 'import defusedxml'`. - -* If you want to use OAuth2 authentication with Google Storage, you - need the `Python requests module - <https://pypi.python.org/pypi/requests/>`_. To check if this module - is installed, try to execute `python3 -c 'import requests'`. - -* If you want to use systemd integration, you need the `Python systemd - module <https://github.com/systemd/python-systemd>`_. To check if - this module is installed, try to execute `python3 -c 'import systemd'`. - * `SQLite <http://www.sqlite.org/>`_ version 3.7.0 or newer. SQLite has to be installed as a *shared library* with development headers. -* The `APSW Python Module <http://code.google.com/p/apsw/>`_. To check - which (if any) version of APWS is installed, run the command :: - - python3 -c 'import apsw; print(apsw.apswversion())' - - The printed version number should be at least 3.7.0. - -* The `Python LLFUSE module - <https://pypi.python.org/pypi/llfuse/>`_. To check which (if any) - version of if this module is installed, execute `python3 -c 'import - llfuse; print(llfuse.__version__)'`. Any version between 1.0 - (inclusive) and 2.0 (exclusive) will do. - -* The `Python dugong module - <https://bitbucket.org/nikratio/python-dugong/>`_. To check if this - module is installed, try to execute `python3 -c 'import dugong; - print(dugong.__version__)'`. This should print a version number. You - need at least version 3.4. +* `Python <http://www.python.org/>`_ 3.3.0 or newer. Make sure to also + install the development headers. -* If your system is using `systemd`_, make sure that the ``systemd`` - Python module is available for Python 3. To check this, try to run - ``python3 -c 'import systemd.daemon'``. +* The following Python modules: + + * `setuptools <https://pypi.python.org/pypi/setuptools>`_, version 1.0 or newer. + * `pycrypto <https://www.dlitz.net/software/pycrypto/>`_ + * `defusedxml <https://pypi.python.org/pypi/defusedxml/>`_ + * `requests <https://pypi.python.org/pypi/requests/>`_ (optional, + required for OAuth2 authentication with Google Storage) + * `systemd <https://github.com/systemd/python-systemd>`_ (optional, + for enabling systemd support). + * `apsw <https://github.com/rogerbinns/apsw>`_, version 3.7.0 or + newer. + * `llfuse <https://bitbucket.org/nikratio/python-llfuse/>`_, any + version between 1.0 (inclusive) and 2.0 (exclusive) + * `dugong <https://bitbucket.org/nikratio/python-dugong/>`_, any + version between 3.4 (inclusive) and 4.0 (exclusive) + * `pytest <http://pytest.org/>`_, version 2.3.3 or newer (optional, to run unit tests) + * `pytest-catchlog <https://github.com/eisensheng/pytest-catchlog>`_ + (optional, to run unit tests) + + To check if a specific module :var:`<module>` is installed, execute + :samp:`python3 -c 'import {<module>}; + print({<module>}.__version__)'`. This will result in an + `ImportError` if the module is not installed, and will print the + installed version if the module is installed. -.. _systemd: http://www.freedesktop.org/wiki/Software/systemd/ .. _inst-s3ql: Installing S3QL =============== -To install S3QL itself, proceed as follows: +To build and install S3QL itself, proceed as follows: 1. Download S3QL from https://bitbucket.org/nikratio/s3ql/downloads 2. Unpack it into a folder of your choice 3. Run `python3 setup.py build_ext --inplace` to build S3QL. -4. Run `python3 runtests.py tests` to run a self-test. If this fails, ask +4. Run `python3 -m pytest tests/` to run a self-test. If this fails, ask for help on the `mailing list <http://groups.google.com/group/s3ql>`_ or report a bug in the `issue tracker <https://bitbucket.org/nikratio/s3ql/issues>`_. @@ -137,24 +114,21 @@ Mercurial repository, a bit more effort is required. You'll also need: * Version 1.2b1 or newer of the Sphinx_ document processor. -* The `py.test`_ testing tool, version 2.3.3 or newer. - With these additional dependencies installed, S3QL can be build and tested with :: python3 setup.py build_cython python3 setup.py build_ext --inplace - py.test tests/ + python3 -m pytest tests/ -Note that when building from the Mercurial repository, building and -testing is done with several additional checks. This may cause +Note that when building from the Mercurial or Git repository, building +and testing is done with several additional checks. This may cause compilation and/or tests to fail even though there are no problems -with functionality. For example, when building from the Mercurial -repository, any use of functions that are scheduled for deprecation in -future Python version will cause tests to fail. If you would rather -just check for functionality, you can delete the :file:`MANIFEST.in` -file. In that case, the build system will behave as it does for a -regular release. +with functionality. For example, any use of functions that are +scheduled for deprecation in future Python version will cause tests to +fail. If you would rather just check for functionality, you can delete +the :file:`MANIFEST.in` file. In that case, the build system will +behave as it does for a regular release. The HTML and PDF documentation can be generated with :: @@ -168,12 +142,11 @@ and S3QL can be installed as usual with :: Running tests requiring remote servers ====================================== -By default, the `runtest.py` (or `py.test`) script skips all tests -that require connection to a remote storage backend. If you would like -to run these tests too (which is always a good idea), you have to -create additional entries in your `~/.s3ql/authinfo2` file that tell -S3QL what server and credentials to use for these tests. These entries -have the following form:: +By default, tests requiring a connection to a remote storage backend +are skipped. If you would like to run these tests too (which is always +a good idea), you have to create additional entries in your +`~/.s3ql/authinfo2` file that tell S3QL what server and credentials to +use for these tests. These entries have the following form:: [<BACKEND>-test] backend-login: <user> @@ -204,4 +177,3 @@ being skipped by passing the :cmdopt:`-rs` argument to .. _Cython: http://www.cython.org/ .. _Sphinx: http://sphinx.pocoo.org/ -.. _py.test: http://pytest.org/ @@ -2,7 +2,7 @@ allow_hosts = None [egg_info] -tag_svn_revision = 0 tag_build = +tag_svn_revision = 0 tag_date = 0 @@ -23,22 +23,15 @@ from setuptools import Extension from distutils.version import LooseVersion import os import subprocess -import warnings -import shutil import re from glob import glob import faulthandler faulthandler.enable() -# When running from HG repo, enable all warnings basedir = os.path.abspath(os.path.dirname(sys.argv[0])) DEVELOPER_MODE = os.path.exists(os.path.join(basedir, 'MANIFEST.in')) if DEVELOPER_MODE: print('MANIFEST.in exists, running in developer mode') - warnings.resetwarnings() - # We can't use `error`, because e.g. Sphinx triggers a - # DeprecationWarning. - warnings.simplefilter('default') # Add S3QL sources sys.path.insert(0, os.path.join(basedir, 'src')) @@ -189,7 +182,6 @@ def main(): }, install_requires=required_pkgs, cmdclass={'upload_docs': upload_docs, - 'make_testscript': make_testscript, 'build_cython': build_cython, 'build_sphinx': build_docs }, command_options={ 'sdist': { 'formats': ('setup.py', 'bztar') } }, @@ -260,32 +252,6 @@ class upload_docs(setuptools.Command): subprocess.check_call(['rsync', '-aHv', '--del', os.path.join(basedir, 'doc', 'manual.pdf'), 'ebox.rath.org:/srv/www.rath.org/s3ql-docs/']) -class make_testscript(setuptools.Command): - user_options = [] - boolean_options = [] - description = "Generate standalone py.test script" - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - import pytest - pytest.main(['--genscript', 'runtests.py']) - - # Fixup shebang - with open('runtests.py.tmp', 'wb') as ofh, \ - open('runtests.py', 'rb') as ifh: - ofh.write(b'#!/usr/bin/env python3\n') - ifh.readline() - shutil.copyfileobj(ifh, ofh) - os.rename('runtests.py.tmp', 'runtests.py') - - # Make executable - os.chmod('runtests.py', 0o755) - def fix_docutils(): '''Work around https://bitbucket.org/birkenfeld/sphinx/issue/1154/''' diff --git a/src/s3ql.egg-info/PKG-INFO b/src/s3ql.egg-info/PKG-INFO index ee234f3..7cf1735 100644 --- a/src/s3ql.egg-info/PKG-INFO +++ b/src/s3ql.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: s3ql -Version: 2.17.1 +Version: 2.17.1-hg2 Summary: a full-featured file system for online data storage Home-page: https://bitbucket.org/nikratio/s3ql/ Author: Nikolaus Rath @@ -92,13 +92,12 @@ Description: .. Development Status ================== - S3QL is considered stable and suitable for production use. However, - upgrades from one minor version to the next (e.g. *2.x* to *2.x+1*) - may change the public interface (e.g. different command line options), - or require the file system structure to be upgraded (so that the file - system can no longer be accessed by older releases). Therefore, it is - strongly recommended to read the changelog (`Changes.txt` in the S3QL - tarball) before upgrading. + S3QL is considered stable and suitable for production use. Starting + with version 2.17.1, S3QL uses semantic versioning. This means that + backwards-incompatible versions (e.g., versions that require an + upgrade of the file system revision) will be reflected in an increase + of the major version number. + Supported Platforms =================== @@ -111,22 +110,6 @@ Description: .. try to fix them. - Which Version Should I Download? - ================================ - - Short answer: if your system supports Python 3.3 or newer, download - the most recent *2.x* version. - - Long answer: there are two supported branches of S3QL. Both branches - are suitable for production use. The *maint-1.x* branch (version - numbers *1.x*) is no longer actively developed and receives only - selected high-impact bugfixes. It is provided for systems without - Python 3 support. For systems with Python 3.3 or newer, it is - recommended run the *default* S3QL branch (with version numbers - *2.x*). This branch is actively developed and has a number of new - features that are not available in the *1.x* versions. - - Typical Usage ============= diff --git a/src/s3ql.egg-info/SOURCES.txt b/src/s3ql.egg-info/SOURCES.txt index dc1fed6..654bb97 100644 --- a/src/s3ql.egg-info/SOURCES.txt +++ b/src/s3ql.egg-info/SOURCES.txt @@ -228,6 +228,7 @@ src/s3ql/backends/swiftks.py tests/common.py tests/conftest.py tests/mock_server.py +tests/mytest.py tests/pytest.ini tests/t1_backends.py tests/t1_dump.py diff --git a/src/s3ql/__init__.py b/src/s3ql/__init__.py index 0843967..4259f03 100644 --- a/src/s3ql/__init__.py +++ b/src/s3ql/__init__.py @@ -22,7 +22,7 @@ __all__ = [ 'adm', 'backends', 'block_cache', 'common', 'calc_mro', 'REV_VER_MAP', 'RELEASE', 'BUFSIZE', 'CTRL_NAME', 'CTRL_INODE' ] -VERSION = '2.17.1' +VERSION = '2.17.1+hg2' RELEASE = '%s' % VERSION # TODO: On next revision bump, consider removing support for TIME diff --git a/src/s3ql/adm.py b/src/s3ql/adm.py index c8ca0db..d0e6757 100644 --- a/src/s3ql/adm.py +++ b/src/s3ql/adm.py @@ -64,7 +64,6 @@ def parse_args(args): parser.add_backend_options() parser.add_cachedir() parser.add_version() - parser.add_fatal_warnings() options = parser.parse_args(args) diff --git a/src/s3ql/cp.py b/src/s3ql/cp.py index 159b1b5..a6e31a4 100644 --- a/src/s3ql/cp.py +++ b/src/s3ql/cp.py @@ -35,7 +35,6 @@ def parse_args(args): parser.add_debug() parser.add_quiet() parser.add_version() - parser.add_fatal_warnings() parser.add_argument('source', help='source directory', type=(lambda x: x.rstrip('/'))) diff --git a/src/s3ql/ctrl.py b/src/s3ql/ctrl.py index ad5f4ae..ab89cc8 100644 --- a/src/s3ql/ctrl.py +++ b/src/s3ql/ctrl.py @@ -34,7 +34,6 @@ def parse_args(args): parser.add_debug() parser.add_quiet() parser.add_version() - parser.add_fatal_warnings() subparsers = parser.add_subparsers(metavar='<action>', dest='action', help='may be either of') diff --git a/src/s3ql/fsck.py b/src/s3ql/fsck.py index 1235da0..3b03414 100644 --- a/src/s3ql/fsck.py +++ b/src/s3ql/fsck.py @@ -1130,7 +1130,6 @@ def parse_args(args): parser.add_backend_options() parser.add_version() parser.add_storage_url() - parser.add_fatal_warnings() parser.add_argument("--batch", action="store_true", default=False, help="If user input is required, exit without prompting.") diff --git a/src/s3ql/lock.py b/src/s3ql/lock.py index 79dd491..645e321 100644 --- a/src/s3ql/lock.py +++ b/src/s3ql/lock.py @@ -29,7 +29,6 @@ def parse_args(args): parser.add_debug() parser.add_quiet() parser.add_version() - parser.add_fatal_warnings() parser.add_argument('path', metavar='<path>', nargs='+', help='Directories to make immutable.', diff --git a/src/s3ql/logging.py b/src/s3ql/logging.py index f61c733..b66482e 100644 --- a/src/s3ql/logging.py +++ b/src/s3ql/logging.py @@ -12,27 +12,6 @@ import warnings import sys import os.path -# Logging messages with severities larger or equal -# than this value will raise exceptions. -EXCEPTION_SEVERITY = logging.CRITICAL+1 - - -class LoggingError(Exception): - ''' - Raised when a `Logger` instance is used to log a message with - a severity larger than its `exception_severity`. - ''' - - formatter = logging.Formatter('%(message)s') - - def __init__(self, record): - super().__init__() - self.record = record - - def __str__(self): - return 'Unexpected log message: ' + self.formatter.format(self.record) - - class QuietError(Exception): ''' QuietError is the base class for exceptions that should not result @@ -53,6 +32,14 @@ class QuietError(Exception): def __str__(self): return self.msg +class MyFormatter(logging.Formatter): + '''Prepend severity to log message if it exceeds threshold''' + + def format(self, record): + s = super().format(record) + if record.levelno > logging.INFO: + s = '%s: %s' % (record.levelname, s) + return s def create_handler(target): '''Create logging handler for given target''' @@ -75,7 +62,7 @@ def create_handler(target): try: handler = logging.handlers.RotatingFileHandler(fullpath, - maxBytes=1024 ** 2, backupCount=5) + maxBytes=10 * 1024**2, backupCount=5) except PermissionError: raise QuietError('No permission to write log file %s' % fullpath, exitcode=10) @@ -88,6 +75,12 @@ def create_handler(target): return handler def setup_logging(options): + + # We want to be able to detect warnings and higher severities + # in the captured test output. 'critical' has too many potential + # false positives, so we rename this level to "FATAL". + logging.addLevelName(logging.CRITICAL, 'FATAL') + root_logger = logging.getLogger() if root_logger.handlers: root_logger.debug("Logging already initialized.") @@ -99,8 +92,8 @@ def setup_logging(options): elif options.debug and (not hasattr(options, 'log') or not options.log): # When we have debugging enabled but no separate log target, # make stdout logging more detailed. - formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(process)s %(threadName)s ' - '%(name)s.%(funcName)s: %(message)s', + formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(process)s %(levelname)-8s ' + '%(threadName)s %(name)s.%(funcName)s: %(message)s', datefmt="%Y-%m-%d %H:%M:%S") stdout_handler.setFormatter(formatter) stdout_handler.setLevel(logging.NOTSET) @@ -119,24 +112,7 @@ def setup_logging(options): else: root_logger.setLevel(logging.INFO) logging.disable(logging.DEBUG) - - - if hasattr(options, 'fatal_warnings') and options.fatal_warnings: - global EXCEPTION_SEVERITY - EXCEPTION_SEVERITY = logging.WARNING - - # When ResourceWarnings are emitted, any exceptions thrown - # by warning.showwarning() are lost (not even printed to stdout) - # so we cannot rely on our logging mechanism to turn these - # warnings into (visible) exceptions. Instead, we assume that - # --fatal-warnings implies that ResourceWarnings should be - # enabled and tell the warnings module to raise exceptions - # immediately (so that they're at least printed to stderr). - warnings.simplefilter('error', ResourceWarning) - - else: - logging.captureWarnings(capture=True) - + logging.captureWarnings(capture=True) return stdout_handler @@ -150,35 +126,24 @@ def setup_excepthook(): def excepthook(type_, val, tb): root_logger = logging.getLogger() if isinstance(val, QuietError): - # force_log attribute ensures that logging handler will - # not raise exception (if EXCEPTION_SEVERITY is set) - root_logger.error(val.msg, extra={ 'force_log': True }) + root_logger.error(val.msg) sys.exit(val.exitcode) else: - # force_log attribute ensures that logging handler will - # not raise exception (if EXCEPTION_SEVERITY is set) root_logger.error('Uncaught top-level exception:', - exc_info=(type_, val, tb), - extra={ 'force_log': True}) + exc_info=(type_, val, tb)) sys.exit(1) sys.excepthook = excepthook - def add_stdout_logging(quiet=False): - '''Add stdout logging handler to root logger - - If *quiet* is True, logging is sent to stderr rather than stdout, and only - messages with severity WARNING are printed. - ''' + '''Add stdout logging handler to root logger''' root_logger = logging.getLogger() - formatter = logging.Formatter('%(message)s') + formatter = MyFormatter('%(message)s') + handler = logging.StreamHandler(sys.stderr) if quiet: - handler = logging.StreamHandler(sys.stderr) handler.setLevel(logging.WARNING) else: - handler = logging.StreamHandler(sys.stdout) handler.setLevel(logging.INFO) handler.setFormatter(formatter) root_logger.addHandler(handler) @@ -188,9 +153,6 @@ class Logger(logging.getLoggerClass()): ''' This class has the following features in addition to `logging.Logger`: - * Loggers can automatically raise exceptions when a log message exceeds - a specified severity. This is useful when running unit tests. - * Log messages that are emitted with an *log_once* attribute in the *extra* parameter are only emitted once per logger. ''' @@ -200,10 +162,6 @@ class Logger(logging.getLoggerClass()): self.log_cache = set() def handle(self, record): - if (record.levelno >= EXCEPTION_SEVERITY - and not hasattr(record, 'force_log')): - raise LoggingError(record) - if hasattr(record, 'log_once') and record.log_once: id_ = hash((record.name, record.levelno, record.msg, record.args, record.exc_info)) @@ -211,28 +169,9 @@ class Logger(logging.getLoggerClass()): return self.log_cache.add(id_) - # Do not call superclass method directly so that we can - # re-use this method when monkeypatching the root logger. - return self._handle_real(record) - - def _handle_real(self, record): return super().handle(record) - +logging.setLoggerClass(Logger) # Convenience object for use in logging calls, e.g. # log.warning('This will be printed only once', extra=LOG_ONCE) LOG_ONCE = { 'log_once': True } - -# Ensure that no handlers have been created yet -loggers = logging.Logger.manager.loggerDict -if len(loggers) != 0: - raise ImportError('%s must be imported before loggers are created! ' - 'Existing loggers: %s' % (__name__, loggers.keys())) - -# Monkeypatch the root logger -#pylint: disable=W0212 -root_logger_class = type(logging.getLogger()) -root_logger_class._handle_real = root_logger_class.handle -root_logger_class.handle = Logger.handle - -logging.setLoggerClass(Logger) diff --git a/src/s3ql/mkfs.py b/src/s3ql/mkfs.py index b6da7ce..310a78a 100644 --- a/src/s3ql/mkfs.py +++ b/src/s3ql/mkfs.py @@ -38,7 +38,6 @@ def parse_args(args): parser.add_backend_options() parser.add_version() parser.add_storage_url() - parser.add_fatal_warnings() parser.add_argument("-L", default='', help="Filesystem label", dest="label", metavar='<name>',) @@ -90,8 +89,8 @@ def main(args=None): if options.max_obj_size < 1024: # This warning should never be converrted to an exception - log.warning('Warning: maximum object sizes less than 1 MiB will seriously degrade ' - 'performance.', extra={ 'force_log': True }) + log.warning('Maximum object sizes less than 1 MiB will degrade ' + 'performance.', extra={ 'force_log': True }) plain_backend = get_backend(options, raw=True) atexit.register(plain_backend.close) @@ -100,10 +99,9 @@ def main(args=None): "the 'Important Rules to Avoid Loosing Data' section.") if isinstance(plain_backend, s3.Backend) and '.' in plain_backend.bucket_name: - log.warning('***Warning*** S3 Buckets with names containing dots cannot be ' - 'accessed using SSL!') - log.warning('(cf. https://forums.aws.amazon.com/thread.jspa?threadID=130560)') - + log.warning('S3 Buckets with names containing dots cannot be ' + 'accessed using SSL!' + '(cf. https://forums.aws.amazon.com/thread.jspa?threadID=130560)') if 's3ql_metadata' in plain_backend: if not options.force: diff --git a/src/s3ql/mount.py b/src/s3ql/mount.py index ffb9b2f..f2fef6f 100644 --- a/src/s3ql/mount.py +++ b/src/s3ql/mount.py @@ -139,7 +139,7 @@ def main(args=None): options.cachesize = min(rec_cachesize, 0.8 * avail_cache) log.info('Setting cache size to %d MB', options.cachesize / 1024) elif options.cachesize > avail_cache: - log.warning('Warning! Requested cache size %d MB, but only %d MB available', + log.warning('Requested cache size %d MB, but only %d MB available', options.cachesize / 1024, avail_cache / 1024) if options.nfs: @@ -342,8 +342,8 @@ def determine_threads(options): if threads > 0: log.info('Using %d upload threads (memory limited).', threads) else: - log.warning('Warning: compression will require %d MiB memory ' - '(%d%% of total system memory', mem_per_thread / 1024 ** 2, + log.warning('Compression will require %d MiB memory ' + '(%d%% of total system memory', mem_per_thread / 1024 ** 2, mem_per_thread * 100 / memory) threads = 1 return threads @@ -504,7 +504,6 @@ def parse_args(args): parser.add_backend_options() parser.add_version() parser.add_storage_url() - parser.add_fatal_warnings() parser.add_argument("mountpoint", metavar='<mountpoint>', type=os.path.abspath, help='Where to mount the file system') diff --git a/src/s3ql/parse_args.py b/src/s3ql/parse_args.py index f7361d1..163b997 100644 --- a/src/s3ql/parse_args.py +++ b/src/s3ql/parse_args.py @@ -177,13 +177,6 @@ class ArgumentParser(argparse.ArgumentParser): 'will be rotated when they reach 1 MiB, and at most 5 old log ' 'files will be kept. Default: ``%(default)s``') - def add_fatal_warnings(self): - # Make all log messages of severity warning or higher raise - # exceptions. This option is not listed in the --help output and used by - # the unit tests. - self.add_argument("--fatal-warnings", action='store_true', default=False, - help=argparse.SUPPRESS) - def add_storage_url(self): self.add_argument("storage_url", metavar='<storage-url>', type=storage_url_type, diff --git a/src/s3ql/remove.py b/src/s3ql/remove.py index c14d125..b0f1395 100644 --- a/src/s3ql/remove.py +++ b/src/s3ql/remove.py @@ -28,7 +28,6 @@ def parse_args(args): parser.add_debug() parser.add_quiet() parser.add_version() - parser.add_fatal_warnings() parser.add_argument('path', metavar='<path>', nargs='+', help='Directories to remove', diff --git a/src/s3ql/statfs.py b/src/s3ql/statfs.py index 533c68a..57c3408 100644 --- a/src/s3ql/statfs.py +++ b/src/s3ql/statfs.py @@ -24,7 +24,6 @@ def parse_args(args): parser.add_debug() parser.add_quiet() parser.add_version() - parser.add_fatal_warnings() parser.add_argument("mountpoint", metavar='<mountpoint>', type=(lambda x: x.rstrip('/')), diff --git a/src/s3ql/umount.py b/src/s3ql/umount.py index 7ec711c..cd55146 100644 --- a/src/s3ql/umount.py +++ b/src/s3ql/umount.py @@ -35,7 +35,6 @@ def parse_args(args): parser.add_debug() parser.add_quiet() parser.add_version() - parser.add_fatal_warnings() parser.add_argument("mountpoint", metavar='<mountpoint>', type=(lambda x: x.rstrip('/')), diff --git a/tests/common.py b/tests/common.py index cc535bb..5f2ec9c 100644 --- a/tests/common.py +++ b/tests/common.py @@ -60,7 +60,7 @@ def install_safe_sleep(): @contextmanager def catch_logmsg(pattern, level=logging.WARNING, count=None): - '''Catch (and ignore) log messages matching *pattern* + '''Catch log messages matching *pattern* *pattern* is matched against the *unformatted* log message, i.e. before any arguments are merged. @@ -75,10 +75,11 @@ def catch_logmsg(pattern, level=logging.WARNING, count=None): @wraps(handle_orig) def handle_new(self, record): - if (record.levelno != level - or not re.search(pattern, record.msg)): - return handle_orig(self, record) - caught[0] += 1 + if (record.levelno == level + and re.search(pattern, record.msg)): + caught[0] += 1 + record.caplog_ignore = True + return handle_orig(self, record) logger_class.handle = handle_new try: @@ -88,8 +89,8 @@ def catch_logmsg(pattern, level=logging.WARNING, count=None): logger_class.handle = handle_orig if count is not None and caught[0] != count: - raise AssertionError('Expected to catch %d log messages, but got only %d' - % (count, caught[0])) + raise AssertionError('Expected to catch %d log %r messages, but got only %d' + % (count, pattern, caught[0])) def retry(timeout, fn, *a, **kw): """Wait for fn(*a, **kw) to return True. diff --git a/tests/conftest.py b/tests/conftest.py index fd1f4a6..c344574 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,28 +25,7 @@ import faulthandler import signal import gc import time - -# Converted to autouse fixture below if capture is activated -def check_test_output(request, capfd): - request.capfd = capfd - def raise_on_exception_in_out(): - # Ensure that capturing has been set up (this may not be the case if one - # of the test fixtures raises an exception) - try: - (stdout, stderr) = capfd.readouterr() - except AttributeError: - return - - # Write back what we've read (so that it will still be printed. - sys.stdout.write(stdout) - sys.stderr.write(stderr) - - if ('exception' in stderr.lower() - or 'exception' in stdout.lower()): - raise AssertionError('Suspicious output to stderr') - - request.addfinalizer(raise_on_exception_in_out) - +import re # If a test fails, wait a moment before retrieving the captured # stdout/stderr. When using a server process (like in t4_fuse.py), this makes @@ -73,6 +52,95 @@ def s3ql_cmd_argv(request): request.cls.s3ql_cmd_argv = lambda self, cmd: [ sys.executable, os.path.join(basedir, 'bin', cmd) ] +@pytest.fixture() +def pass_capfd(request, capfd): + '''Provide capfd object to UnitTest instances''' + request.instance.capfd = capfd + +# Fail tests if the result in log messages of severity WARNING or more. +# Previously (as of Mercurial commit 192dd923daa8, 2016-03-10) we instead +# installed a custom Logger class that would immediately raise an +# exception. However, this solution seemed ugly because normally the logging +# methods suppress any exceptions. Therefore, exception handlers (which are +# especially likely to log warnings) may rely on that and an unexpected +# exception may result in improper clean-up. Furthermore, the custom logger +# class required a second hack to allow logging the "unexpected warning message" +# exception itself from sys.excepthook. Checking the logs after execution has +# the drawback that we abort later, and that the failure seems to come from the +# teardown method, but seems like an overall better solution. +def check_test_log(caplog): + for record in caplog.records(): + if (record.levelno >= logging.WARNING and + not getattr(record, 'caplog_ignore', False)): + raise AssertionError('Logger received warning messages') + +def check_test_output(capfd): + (stdout, stderr) = capfd.readouterr() + + # Write back what we've read (so that it will still be printed. + sys.stdout.write(stdout) + sys.stderr.write(stderr) + + # Strip out false positives + for (pattern, flags, count) in capfd.false_positives: + cp = re.compile(pattern, flags) + (stdout, cnt) = cp.subn('', stdout, count=count) + if count == 0 or count - cnt > 0: + stderr = cp.sub('', stderr, count=count - cnt) + + for pattern in ('exception', 'error', 'warning', 'fatal', + 'fault', 'crash(?:ed)?', 'abort(?:ed)'): + cp = re.compile(r'\b{}\b'.format(pattern), re.IGNORECASE | re.MULTILINE) + hit = cp.search(stderr) + if hit: + raise AssertionError('Suspicious output to stderr (matched "%s")' % hit.group(0)) + hit = cp.search(stdout) + if hit: + raise AssertionError('Suspicious output to stdout (matched "%s")' % hit.group(0)) + + +def register_output(self, pattern, count=1, flags=re.MULTILINE): + '''Register *pattern* as false positive for output checking + + This prevents the test from failing because the output otherwise + appears suspicious. + ''' + + self.false_positives.append((pattern, flags, count)) + +# This is a terrible hack that allows us to access the fixtures from the +# pytest_runtest_call hook. Among a lot of other hidden assumptions, it probably +# relies on tests running sequential (i.e., don't dare to use e.g. the xdist +# plugin) +current_cap_fixtures = None +@pytest.yield_fixture(autouse=True) +def save_cap_fixtures(request, capfd, caplog): + global current_cap_fixtures + capfd.false_positives = [] + type(capfd).register_output = register_output + + # Ignore DeprecationWarnings when running unit tests. They are + # unfortunately quite often a result of indirect imports via third party + # modules, so we can't actually fix them. + capfd.register_output(r'^(WARNING: )?(Pending)?DeprecationWarning: [^\n]+$', count=0) + + if request.config.getoption('capture') == 'no': + capfd = None + current_cap_fixtures = (capfd, caplog) + bak = current_cap_fixtures + yield + # Try to catch problems with this hack (e.g. when running + # tests simultaneously) + assert bak is current_cap_fixtures + current_cap_fixtures = None + +@pytest.hookimpl(trylast=True) +def pytest_runtest_call(item): + (capfd, caplog) = current_cap_fixtures + check_test_log(caplog) + if capfd is not None: + check_test_output(capfd) + def pytest_addoption(parser): group = parser.getgroup("terminal reporting") group._addoption("--logdebug", action="append", metavar='<module>', @@ -84,16 +152,7 @@ def pytest_addoption(parser): group._addoption("--installed", action="store_true", default=False, help="Test the installed package.") - def pytest_configure(config): - - # Enable stdout and stderr analysis, unless output capture is disabled - if config.getoption('capture') != 'no': - global check_test_output - check_test_output = pytest.fixture(autouse=True)(check_test_output) - - logdebug = config.getoption('logdebug') - # If we are running from the S3QL source directory, make sure that we # load modules from here basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) @@ -114,42 +173,21 @@ def pytest_configure(config): faulthandler.enable(faultlog_fh) faulthandler.register(signal.SIGUSR1, file=faultlog_fh) - # Enable logging - import s3ql.logging + # Configure logging. We don't set a default handler but rely on + # the catchlog pytest plugin. + logdebug = config.getoption('logdebug') root_logger = logging.getLogger() - if root_logger.handlers: - root_logger.warning("Logging already initialized.") - else: - handler = logging.handlers.RotatingFileHandler( - os.path.join(basedir, 'tests', 'test.log'), - maxBytes=10 * 1024 ** 2, backupCount=0) - if logdebug is None: - formatter = logging.Formatter( - '%(asctime)s.%(msecs)03d [%(process)s] %(threadName)s: ' - '[%(name)s] %(message)s', datefmt="%Y-%m-%d %H:%M:%S") - else: - formatter = logging.Formatter( - '%(asctime)s.%(msecs)03d [%(process)s] %(threadName)s: ' - '[%(name)s.%(funcName)s] %(message)s', datefmt="%Y-%m-%d %H:%M:%S") - - handler.setFormatter(formatter) - root_logger.addHandler(handler) - - if logdebug is not None: - if 'all' in logdebug: - root_logger.setLevel(logging.DEBUG) - else: - for module in logdebug: - logging.getLogger(module).setLevel(logging.DEBUG) - logging.disable(logging.NOTSET) + if logdebug is not None: + logging.disable(logging.NOTSET) + if 'all' in logdebug: + root_logger.setLevel(logging.DEBUG) else: - root_logger.setLevel(logging.WARNING) - - logging.captureWarnings(capture=True) - - # Make errors and warnings fatal - s3ql.logging.EXCEPTION_SEVERITY = logging.WARNING - + for module in logdebug: + logging.getLogger(module).setLevel(logging.DEBUG) + else: + root_logger.setLevel(logging.INFO) + logging.disable(logging.DEBUG) + logging.captureWarnings(capture=True) # Run gc.collect() at the end of every test, so that we get ResourceWarnings # as early as possible. diff --git a/tests/mytest.py b/tests/mytest.py new file mode 100644 index 0000000..1f93333 --- /dev/null +++ b/tests/mytest.py @@ -0,0 +1,9 @@ +import logging +import warnings + +log = logging.getLogger(__name__) + +def test(): + log.info('This is just some info') + log.critical('This is really bad') + warnings.warn("This is not good", UserWarning) diff --git a/tests/t4_adm.py b/tests/t4_adm.py index af338cc..391c906 100755 --- a/tests/t4_adm.py +++ b/tests/t4_adm.py @@ -20,8 +20,7 @@ import unittest import subprocess import pytest - -@pytest.mark.usefixtures('s3ql_cmd_argv') +@pytest.mark.usefixtures('s3ql_cmd_argv', 'pass_capfd') class AdmTests(unittest.TestCase): def setUp(self): @@ -37,7 +36,7 @@ class AdmTests(unittest.TestCase): def mkfs(self): proc = subprocess.Popen(self.s3ql_cmd_argv('mkfs.s3ql') + - ['-L', 'test fs', '--max-obj-size', '500', '--fatal-warnings', + ['-L', 'test fs', '--max-obj-size', '500', '--authfile', '/dev/null', '--cachedir', self.cache_dir, '--quiet', self.storage_url ], stdin=subprocess.PIPE, universal_newlines=True) @@ -47,6 +46,8 @@ class AdmTests(unittest.TestCase): proc.stdin.close() self.assertEqual(proc.wait(), 0) + self.capfd.register_output(r'^WARNING: Maximum object sizes less than ' + '1 MiB will degrade performance\.$', count=1) def test_passphrase(self): self.mkfs() @@ -54,7 +55,7 @@ class AdmTests(unittest.TestCase): passphrase_new = 'sd982jhd' proc = subprocess.Popen(self.s3ql_cmd_argv('s3qladm') + - [ '--quiet', '--fatal-warnings', '--log', 'none', '--authfile', + [ '--quiet', '--log', 'none', '--authfile', '/dev/null', 'passphrase', self.storage_url ], stdin=subprocess.PIPE, universal_newlines=True) @@ -86,7 +87,7 @@ class AdmTests(unittest.TestCase): fh.flush() proc = subprocess.Popen(self.s3ql_cmd_argv('fsck.s3ql') + - [ '--quiet', '--fatal-warnings', '--authfile', fh.name, + [ '--quiet', '--authfile', fh.name, '--cachedir', self.cache_dir, '--log', 'none', self.storage_url ], stdin=subprocess.PIPE, universal_newlines=True) diff --git a/tests/t4_fuse.py b/tests/t4_fuse.py index d1dfade..f9dcde9 100755 --- a/tests/t4_fuse.py +++ b/tests/t4_fuse.py @@ -30,7 +30,7 @@ from pytest import raises as assert_raises # For debugging USE_VALGRIND = False -@pytest.mark.usefixtures('s3ql_cmd_argv') +@pytest.mark.usefixtures('s3ql_cmd_argv', 'pass_capfd') class TestFuse: def setup_method(self, method): @@ -57,7 +57,7 @@ class TestFuse: def mkfs(self, max_obj_size=500): argv = (self.s3ql_cmd_argv('mkfs.s3ql') + [ '-L', 'test fs', '--max-obj-size', str(max_obj_size), - '--fatal-warnings', '--cachedir', self.cache_dir, '--quiet', + '--cachedir', self.cache_dir, '--quiet', '--authfile', '/dev/null', self.storage_url ]) if self.passphrase is None: argv.append('--plain') @@ -73,14 +73,14 @@ class TestFuse: proc.stdin.close() assert proc.wait() == 0 + self.capfd.register_output(r'^WARNING: Maximum object sizes less than ' + '1 MiB will degrade performance\.$', count=1) - def mount(self, fatal_warnings=True, expect_fail=None): + def mount(self, expect_fail=None): cmd = (self.s3ql_cmd_argv('mount.s3ql') + ["--fg", '--cachedir', self.cache_dir, '--log', 'none', '--compress', 'zlib', '--quiet', self.storage_url, self.mnt_dir, '--authfile', '/dev/null' ]) - if fatal_warnings: - cmd.append('--fatal-warnings') self.mount_process = subprocess.Popen(cmd, stdin=subprocess.PIPE, universal_newlines=True) if self.backend_login is not None: @@ -132,7 +132,7 @@ class TestFuse: proc = subprocess.Popen(self.s3ql_cmd_argv('fsck.s3ql') + [ '--force', '--quiet', '--log', 'none', '--cachedir', - self.cache_dir, '--fatal-warnings', '--authfile', + self.cache_dir, '--authfile', authinfo_fh.name, self.storage_url ], stdin=subprocess.PIPE, universal_newlines=True) proc.stdin.close() diff --git a/tests/t5_failsafe.py b/tests/t5_failsafe.py index 6a378e2..75636ed 100755 --- a/tests/t5_failsafe.py +++ b/tests/t5_failsafe.py @@ -24,6 +24,7 @@ from s3ql.backends.local import Backend as LocalBackend from s3ql.common import get_seq_no from s3ql import BUFSIZE +@pytest.mark.usefixtures('pass_capfd') class TestFailsafe(t4_fuse.TestFuse): ''' Test behavior with corrupted backend. In contrast to the tests @@ -74,11 +75,14 @@ class TestFailsafe(t4_fuse.TestFuse): self.backend.store('s3ql_data_1', val[:500] + b'oops' + val[500:], meta) # Try to read - self.mount(fatal_warnings=False) + + self.mount() with pytest.raises(IOError) as exc_info: with open(fname1, 'rb') as fh: fh.read() assert exc_info.value.errno == errno.EIO + self.capfd.register_output(r'^ERROR: Backend returned malformed data for ' + 'block 0 of inode \d+ .+$', count=1) # This should still work with open(fname2, 'rb') as fh: @@ -87,8 +91,11 @@ class TestFailsafe(t4_fuse.TestFuse): # But this should not with pytest.raises(PermissionError): open(fname2, 'wb') + self.capfd.register_output(r'^ERROR: Backend returned malformed data for ' + 'block 0 of inode \d+ .+$', count=1) +@pytest.mark.usefixtures('pass_capfd') class TestNewerMetadata(t4_fuse.TestFuse): ''' Make sure that we turn on failsafe mode and don't overwrite @@ -105,7 +112,7 @@ class TestNewerMetadata(t4_fuse.TestFuse): meta = plain_backend['s3ql_metadata'] # Mount file system - self.mount(fatal_warnings=False) + self.mount() # Increase sequence number seq_no = get_seq_no(plain_backend) @@ -126,7 +133,13 @@ class TestNewerMetadata(t4_fuse.TestFuse): with open(fname + 'barz', 'w') as fh: fh.write('foobar') time.sleep(1) - + self.capfd.register_output(r'^ERROR: Remote metadata is newer than local ' + '\(\d+ vs \d+\), refusing to overwrite(?: and switching ' + 'to failsafe mode)?!$', count=2) + self.capfd.register_output(r'^WARNING: File system errors encountered, marking for ' + 'fsck\.$', count=1) + self.capfd.register_output(r'^ERROR: The locally cached metadata will be ' + '\*lost\* the next .+$', count=1) self.umount() # Assert that remote metadata has not been overwritten diff --git a/tests/t5_full.py b/tests/t5_full.py index bcd60f0..e3011e2 100755 --- a/tests/t5_full.py +++ b/tests/t5_full.py @@ -85,7 +85,7 @@ class RemoteTest: super().teardown_method(method) proc = subprocess.Popen(self.s3ql_cmd_argv('s3qladm') + - [ '--quiet', '--authfile', '/dev/null', '--fatal-warnings', + [ '--quiet', '--authfile', '/dev/null', 'clear', self.storage_url ], stdin=subprocess.PIPE, universal_newlines=True) if self.backend_login is not None: diff --git a/tests/t6_upgrade.py b/tests/t6_upgrade.py index c72aff8..fd24b41 100755 --- a/tests/t6_upgrade.py +++ b/tests/t6_upgrade.py @@ -23,6 +23,7 @@ import tempfile import os import pytest +@pytest.mark.usefixtures('pass_capfd') class TestUpgrade(t4_fuse.TestFuse): def setup_method(self, method): @@ -46,7 +47,7 @@ class TestUpgrade(t4_fuse.TestFuse): def mkfs_old(self, force=False, max_obj_size=500): argv = [ os.path.join(self.basedir_old, 'bin', 'mkfs.s3ql'), '-L', 'test fs', '--max-obj-size', str(max_obj_size), - '--fatal-warnings', '--cachedir', self.cache_dir, '--quiet', + '--cachedir', self.cache_dir, '--quiet', '--authfile', '/dev/null', self.storage_url ] if force: argv.append('--force') @@ -62,6 +63,8 @@ class TestUpgrade(t4_fuse.TestFuse): print(self.passphrase, file=proc.stdin) proc.stdin.close() assert proc.wait() == 0 + self.capfd.register_output(r'^Warning: maximum object sizes less than 1 MiB ' + 'will seriously degrade performance\.$', count=1) def mount_old(self): self.mount_process = subprocess.Popen([os.path.join(self.basedir_old, 'bin', 'mount.s3ql'), @@ -96,7 +99,7 @@ class TestUpgrade(t4_fuse.TestFuse): def upgrade(self): proc = subprocess.Popen(self.s3ql_cmd_argv('s3qladm') + - [ '--fatal-warnings', '--cachedir', self.cache_dir, '--authfile', + [ '--cachedir', self.cache_dir, '--authfile', '/dev/null', '--quiet', 'upgrade', self.storage_url ], stdin=subprocess.PIPE, universal_newlines=True) @@ -138,6 +141,8 @@ class TestUpgrade(t4_fuse.TestFuse): shutil.rmtree(self.cache_dir) self.cache_dir = tempfile.mkdtemp(prefix='s3ql-cache-') self.mount(expect_fail=32) + self.capfd.register_output(r'^ERROR: File system revision too old, please ' + 'run `s3qladm upgrade` first\.$', count=1) # Upgrade if not with_cache: @@ -183,7 +188,7 @@ class RemoteUpgradeTest: super().teardown_method(method) proc = subprocess.Popen(self.s3ql_cmd_argv('s3qladm') + - [ '--quiet', '--authfile', '/dev/null', '--fatal-warnings', + [ '--quiet', '--authfile', '/dev/null', 'clear', self.storage_url ], stdin=subprocess.PIPE, universal_newlines=True) if self.backend_login is not None: |