path: root/doc/latex/manual.tex

% Generated by Sphinx.
\def\sphinxdocclass{report}
\documentclass[letterpaper,10pt,english]{sphinxmanual}
\usepackage[utf8]{inputenc}
\DeclareUnicodeCharacter{00A0}{\nobreakspace}
\usepackage[T1]{fontenc}
\usepackage{babel}
\usepackage{times}
\usepackage[Bjarne]{fncychap}
\usepackage{longtable}
\usepackage{sphinx}


\title{S3QL Documentation}
\date{December 06, 2011}
\release{1.8.1}
\author{Nikolaus Rath}
\newcommand{\sphinxlogo}{}
\renewcommand{\releasename}{Release}
\makeindex

\makeatletter
\def\PYG@reset{\let\PYG@it=\relax \let\PYG@bf=\relax%
    \let\PYG@ul=\relax \let\PYG@tc=\relax%
    \let\PYG@bc=\relax \let\PYG@ff=\relax}
\def\PYG@tok#1{\csname PYG@tok@#1\endcsname}
\def\PYG@toks#1+{\ifx\relax#1\empty\else%
    \PYG@tok{#1}\expandafter\PYG@toks\fi}
\def\PYG@do#1{\PYG@bc{\PYG@tc{\PYG@ul{%
    \PYG@it{\PYG@bf{\PYG@ff{#1}}}}}}}
\def\PYG#1#2{\PYG@reset\PYG@toks#1+\relax+\PYG@do{#2}}

\def\PYG@tok@gd{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}}
\def\PYG@tok@gu{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}}
\def\PYG@tok@gt{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.25,0.82}{##1}}}
\def\PYG@tok@gs{\let\PYG@bf=\textbf}
\def\PYG@tok@gr{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}}
\def\PYG@tok@cm{\def\PYG@tc##1{\textcolor[rgb]{0.50,0.50,0.50}{##1}}}
\def\PYG@tok@vg{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.82,0.44,0.00}{##1}}}
\def\PYG@tok@m{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.38,0.00,0.88}{##1}}}
\def\PYG@tok@mh{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.31,0.50}{##1}}}
\def\PYG@tok@cs{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.80,0.00,0.00}{##1}}}
\def\PYG@tok@ge{\let\PYG@it=\textit}
\def\PYG@tok@vc{\def\PYG@tc##1{\textcolor[rgb]{0.19,0.38,0.56}{##1}}}
\def\PYG@tok@il{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.82}{##1}}}
\def\PYG@tok@go{\def\PYG@tc##1{\textcolor[rgb]{0.50,0.50,0.50}{##1}}}
\def\PYG@tok@cp{\def\PYG@tc##1{\textcolor[rgb]{0.31,0.44,0.56}{##1}}}
\def\PYG@tok@gi{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}}
\def\PYG@tok@gh{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}}
\def\PYG@tok@ni{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.00}{##1}}}
\def\PYG@tok@nl{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.56,0.44,0.00}{##1}}}
\def\PYG@tok@nn{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}}
\def\PYG@tok@no{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.19,0.38}{##1}}}
\def\PYG@tok@na{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.75}{##1}}}
\def\PYG@tok@nb{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
\def\PYG@tok@nc{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.69,0.00,0.38}{##1}}}
\def\PYG@tok@nd{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.31,0.31,0.31}{##1}}}
\def\PYG@tok@ne{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.94,0.00,0.00}{##1}}}
\def\PYG@tok@nf{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.38,0.69}{##1}}}
\def\PYG@tok@si{\def\PYG@bc##1{\colorbox[rgb]{0.88,0.88,0.88}{##1}}}
\def\PYG@tok@s2{\def\PYG@bc##1{\colorbox[rgb]{1.00,0.94,0.94}{##1}}}
\def\PYG@tok@vi{\def\PYG@tc##1{\textcolor[rgb]{0.19,0.19,0.69}{##1}}}
\def\PYG@tok@nt{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.00}{##1}}}
\def\PYG@tok@nv{\def\PYG@tc##1{\textcolor[rgb]{0.56,0.38,0.19}{##1}}}
\def\PYG@tok@s1{\def\PYG@bc##1{\colorbox[rgb]{1.00,0.94,0.94}{##1}}}
\def\PYG@tok@gp{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}}
\def\PYG@tok@sh{\def\PYG@bc##1{\colorbox[rgb]{1.00,0.94,0.94}{##1}}}
\def\PYG@tok@ow{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}}
\def\PYG@tok@sx{\def\PYG@tc##1{\textcolor[rgb]{0.82,0.13,0.00}{##1}}\def\PYG@bc##1{\colorbox[rgb]{1.00,0.94,0.94}{##1}}}
\def\PYG@tok@bp{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}}
\def\PYG@tok@c1{\def\PYG@tc##1{\textcolor[rgb]{0.50,0.50,0.50}{##1}}}
\def\PYG@tok@kc{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\def\PYG@tok@c{\def\PYG@tc##1{\textcolor[rgb]{0.50,0.50,0.50}{##1}}}
\def\PYG@tok@mf{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.38,0.00,0.88}{##1}}}
\def\PYG@tok@err{\def\PYG@tc##1{\textcolor[rgb]{0.94,0.00,0.00}{##1}}\def\PYG@bc##1{\colorbox[rgb]{0.94,0.63,0.63}{##1}}}
\def\PYG@tok@kd{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\def\PYG@tok@ss{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.38,0.00}{##1}}}
\def\PYG@tok@sr{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}\def\PYG@bc##1{\colorbox[rgb]{1.00,0.94,1.00}{##1}}}
\def\PYG@tok@mo{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.25,0.00,0.88}{##1}}}
\def\PYG@tok@mi{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.82}{##1}}}
\def\PYG@tok@kn{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\def\PYG@tok@o{\def\PYG@tc##1{\textcolor[rgb]{0.19,0.19,0.19}{##1}}}
\def\PYG@tok@kr{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\def\PYG@tok@s{\def\PYG@bc##1{\colorbox[rgb]{1.00,0.94,0.94}{##1}}}
\def\PYG@tok@kp{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.19,0.50}{##1}}}
\def\PYG@tok@w{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}}
\def\PYG@tok@kt{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.19,0.19,0.56}{##1}}}
\def\PYG@tok@sc{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.25,0.82}{##1}}}
\def\PYG@tok@sb{\def\PYG@bc##1{\colorbox[rgb]{1.00,0.94,0.94}{##1}}}
\def\PYG@tok@k{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\def\PYG@tok@se{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.38,0.38,0.38}{##1}}\def\PYG@bc##1{\colorbox[rgb]{1.00,0.94,0.94}{##1}}}
\def\PYG@tok@sd{\def\PYG@tc##1{\textcolor[rgb]{0.82,0.25,0.13}{##1}}}

\def\PYGZbs{\char`\\}
\def\PYGZus{\char`\_}
\def\PYGZob{\char`\{}
\def\PYGZcb{\char`\}}
\def\PYGZca{\char`\^}
\def\PYGZsh{\char`\#}
\def\PYGZpc{\char`\%}
\def\PYGZdl{\char`\$}
\def\PYGZti{\char`\~}
% for compatibility with earlier versions
\def\PYGZat{@}
\def\PYGZlb{[}
\def\PYGZrb{]}
\makeatother

\begin{document}

\maketitle
\tableofcontents
\phantomsection\label{index::doc}



\chapter{About S3QL}
\label{about:about-s3ql}\label{about::doc}\label{about:s3ql-user-s-guide}
S3QL is a file system that stores all its data online using storage
services like \href{http://code.google.com/apis/storage/}{Google Storage}, \href{http://aws.amazon.com/s3AmazonS3}{Amazon S3} or \href{http://openstack.org/projects/storage/}{OpenStack}. S3QL effectively provides
a hard disk of dynamic, infinite capacity that can be accessed from
any computer with internet access.

S3QL is a standard conforming, full featured UNIX file system that is
conceptually indistinguishable from any local file system.
Furthermore, S3QL has additional features like compression,
encryption, data de-duplication, immutable trees and snapshotting
which make it especially suitable for online backup and archival.

S3QL is designed to favor simplicity and elegance over performance and
feature-creep. Care has been taken to make the source code as
readable and serviceable as possible. Solid error detection and error
handling have been included from the very first line, and S3QL comes
with extensive automated test cases for all its components.


\section{Features}
\label{about:features}\begin{itemize}
\item {} 
\textbf{Transparency.} Conceptually, S3QL is indistinguishable from a
local file system. For example, it supports hardlinks, symlinks,
ACLs and standard unix permissions, extended attributes and file
sizes up to 2 TB.

\item {} 
\textbf{Dynamic Size.} The size of an S3QL file system grows and shrinks
dynamically as required.

\item {} 
\textbf{Compression.} Before storage, all data may compressed with the
LZMA, bzip2 or deflate (gzip) algorithm.

\item {} 
\textbf{Encryption.} After compression (but before upload), all data can
AES encrypted with a 256 bit key. An additional SHA256 HMAC checksum
is used to protect the data against manipulation.

\item {} 
\textbf{Data De-duplication.} If several files have identical contents,
the redundant data will be stored only once. This works across all
files stored in the file system, and also if only some parts of the
files are identical while other parts differ.

\item {} 
\textbf{Immutable Trees.} Directory trees can be made immutable, so that
their contents can no longer be changed in any way whatsoever. This
can be used to ensure that backups can not be modified after they
have been made.

\item {} 
\textbf{Copy-on-Write/Snapshotting.} S3QL can replicate entire directory
trees without using any additional storage space. Only if one of the
copies is modified, the part of the data that has been modified will
take up additional storage space. This can be used to create
intelligent snapshots that preserve the state of a directory at
different points in time using a minimum amount of space.

\item {} 
\textbf{High Performance independent of network latency.} All operations
that do not write or read file contents (like creating directories
or moving, renaming, and changing permissions of files and
directories) are very fast because they are carried out without any
network transactions.

S3QL achieves this by saving the entire file and directory structure
in a database. This database is locally cached and the remote
copy updated asynchronously.

\item {} 
\textbf{Support for low bandwidth connections.} S3QL splits file contents
into smaller blocks and caches blocks locally. This minimizes both
the number of network transactions required for reading and writing
data, and the amount of data that has to be transferred when only
parts of a file are read or written.

\end{itemize}


\section{Development Status}
\label{about:development-status}
After two years of beta-testing by about 93 users did not reveal any
data-critical bugs, S3QL was declared \textbf{stable} with the release of
version 1.0 on May 13th, 2011. Note that this does not mean that S3QL
is bug-free. S3QL still has several known, and probably many more
unknown bugs. However, there is a high probability that these bugs
will, although being inconvenient, not endanger any stored data.

Please report any problems on the \href{http://groups.google.com/group/s3ql}{mailing list} or the \href{http://code.google.com/p/s3ql/issues/list}{issue tracker}.


\chapter{Installation}
\label{installation:installation}\label{installation::doc}
S3QL depends on several other programs and libraries that have to be
installed first. The best method to satisfy these dependencies depends
on your distribution. In some cases S3QL and all its dependencies can
be installed with as little as three commands, while in other cases more work
may be required.

The \href{http://code.google.com/p/s3ql/w/list}{S3QL Wiki} contains
installation instructions for quite a few different Linux
distributions. You should only use the generic instructions in this
manual if your distribution is not included in the \href{http://code.google.com/p/s3ql/w/list?q=label:Installation}{distribution-specific
installation instructions} on the wiki.


\section{Dependencies}
\label{installation:dependencies}
The following is a list of the programs and libraries required for
running S3QL. Generally, you should first check if your distribution
already provides a suitable packages and only install from source if
that is not the case.
\begin{itemize}
\item {} 
Kernel: Linux 2.6.9 or newer or FreeBSD with \href{http://www.freshports.org/sysutils/fusefs-kmod/}{FUSE4BSD}. Starting with
kernel 2.6.26 you will get significantly better write performance,
so under Linux you should actually use \emph{2.6.26 or newer whenever
possible}.

\item {} 
\href{http://www.python.org/}{Python}, version 2.6.6 or newer, but not
Python 3.x. Make sure to also install the development headers.

\item {} 
The \href{http://pypi.python.org/pypi/pycryptopp}{PyCrypto++ Python Module}. To check if this module
is installed, try to execute \code{python -c 'import pycryptopp'}.

\item {} 
The \href{http://pypi.python.org/pypi/argparse}{argparse Python Module}. To check if this module is
installed, try to execute \code{python -c 'import argparse; print
argparse.\_\_version\_\_'}. If argparse is installed, this will print
the version number. You need version 1.1 or later.

\item {} 
\href{http://www.sqlite.org/}{SQLite} version 3.7.0 or newer. SQLite
has to be installed as a \emph{shared library} with development headers.

\item {} 
The \href{http://code.google.com/p/apsw/}{APSW Python Module}. To check
which (if any) version of APWS is installed, run the command

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{python -c 'import apsw; print apsw.apswversion()'}
\end{Verbatim}

The printed version number should be at least 3.7.0. Note that APSW
must be linked \emph{dynamically} against SQLite, so you can \emph{not} use
the Ubuntu PPA at
\href{https://launchpad.net/~ubuntu-rogerbinns/+archive/apsw}{https://launchpad.net/\textasciitilde{}ubuntu-rogerbinns/+archive/apsw} (these
packages are statically linked).

\item {} 
The \href{http://pypi.python.org/pypi/pyliblzma}{PyLibLZMA Python module}. To check if this module
is installed, execute \code{python -c 'import lzma; print
lzma.\_\_version\_\_'}. This should print a version number. You need at
least version 0.5.3.

\item {} 
The \href{http://code.google.com/p/python-llfuse/}{Python LLFUSE module}. To check if this module
is installed, execute \code{python -c 'import llfuse; print
llfuse.\_\_version\_\_'}. This should print a version number. You need at
least version 0.37.

Note that early S3QL versions shipped with a built-in version of
this module. If you are upgrading from such a version, make sure to
completely remove the old S3QL version first.

\end{itemize}


\section{Installing S3QL}
\label{installation:inst-s3ql}\label{installation:installing-s3ql}
To install S3QL itself, proceed as follows:
\begin{enumerate}
\item {} 
Download S3QL from \href{http://code.google.com/p/s3ql/downloads/list}{http://code.google.com/p/s3ql/downloads/list}

\item {} 
Unpack it into a folder of your choice

\item {} 
Run \code{python setup.py build} to build S3QL.

\item {} 
Run \code{python setup.py test} to run a self-test. If this fails, ask
for help on the \href{http://groups.google.com/group/s3ql}{mailing list} or report a bug in the
\href{http://code.google.com/p/s3ql/issues/list}{issue tracker}.

\end{enumerate}

Now you have three options:
\begin{itemize}
\item {} 
You can run the S3QL commands from the \code{bin/} directory.

\item {} 
You can install S3QL system-wide for all users. To do that, you
have to run \code{sudo python setup.py install}.

\item {} 
You can install S3QL into \code{\textasciitilde{}/.local} by executing \code{python
setup.py install -{-}user}. In this case you should make sure that
\code{\textasciitilde{}/.local/bin} is in your \code{\$PATH} environment variable.

\end{itemize}


\chapter{General Information}
\label{general:general-information}\label{general::doc}

\section{Terminology}
\label{general:terminology}
S3QL can store data at different service providers and using different
protocols. The term \emph{backend} refers to both the part of S3QL that
implements communication with a specific storage service and the
storage service itself. Most backends can hold more than one S3QL file
system and thus require some additional information that specifies the
file system location within the backend. This location is called a
\emph{bucket} (for historical reasons).

Many S3QL commands expect a \emph{storage url} as a parameter. A storage
url specifies both the backend and the bucket and thus uniquely
identifies an S3QL file system. The form of the storage url depends on
the backend and is described together with the
{\hyperref[backends:storage-backends]{\emph{Storage Backends}}}.


\section{Storing Authentication Information}
\label{general:storing-authentication-information}\label{general:bucket-pw}
Normally, S3QL reads username and password for the backend as well as
an encryption passphrase for the bucket from the terminal. Most
commands also accept an \code{-{-}authfile} parameter that can be
used to read this information from a file instead.

The authentication file consists of sections, led by a \code{{[}section{]}}
header and followed by \code{name: value} entries. The section headers
themselves are not used by S3QL but have to be unique within the file.

In each section, the following entries can be defined:
\begin{quote}\begin{description}
\item[{storage-url}] \leavevmode
Specifies the storage url to which this section applies. If a
storage url starts with the value of this entry, the section is
considered applicable.

\item[{backend-login}] \leavevmode
Specifies the username to use for authentication with the backend.

\item[{backend-password}] \leavevmode
Specifies the password to use for authentication with the backend.

\item[{bucket-passphrase}] \leavevmode
Specifies the passphrase to use to decrypt the bucket (if it is
encrypted).

\end{description}\end{quote}

When reading the authentication file, S3QL considers every applicable
section in order and uses the last value that it found for each entry.
For example, consider the following authentication file:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{g+ge}{[s3]}
\PYG{l}{storage-url: s3://}
\PYG{l}{backend-login: joe}
\PYG{l}{backend-password: notquitesecret}

\PYG{g+ge}{[bucket1]}
\PYG{l}{storage-url: s3://joes-first-bucket}
\PYG{l}{bucket-passphrase: neitheristhis}

\PYG{g+ge}{[bucket2]}
\PYG{l}{storage-url: s3://joes-second-bucket}
\PYG{l}{bucket-passphrase: swordfish}

\PYG{g+ge}{[bucket3]}
\PYG{l}{storage-url: s3://joes-second-bucket/with-prefix}
\PYG{l}{backend-login: bill}
\PYG{l}{backend-password: bi23ll}
\PYG{l}{bucket-passphrase: ll23bi}
\end{Verbatim}

With this authentication file, S3QL would try to log in as ``joe''
whenever the s3 backend is used, except when accessing a storage url
that begins with ``s3://joes-second-bucket/with-prefix''. In that case,
the last section becomes active and S3QL would use the ``bill''
credentials. Furthermore, bucket encryption passphrases will be used
for storage urls that start with ``s3://joes-first-bucket'' or
``s3://joes-second-bucket''.

The authentication file is parsed by the \href{http://docs.python.org/library/configparser.html}{Python ConfigParser
module}.


\section{On Backend Reliability}
\label{general:backend-reliability}\label{general:on-backend-reliability}
S3QL has been designed for use with a storage backend where data loss
is so infrequent that it can be completely neglected (e.g. the Amazon
S3 backend). If you decide to use a less reliable backend, you should
keep the following warning in mind and read this section carefully.

\begin{notice}{warning}{Warning:}
S3QL is not able to compensate for any failures of the backend. In
particular, it is not able reconstruct any data that has been lost
or corrupted by the backend. The persistence and durability of data
stored in an S3QL file system is limited and determined by the
backend alone.
\end{notice}

On the plus side, if a backend looses or corrupts some of the stored
data, S3QL \emph{will} detect the problem. Missing data will be detected
when running \code{fsck.s3ql} or when attempting to access the data in the
mounted file system. In the later case you will get an IO Error, and
on unmounting S3QL will warn you that the file system is damaged and
you need to run \code{fsck.s3ql}.

\code{fsck.s3ql} will report all the affected files and move them into the
\code{/lost+found} directory of the file system.

You should be aware that, because of S3QL's data de-duplication
feature, the consequences of a data loss in the backend can be
significantly more severe than you may expect. More concretely, a data
loss in the backend at time \emph{x} may cause data that is written \emph{after}
time \emph{x} to be lost as well. What may happen is this:
\begin{enumerate}
\item {} 
You store an important file in the S3QL file system.

\item {} 
The backend looses the data blocks of this file. As long as you
do not access the file or run \code{fsck.s3ql}, S3QL
is not aware that the data has been lost by the backend.

\item {} 
You save an additional copy of the important file in a different
location on the same S3QL file system.

\item {} 
S3QL detects that the contents of the new file are identical to the
data blocks that have been stored earlier. Since at this point S3QL
is not aware that these blocks have been lost by the backend, it
does not save another copy of the file contents in the backend but
relies on the (presumably) existing blocks instead.

\item {} 
Therefore, even though you saved another copy, you still do not
have a backup of the important file (since both copies refer to the
same data blocks that have been lost by the backend).

\end{enumerate}

As one can see, this effect becomes the less important the more often
one runs \code{fsck.s3ql}, since \code{fsck.s3ql} will make S3QL aware of any
blocks that the backend may have lost. Figuratively, this establishes
a ``checkpoint'': data loss in the backend that occurred before running
\code{fsck.s3ql} can not affect any file system operations performed after
running \code{fsck.s3ql}.

Nevertheless, the recommended way to use S3QL is in combination with a
sufficiently reliable storage backend. In that case none of the above
will ever be a concern.


\chapter{Storage Backends}
\label{backends:id1}\label{backends::doc}\label{backends:storage-backends}
The following backends are currently available in S3QL:


\section{Google Storage}
\label{backends:google-storage}
\href{http://code.google.com/apis/storage/}{Google Storage} is an online
storage service offered by Google. It is the most feature-rich service
supported by S3QL and S3QL offers the best performance when used with
the Google Storage backend.

To use the Google Storage backend, you need to have (or sign up for) a
Google account, and then \href{http://code.google.com/apis/storage/docs/signup.html}{activate Google Storage} for your
account. The account is free, you will pay only for the amount of
storage and traffic that you actually use. Once you have created the
account, make sure to \href{http://code.google.com/apis/storage/docs/reference/v1/apiversion1.html\#enabling}{activate legacy access}.

To create a Google Storage bucket, you can use e.g. the \href{https://sandbox.google.com/storage/}{Google
Storage Manager}. The
storage URL for accessing the bucket in S3QL is then

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{gs://}\PYG{n+nv}{\textless{}bucketname\textgreater{}}\PYG{l}{/}\PYG{n+nv}{\textless{}prefix\textgreater{}}
\end{Verbatim}

Here \emph{bucketname} is the name of the bucket, and \emph{prefix} can be
an arbitrary prefix that will be prepended to all object names used by
S3QL. This allows you to store several S3QL file systems in the same
Google Storage bucket.

Note that the backend login and password for accessing your Google
Storage bucket are not your Google account name and password, but the
\emph{Google Storage developer access key} and \emph{Google Storage developer
secret} that you can manage with the \href{https://code.google.com/apis/console/\#:storage:legacy}{Google Storage key management
tool}.

If you would like S3QL to connect using HTTPS instead of standard
HTTP, start the storage url with \code{gss://} instead of \code{gs://}. Note
that at this point S3QL does not perform any server certificate
validation (see \href{http://code.google.com/p/s3ql/issues/detail?id=267}{issue 267}).


\section{Amazon S3}
\label{backends:amazon-s3}
\href{http://aws.amazon.com/s3}{Amazon S3} is the online storage service
offered by \href{http://aws.amazon.com/}{Amazon Web Services (AWS)}. To
use the S3 backend, you first need to sign up for an AWS account. The
account is free, you will pay only for the amount of storage and
traffic that you actually use. After that, you need to create a bucket
that will hold the S3QL file system, e.g. using the \href{https://console.aws.amazon.com/s3/home}{AWS Management
Console}. For best
performance, it is recommend to create the bucket in the
geographically closest storage region, but not the US Standard
region (see below).

The storage URL for accessing S3 buckets in S3QL has the form

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3://}\PYG{n+nv}{\textless{}bucketname\textgreater{}}\PYG{l}{/}\PYG{n+nv}{\textless{}prefix\textgreater{}}
\end{Verbatim}

Here \emph{bucketname} is the name of the bucket, and \emph{prefix} can be
an arbitrary prefix that will be prepended to all object names used by
S3QL. This allows you to store several S3QL file systems in the same
S3 bucket.

Note that the backend login and password for accessing S3 are not the
user id and password that you use to log into the Amazon Webpage, but
the \emph{AWS access key id} and \emph{AWS secret access key} shown under \href{https://aws-portal.amazon.com/gp/aws/developer/account/index.html?ie=UTF8\&action=access-key}{My
Account/Access Identifiers}.

If you would like S3QL to connect using HTTPS instead of standard
HTTP, start the storage url with \code{s3s://} instead of \code{s3://}. Note
that, as of May 2011, Amazon S3 is faster when accessed using a
standard HTTP connection, and that S3QL does not perform any server
certificate validation (see \href{http://code.google.com/p/s3ql/issues/detail?id=267}{issue 267}).


\subsection{Reduced Redundancy Storage (RRS)}
\label{backends:reduced-redundancy-storage-rrs}
S3QL does not allow the use of \href{http://aws.amazon.com/s3/\#protecting}{reduced redundancy storage}. The reason for that is a
combination of three factors:
\begin{itemize}
\item {} 
RRS has a relatively low reliability, on average you loose one
out of every ten-thousand objects a year. So you can expect to
occasionally loose some data.

\item {} 
When \code{fsck.s3ql} asks S3 for a list of the stored objects, this list
includes even those objects that have been lost. Therefore
\code{fsck.s3ql} \emph{can not detect lost objects} and lost data will only
become apparent when you try to actually read from a file whose data
has been lost. This is a (very unfortunate) peculiarity of Amazon
S3.

\item {} 
Due to the data de-duplication feature of S3QL, unnoticed lost
objects may cause subsequent data loss later in time (see
{\hyperref[general:backend-reliability]{\emph{On Backend Reliability}}} for details).

\end{itemize}


\subsection{Potential issues when using the US Standard storage region}
\label{backends:potential-issues-when-using-the-us-standard-storage-region}
In the US Standard storage region, Amazon S3 does not guarantee read
after create consistency. This means that after a new object has been
stored, requests to read this object may still fail for a little
while. While the file system is mounted, S3QL is able to automatically
handle all issues related to this so-called eventual consistency.
However, problems may arise during the mount process and when the file
system is checked:

Suppose that you mount the file system, store some new data, delete
some old data and unmount it again. Now there is no guarantee that
these changes will be visible immediately. At least in theory it is
therefore possible that if you mount the file system again, S3QL
does not see any of the changes that you have done and presents you
an ``old version'' of the file system without them. Even worse, if you
notice the problem and unmount the file system, S3QL will upload the
old status (which S3QL necessarily has to consider as current) and
thereby permanently override the newer version (even though this
change may not become immediately visible either).

The same problem applies when checking the file system. If S3
provides S3QL with only partially updated data, S3QL has no way to
find out if this a real consistency problem that needs to be fixed or
if it is only a temporary problem that will resolve itself
automatically (because there are still changes that have not become
visible yet).

The likelihood of this to happen is rather low. In practice, most
objects are ready for retrieval just a few seconds after they have
been stored, so to trigger this problem one would have to unmount and
remount the file system in a very short time window. However, since S3
does not place any upper limit on the length of this window, it is
recommended to not place S3QL buckets in the US Standard storage
region. As of May 2011, all other storage regions provide stronger
consistency guarantees that completely eliminate any of the described
problems.


\section{S3 compatible}
\label{backends:s3-compatible}
S3QL is also able to access other, S3 compatible storage services like
\href{http://www.openstack.org/}{OpenStack} for which no specific backend exists. Note that when
accessing such services, only the lowest common denominator of
available features can be used, so it is generally recommended to use
a service specific backend instead.

The storage URL for accessing an arbitrary S3 compatible storage
service is

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3c://}\PYG{n+nv}{\textless{}hostname\textgreater{}}\PYG{l}{:}\PYG{n+nv}{\textless{}port\textgreater{}}\PYG{l}{/}\PYG{n+nv}{\textless{}bucketname\textgreater{}}\PYG{l}{/}\PYG{n+nv}{\textless{}prefix\textgreater{}}
\end{Verbatim}

or

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3cs://}\PYG{n+nv}{\textless{}hostname\textgreater{}}\PYG{l}{:}\PYG{n+nv}{\textless{}port\textgreater{}}\PYG{l}{/}\PYG{n+nv}{\textless{}bucketname\textgreater{}}\PYG{l}{/}\PYG{n+nv}{\textless{}prefix\textgreater{}}
\end{Verbatim}

to use HTTPS connections. Note, however, that at this point S3QL does
not verify the server certificate (cf. \href{http://code.google.com/p/s3ql/issues/detail?id=267}{issue 267}).


\section{Local}
\label{backends:openstack}\label{backends:local}
S3QL is also able to store its data on the local file system. This can
be used to backup data on external media, or to access external
services that S3QL can not talk to directly (e.g., it is possible to
store data over SSH by first mounting the remote system using
\href{http://fuse.sourceforge.net/sshfs.html}{sshfs}, then using the local backend to store the data in the sshfs
mountpoint).

The storage URL for local storage is

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{local://}\PYG{n+nv}{\textless{}path\textgreater{}}
\end{Verbatim}

Note that you have to write three consecutive slashes to specify an
absolute path, e.g. \code{local:///var/archive}. Also, relative paths will
automatically be converted to absolute paths before the authentication
file is read, i.e. if you are in the \code{/home/john} directory and try to
mount \code{local://bucket}, the corresponding section in the
authentication file must match the storage url
\code{local:///home/john/bucket}.


\section{SSH/SFTP}
\label{backends:ssh-sftp}
Previous versions of S3QL included an SSH/SFTP backend. With newer
S3QL versions, it is recommended to instead combine the local backend
with \href{http://fuse.sourceforge.net/sshfs.html}{sshfs} (cf. {\hyperref[tips:ssh-tipp]{\emph{SSH Backend}}}).


\chapter{File System Creation}
\label{mkfs::doc}\label{mkfs:file-system-creation}
A S3QL file system is created with the \code{mkfs.s3ql} command. It has the
following syntax:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{mkfs.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}storage url\textgreater{}}
\end{Verbatim}

This command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default:
\code{\textasciitilde{}/.s3ql)}
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file
(default: \code{\textasciitilde{}/.s3ql/authinfo2)}
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to
get debug messages from all modules. This option can
be specified multiple times.
\item [-{-}quiet]  
be really quiet
\item [-{-}version]  
just print program version and exit
\item [-L \textless{}name\textgreater{}]  
Filesystem label
\item [-{-}max-obj-size \textless{}size\textgreater{}]  
Maximum size of storage objects in KB. Files bigger
than this will be spread over multiple objects in the
storage backend. Default: 10240 KB.
\item [-{-}plain]  
Create unencrypted file system.
\item [-{-}force]  
Overwrite any existing data.
\end{optionlist}
\end{quote}

Unless you have specified the \code{-{-}plain} option, \code{mkfs.s3ql} will ask
you to enter an encryption password. This password will \emph{not} be read
from an authentication file specified with the \code{-{-}authfile}
option to prevent accidental creation of an encrypted bucket.


\chapter{Managing Buckets}
\label{adm::doc}\label{adm:managing-buckets}
The \code{s3qladm} command performs various operations on S3QL buckets.
The file system contained in the bucket \emph{must not be mounted} when
using \code{s3qladm} or things will go wrong badly.

The syntax is

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qladm }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}action\textgreater{}}\PYG{l}{ }\PYG{n+nv}{\textless{}storage-url\textgreater{}}
\end{Verbatim}

where \code{action} may be either of \textbf{passphrase},
\textbf{upgrade}, \textbf{clear} or \textbf{download-metadata}.

The \textbf{s3qladm} accepts the following general options, no
matter what specific action is being invoked:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get
debug messages from all modules. This option can be
specified multiple times.
\item [-{-}quiet]  
be really quiet
\item [-{-}log \textless{}target\textgreater{}]  
Write logging info into this file. File will be rotated
when it reaches 1 MB, and at most 5 old log files will be
kept. Specify \code{none} to disable logging. Default:
\code{none}
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file (default:
\code{\textasciitilde{}/.s3ql/authinfo2)}
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)}
\item [-{-}version]  
just print program version and exit
\end{optionlist}
\end{quote}

Hint: run \code{s3qladm \textless{}action\textgreater{} -{-}help} to get help on the additional
arguments that the different actions take.


\section{Changing the Passphrase}
\label{adm:changing-the-passphrase}
To change the passphrase a bucket, use the \code{s3qladm} command:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qladm passphrase  }\PYG{n+nv}{\textless{}storage url\textgreater{}}
\end{Verbatim}

The passphrase can only be changed when the bucket is not mounted.


\section{Upgrading the file system}
\label{adm:upgrading-the-file-system}
If you have installed a new version of S3QL, it may sometimes be
necessary to upgrade the file system metadata as well. Note that in
this case the file system can no longer be accessed with older
versions of S3QL after the upgrade.

During the upgrade you have to make sure that the command is not
interrupted, and that no one else tries to mount, check or upgrade the
file system at the same time.

To upgrade a file system from the previous to the current revision,
execute

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qladm upgrade }\PYG{n+nv}{\textless{}storage url\textgreater{}}
\end{Verbatim}


\section{Deleting a file system}
\label{adm:deleting-a-file-system}
A file system can be deleted with:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qladm clear }\PYG{n+nv}{\textless{}storage url\textgreater{}}
\end{Verbatim}

This physically deletes all the data and file system structures.


\section{Restoring Metadata Backups}
\label{adm:restoring-metadata-backups}
If the most-recent copy of the file system metadata has been damaged
irreparably, it is possible to restore one of the automatically
created backup copies.

The command

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qladm download-metadata }\PYG{n+nv}{\textless{}storage url\textgreater{}}
\end{Verbatim}

will give you a list of the available metadata backups and allow you
to download them. This will create two new files in the current
directory, ending in \code{.db} and \code{.params}. To actually use the
downloaded backup, you need to move these files into the \code{\textasciitilde{}/.s3ql/}
directory and run \code{fsck.s3ql}.

\begin{notice}{warning}{Warning:}
You should probably not use this functionality without having asked
for help on the mailing list first (see {\hyperref[resources:resources]{\emph{Further Resources / Getting Help}}}).
\end{notice}


\chapter{Mounting}
\label{mount:mounting}\label{mount::doc}
A S3QL file system is mounted with the \code{mount.s3ql} command. It has
the following syntax:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{mount.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}storage url\textgreater{}}\PYG{l}{ }\PYG{n+nv}{\textless{}mountpoint\textgreater{}}
\end{Verbatim}

\begin{notice}{note}{Note:}
S3QL is not a network file system like \href{http://en.wikipedia.org/wiki/Network\_File\_System\_\%28protocol\%29}{NFS}
or \href{http://en.wikipedia.org/wiki/CIFS}{CIFS}. It can only be
mounted on one computer at a time.
\end{notice}

This command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}log \textless{}target\textgreater{}]  
Write logging info into this file. File will be
rotated when it reaches 1 MB, and at most 5 old log
files will be kept. Specify \code{none} to disable
logging. Default: \code{\textasciitilde{}/.s3ql/mount.log}
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default:
\code{\textasciitilde{}/.s3ql)}
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file
(default: \code{\textasciitilde{}/.s3ql/authinfo2)}
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to
get debug messages from all modules. This option can
be specified multiple times.
\item [-{-}quiet]  
be really quiet
\item [-{-}version]  
just print program version and exit
\item [-{-}cachesize \textless{}size\textgreater{}]  
Cache size in kb (default: 102400 (100 MB)). Should be
at least 10 times the maximum object size of the
filesystem, otherwise an object may be retrieved and
written several times during a single write() or
read() operation.
\item [-{-}max-cache-entries \textless{}num\textgreater{}]  
Maximum number of entries in cache (default: 768).
Each cache entry requires one file descriptor, so if
you increase this number you have to make sure that
your process file descriptor limit (as set with
\code{ulimit -n}) is high enough (at least the number of
cache entries + 100).
\item [-{-}min-obj-size \textless{}size\textgreater{}]  
Minimum size of storage objects in KB. Files smaller
than this may be combined into groups that are stored
as single objects in the storage backend. Default: 512
KB.
\item [-{-}allow-other]  
Normally, only the user who called \code{mount.s3ql} can
access the mount point. This user then also has full
access to it, independent of individual file
permissions. If the \code{-{-}allow-other} option is
specified, other users can access the mount point as
well and individual file permissions are taken into
account for all users.
\item [-{-}allow-root]  
Like \code{-{-}allow-other}, but restrict access to the
mounting user and the root user.
\item [-{-}fg]  
Do not daemonize, stay in foreground
\item [-{-}single]  
Run in single threaded mode. If you don't understand
this, then you don't need it.
\item [-{-}upstart]  
Stay in foreground and raise SIGSTOP once mountpoint
is up.
\item [-{-}profile]  
Create profiling information. If you don't understand
this, then you don't need it.
\item [-{-}compress \textless{}name\textgreater{}]  
Compression algorithm to use when storing new data.
Allowed values: \code{lzma}, \code{bzip2}, \code{zlib}, none.
(default: \code{lzma})
\item [-{-}metadata-upload-interval \textless{}seconds\textgreater{}]  
Interval in seconds between complete metadata uploads.
Set to 0 to disable. Default: 24h.
\item [-{-}threads \textless{}no\textgreater{}]  
Number of parallel upload threads to use (default:
auto).
\item [-{-}nfs]  
Support export of S3QL file systems over NFS (default:
False)
\end{optionlist}
\end{quote}


\section{Compression Algorithms}
\label{mount:compression-algorithms}
S3QL supports three compression algorithms, LZMA, Bzip2 and zlib (with
LZMA being the default). The compression algorithm can be specified
freely whenever the file system is mounted, since it affects only the
compression of new data blocks.

Roughly speaking, LZMA is slower but achieves better compression
ratios than Bzip2, while Bzip2 in turn is slower but achieves better
compression ratios than zlib.

For maximum file system performance, the best algorithm therefore
depends on your network connection speed: the compression algorithm
should be fast enough to saturate your network connection.

To find the optimal algorithm for your system, S3QL ships with a
program called \code{benchmark.py} in the \code{contrib} directory. You should
run this program on a file that has a size that is roughly equal to
the block size of your file system and has similar contents. It will
then determine the compression speeds for the different algorithms and
the upload speeds for the specified backend and recommend the best
algorithm that is fast enough to saturate your network connection.

Obviously you should make sure that there is little other system load
when you run \code{benchmark.py} (i.e., don't compile software or encode
videos at the same time).


\section{Parallel Compression}
\label{mount:parallel-compression}
If you are running S3QL on a system with multiple cores, you might
want to set the \code{-{-}threads} value larger than one. This will
instruct S3QL to compress and encrypt several blocks at the same time.

If you want to do this in combination with using the LZMA compression
algorithm, you should keep an eye on memory usage though. Every
LZMA compression threads requires about 200 MB of RAM.

\begin{notice}{note}{Note:}
To determine the optimal compression algorithm for your network
connection when using multiple threads, you can pass the
\code{-{-}compression-threads} option to  \code{contrib/benchmark.py}.
\end{notice}


\section{Notes about Caching}
\label{mount:notes-about-caching}
S3QL maintains a local cache of the file system data to speed up
access. The cache is block based, so it is possible that only parts of
a file are in the cache.


\subsection{Maximum Number of Cache Entries}
\label{mount:maximum-number-of-cache-entries}
The maximum size of the cache can be configured with the \code{-{-}cachesize}
option. In addition to that, the maximum number of objects in the
cache is limited by the \code{-{-}max-cache-entries} option, so it is
possible that the cache does not grow up to the maximum cache size
because the maximum number of cache elements has been reached. The
reason for this limit is that each cache entry requires one open
file descriptor, and Linux distributions usually limit the total
number of file descriptors per process to about a thousand.

If you specify a value for \code{-{-}max-cache-entries}, you should therefore
make sure to also configure your system to increase the maximum number
of open file handles. This can be done temporarily with the \code{umask -n}
command. The method to permanently change this limit system-wide
depends on your distribution.


\subsection{Cache Flushing and Expiration}
\label{mount:cache-flushing-and-expiration}
S3QL flushes changed blocks in the cache to the backend whenever a block
has not been accessed for at least 10 seconds. Note that when a block is
flushed, it still remains in the cache.

Cache expiration (i.e., removal of blocks from the cache) is only done
when the maximum cache size is reached. S3QL always expires the least
recently used blocks first.


\section{Automatic Mounting}
\label{mount:automatic-mounting}
If you want to mount and umount an S3QL file system automatically at
system startup and shutdown, you should do so with one dedicated S3QL
init script for each S3QL file system.

If your system is using upstart, an appropriate job can be defined
as follows (and should be placed in \code{/etc/init/}):

\begin{Verbatim}[commandchars=\\\{\},numbers=left,firstnumber=1,stepnumber=1]
\PYG{l}{description	"S3QL Backup File System"}
\PYG{l}{author		"Nikolaus Rath }\PYG{n+nv}{\textless{}Nikolaus@rath.org\textgreater{}}\PYG{l}{"}

\PYG{l}{\PYGZsh{} This assumes that eth0 provides your internet connection}
\PYG{l}{start on (filesystem and net-device-up IFACE=eth0)}
\PYG{l}{stop on runlevel }\PYG{g+ge}{[016]}

\PYG{l}{env BUCKET="s3://my-backup-bla"}
\PYG{l}{env MOUNTPOINT="/mnt/backup"}

\PYG{l}{expect stop}

\PYG{l}{script}
\PYG{l}{    \PYGZsh{} Redirect stdout and stderr into the system log}
\PYG{l}{    DIR=\PYGZdl{}(mktemp -d)}
\PYG{l}{    mkfifo "\PYGZdl{}DIR/LOG\PYGZus{}FIFO"}
\PYG{l}{    logger -t s3ql -p local0.info }\PYG{n+nv}{\textless{} "\PYGZdl{}DIR/LOG\PYGZus{}FIFO" \&}
\PYG{n+nv}{    exec \textgreater{}}\PYG{l}{ "\PYGZdl{}DIR/LOG\PYGZus{}FIFO"}
\PYG{l}{    exec 2\textgreater{}\&1}
\PYG{l}{    rm -rf "\PYGZdl{}DIR"}

\PYG{l}{    \PYGZsh{} Check and mount file system}
\PYG{l}{    fsck.s3ql --batch "\PYGZdl{}BUCKET"}
\PYG{l}{    exec mount.s3ql --upstart "\PYGZdl{}BUCKET" "\PYGZdl{}MOUNTPOINT"}
\PYG{l}{end script}

\PYG{l}{pre-stop script}
\PYG{l}{    umount.s3ql "\PYGZdl{}MOUNTPOINT"}
\PYG{l}{end script}
\end{Verbatim}

\begin{notice}{note}{Note:}
In principle, it is also possible to automatically mount an S3QL
file system with an appropriate entry in \code{/etc/fstab}. However,
this is not recommended for several reasons:
\begin{itemize}
\item {} 
file systems mounted in \code{/etc/fstab} will be unmounted with the
\code{umount} command, so your system will not wait until all data has
been uploaded but shutdown (or restart) immediately (this is a
FUSE limitation, see \href{http://code.google.com/p/s3ql/issues/detail?id=159}{issue 159}).

\item {} 
There is no way to tell the system that mounting S3QL requires a
Python interpreter to be available, so it may attempt to run
\code{mount.s3ql} before it has mounted the volume containing the
Python interpreter.

\item {} 
There is no standard way to tell the system that internet
connection has to be up before the S3QL file system can be
mounted.

\end{itemize}
\end{notice}


\chapter{Advanced S3QL Features}
\label{special:advanced-s3ql-features}\label{special::doc}

\section{Snapshotting and Copy-on-Write}
\label{special:snapshotting-and-copy-on-write}\label{special:s3qlcp}
The command \code{s3qlcp} can be used to duplicate a directory tree without
physically copying the file contents. This is made possible by the
data de-duplication feature of S3QL.

The syntax of \code{s3qlcp} is:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlcp }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}src\textgreater{}}\PYG{l}{ }\PYG{n+nv}{\textless{}target\textgreater{}}
\end{Verbatim}

This will replicate the contents of the directory \code{\textless{}src\textgreater{}} in the
directory \code{\textless{}target\textgreater{}}. \code{\textless{}src\textgreater{}} has to be an existing directory and
\code{\textless{}target\textgreater{}} must not exist. Moreover, both directories have to be
within the same S3QL file system.

The replication will not take any additional space. Only if one of
directories is modified later on, the modified data will take
additional storage space.

\code{s3qlcp} can only be called by the user that mounted the file system
and (if the file system was mounted with \code{-{-}allow-other} or \code{-{-}allow-root})
the root user. This limitation might be removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}).

Note that:
\begin{itemize}
\item {} 
After the replication, both source and target directory will still
be completely ordinary directories. You can regard \code{\textless{}src\textgreater{}} as a
snapshot of \code{\textless{}target\textgreater{}} or vice versa. However, the most common
usage of \code{s3qlcp} is to regularly duplicate the same source
directory, say \code{documents}, to different target directories. For a
e.g. monthly replication, the target directories would typically be
named something like \code{documents\_January} for the replication in
January, \code{documents\_February} for the replication in February etc.
In this case it is clear that the target directories should be
regarded as snapshots of the source directory.

\item {} 
Exactly the same effect could be achieved by an ordinary copy
program like \code{cp -a}. However, this procedure would be orders of
magnitude slower, because \code{cp} would have to read every file
completely (so that S3QL had to fetch all the data over the network
from the backend) before writing them into the destination folder.

\end{itemize}


\subsection{Snapshotting vs Hardlinking}
\label{special:snapshotting-vs-hardlinking}
Snapshot support in S3QL is inspired by the hardlinking feature that
is offered by programs like \href{http://www.samba.org/rsync}{rsync} or
\href{http://savannah.nongnu.org/projects/storebackup}{storeBackup}.
These programs can create a hardlink instead of copying a file if an
identical file already exists in the backup. However, using hardlinks
has two large disadvantages:
\begin{itemize}
\item {} 
backups and restores always have to be made with a special program
that takes care of the hardlinking. The backup must not be touched
by any other programs (they may make changes that inadvertently
affect other hardlinked files)

\item {} 
special care needs to be taken to handle files which are already
hardlinked (the restore program needs to know that the hardlink was
not just introduced by the backup program to safe space)

\end{itemize}

S3QL snapshots do not have these problems, and they can be used with
any backup program.


\section{Getting Statistics}
\label{special:s3qlstat}\label{special:getting-statistics}
You can get more information about a mounted S3QL file system with the
\code{s3qlstat} command. It has the following syntax:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlstat }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}mountpoint\textgreater{}}
\end{Verbatim}

Probably the most interesting numbers are the total size of your data,
the total size after duplication, and the final size after
de-duplication and compression.

\code{s3qlstat} can only be called by the user that mounted the file system
and (if the file system was mounted with \code{-{-}allow-other} or \code{-{-}allow-root})
the root user. This limitation might be removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}).

For a full list of available options, run \code{s3qlstat -{-}help}.


\section{Immutable Trees}
\label{special:immutable-trees}\label{special:s3qllock}
The command \textbf{s3qllock} can be used to make a directory tree
immutable. Immutable trees can no longer be changed in any way
whatsoever. You can not add new files or directories and you can not
change or delete existing files and directories. The only way to get
rid of an immutable tree is to use the \textbf{s3qlrm} command (see
below).

For example, to make the directory tree beneath the directory
\code{2010-04-21} immutable, execute

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qllock 2010-04-21}
\end{Verbatim}

Immutability is a feature designed for backups. Traditionally, backups
have been made on external tape drives. Once a backup was made, the
tape drive was removed and locked somewhere in a shelf. This has the
great advantage that the contents of the backup are now permanently
fixed. Nothing (short of physical destruction) can change or delete
files in the backup.

In contrast, when backing up into an online storage system like S3QL,
all backups are available every time the file system is mounted.
Nothing prevents a file in an old backup from being changed again
later on. In the worst case, this may make your entire backup system
worthless. Imagine that your system gets infected by a nasty virus
that simply deletes all files it can find -- if the virus is active
while the backup file system is mounted, the virus will destroy all
your old backups as well!

Even if the possibility of a malicious virus or trojan horse is
excluded, being able to change a backup after it has been made is
generally not a good idea. A common S3QL use case is to keep the file
system mounted at all times and periodically create backups with
\textbf{rsync -a}. This allows every user to recover her files from a
backup without having to call the system administrator. However, this
also allows every user to accidentally change or delete files \emph{in} one
of the old backups.

Making a backup immutable protects you against all these problems.
Unless you happen to run into a virus that was specifically programmed
to attack S3QL file systems, backups can be neither deleted nor
changed after they have been made immutable.


\section{Fast Recursive Removal}
\label{special:s3qlrm}\label{special:fast-recursive-removal}
The \code{s3qlrm} command can be used to recursively delete files and
directories on an S3QL file system. Although \code{s3qlrm} is faster than
using e.g. \code{rm -r}, the main reason for its existence is that it
allows you to delete immutable trees as well. The syntax is rather
simple:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlrm }\PYG{n+nv}{\textless{}directory\textgreater{}}
\end{Verbatim}

Be warned that there is no additional confirmation. The directory will
be removed entirely and immediately.


\section{Runtime Configuration}
\label{special:runtime-configuration}\label{special:s3qlctrl}
The \code{s3qlctrl} can be used to control a mounted S3QL file system. Its
syntax is

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlctrl }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}action\textgreater{}}\PYG{l}{ }\PYG{n+nv}{\textless{}mountpoint\textgreater{}}\PYG{l}{ ...}
\end{Verbatim}

\code{\textless{}mountpoint\textgreater{}} must be the location of a mounted S3QL file system.
For a list of valid options, run \code{s3qlctrl -{-}help}. \code{\textless{}action\textgreater{}}
may be either of:
\begin{quote}
\begin{quote}\begin{description}
\item[{flushcache}] \leavevmode
Flush file system cache. The command blocks until the cache has
been flushed.

\item[{log}] \leavevmode
Change log level.

\item[{cachesize}] \leavevmode
Change file system cache size.

\item[{upload-meta}] \leavevmode
Trigger a metadata upload.

\end{description}\end{quote}
\end{quote}


\chapter{Unmounting}
\label{umount::doc}\label{umount:unmounting}
To unmount an S3QL file system, use the command:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{umount.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}mountpoint\textgreater{}}
\end{Verbatim}

This will block until all data has been committed to the storage
backend.

Only the user who mounted the file system with \textbf{mount.s3ql}
is able to unmount it again. If you are root and want to unmount an
S3QL file system mounted by an ordinary user, you have to use the
\textbf{fusermount -u} or \textbf{umount} command instead. Note
that these commands do not block until all data has been uploaded, so
if you use them instead of \code{umount.s3ql} then you should manually wait
for the \code{mount.s3ql} process to terminate before shutting down the
system.

The \textbf{umount.s3ql} command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug]  
activate debugging output
\item [-{-}quiet]  
be really quiet
\item [-{-}version]  
just print program version and exit
\item [-{-}lazy, -z]  
Lazy umount. Detaches the file system immediately, even if
there are still open files. The data will be uploaded in the
background once all open files have been closed.
\end{optionlist}
\end{quote}

If, for some reason, the \code{umount.sql} command does not work, the file
system can also be unmounted with \code{fusermount -u -z}. Note that this
command will return immediately and the file system may continue to
upload data in the background for a while longer.


\chapter{Checking for Errors}
\label{fsck:checking-for-errors}\label{fsck::doc}
If, for some reason, the filesystem has not been correctly unmounted,
or if you suspect that there might be errors, you should run the
\code{fsck.s3ql} utility. It has the following syntax:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{fsck.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}storage url\textgreater{}}
\end{Verbatim}

This command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}log \textless{}target\textgreater{}]  
Write logging info into this file. File will be rotated
when it reaches 1 MB, and at most 5 old log files will be
kept. Specify \code{none} to disable logging. Default:
\code{\textasciitilde{}/.s3ql/fsck.log}
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)}
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file (default:
\code{\textasciitilde{}/.s3ql/authinfo2)}
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get
debug messages from all modules. This option can be
specified multiple times.
\item [-{-}quiet]  
be really quiet
\item [-{-}version]  
just print program version and exit
\item [-{-}batch]  
If user input is required, exit without prompting.
\item [-{-}force]  
Force checking even if file system is marked clean.
\end{optionlist}
\end{quote}


\chapter{Contributed Programs}
\label{contrib:contributed-programs}\label{contrib::doc}
S3QL comes with a few contributed programs that are not part of the
core distribution (and are therefore not installed automatically by
default), but which may nevertheless be useful. These programs are in
the \code{contrib} directory of the source distribution or in
\code{/usr/share/doc/s3ql/contrib} if you installed S3QL from a package.


\section{benchmark.py}
\label{contrib:benchmark-py}
This program measures S3QL write performance, uplink bandwidth and
compression speed to determine the limiting factor. It also gives
recommendation for compression algorithm and number of upload threads
to achieve maximum performance.


\section{s3\_copy.py}
\label{contrib:s3-copy-py}
This program physically duplicates Amazon S3 bucket. It can be used to
migrate buckets to a different storage region or storage class
(standard or reduced redundancy).


\section{pcp.py}
\label{contrib:pcp-py}\label{contrib:pcp}
\code{pcp.py} is a wrapper program that starts several rsync processes to
copy directory trees in parallel. This is important because
transferring files in parallel significantly enhances performance when
copying data from an S3QL file system (see {\hyperref[tips:copy-performance]{\emph{Improving copy performance}}} for
details).

To recursively copy the directory \code{/mnt/home-backup} into
\code{/home/joe} using 8 parallel processes and preserving permissions,
you would execute

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{pcp.py -a --processes=8 /mnt/home-backup/ /home/joe}
\end{Verbatim}


\section{s3\_backup.sh}
\label{contrib:s3-backup-sh}
This is an example script that demonstrates how to set up a simple but
powerful backup solution using S3QL and \href{http://samba.org/rsync}{rsync}.

The \code{s3\_backup.sh} script automates the following steps:
\begin{enumerate}
\item {} 
Mount the file system

\item {} 
Replicate the previous backup with {\hyperref[special:s3qlcp]{\emph{s3qlcp}}}

\item {} 
Update the new copy with the data from the backup source using rsync

\item {} 
Make the new backup immutable with {\hyperref[special:s3qllock]{\emph{s3qllock}}}

\item {} 
Delete old backups that are no longer needed

\item {} 
Unmount the file system

\end{enumerate}

The backups are stored in directories of the form
\code{YYYY-MM-DD\_HH:mm:SS} and the {\hyperref[contrib:expire-backups-py]{expire\_backups.py}} command is used to
delete old backups.


\section{expire\_backups.py}
\label{contrib:expire-backups-py}
\textbf{expire\_backups.py} is a program to intelligently remove old
backups that are no longer needed.

To define what backups you want to keep for how long, you define a
number of \emph{age ranges}. \textbf{expire\_backups} ensures that you
will have at least one backup in each age range at all times. It will
keep exactly as many backups as are required for that and delete any
backups that become redundant.

Age ranges are specified by giving a list of range boundaries in terms
of backup cycles. Every time you create a new backup, the existing
backups age by one cycle.

Example: when \textbf{expire\_backups} is called with the age range
definition \code{1 3 7 14 31}, it will guarantee that you always have the
following backups available:
\begin{enumerate}
\item {} 
A backup that is 0 to 1 cycles old (i.e, the most recent backup)

\item {} 
A backup that is 1 to 3 cycles old

\item {} 
A backup that is 3 to 7 cycles old

\item {} 
A backup that is 7 to 14 cycles old

\item {} 
A backup that is 14 to 31 cycles old

\end{enumerate}

\begin{notice}{note}{Note:}
If you do backups in fixed intervals, then one cycle will be
equivalent to the backup interval. The advantage of specifying the
age ranges in terms of backup cycles rather than days or weeks is
that it allows you to gracefully handle irregular backup intervals.
Imagine that for some reason you do not turn on your computer for
one month. Now all your backups are at least a month old, and if you
had specified the above backup strategy in terms of absolute ages,
they would all be deleted! Specifying age ranges in terms of backup
cycles avoids these sort of problems.
\end{notice}

\textbf{expire\_backups} usage is simple. It requires backups to have
names of the forms \code{year-month-day\_hour:minute:seconds}
(\code{YYYY-MM-DD\_HH:mm:ss}) and works on all backups in the current
directory. So for the above backup strategy, the correct invocation
would be:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{expire\PYGZus{}backups.py 1 3 7 14 31}
\end{Verbatim}

When storing your backups on an S3QL file system, you probably want to
specify the \code{-{-}use-s3qlrm} option as well. This tells
\textbf{expire\_backups} to use the {\hyperref[special:s3qlrm]{\emph{s3qlrm}}} command to
delete directories.

\textbf{expire\_backups} uses a ``state file'' to keep track which
backups are how many cycles old (since this cannot be inferred from
the dates contained in the directory names). The standard name for
this state file is \code{.expire\_backups.dat}. If this file gets
damaged or deleted, \textbf{expire\_backups} no longer knows the ages
of the backups and refuses to work. In this case you can use the
\code{-{-}reconstruct-state} option to try to reconstruct the state
from the backup dates. However, the accuracy of this reconstruction
depends strongly on how rigorous you have been with making backups (it
is only completely correct if the time between subsequent backups has
always been exactly the same), so it's generally a good idea not to
tamper with the state file.

For a full list of available options, run \textbf{expire\_backups.py
--help}.


\section{s3ql.conf}
\label{contrib:s3ql-conf}
\code{s3ql.conf} is an example upstart job definition file. It defines a
job that automatically mounts an S3QL file system on system start, and
properly unmounts it when the system is shut down.


\chapter{Tips \& Tricks}
\label{tips:tips-tricks}\label{tips::doc}

\section{SSH Backend}
\label{tips:ssh-tipp}\label{tips:ssh-backend}
By combining S3QL's local backend with \href{http://fuse.sourceforge.net/sshfs.html}{sshfs}, it is possible to store an
S3QL file system on arbitrary SSH servers: first mount the remote
target directory into the local filesystem,

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{sshfs user@my.server.com:/mnt/s3ql /mnt/sshfs}
\end{Verbatim}

and then give the mountpoint to S3QL as a local destination:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{mount.s3ql local:///mnt/sshfs/mybucket /mnt/s3ql}
\end{Verbatim}


\section{Permanently mounted backup file system}
\label{tips:permanently-mounted-backup-file-system}
If you use S3QL as a backup file system, it can be useful to mount the
file system permanently (rather than just mounting it for a backup and
unmounting it afterwards). Especially if your file system becomes
large, this saves you long mount- and unmount times if you only want
to restore a single file.

If you decide to do so, you should make sure to
\begin{itemize}
\item {} 
Use {\hyperref[special:s3qllock]{\emph{s3qllock}}} to ensure that backups are immutable
after they have been made.

\item {} 
Call {\hyperref[special:s3qlctrl]{\emph{s3qlctrl upload-meta}}} right after a every
backup to make sure that the newest metadata is stored safely (if
you do backups often enough, this may also allow you to set the
\code{-{-}metadata-upload-interval} option of \textbf{mount.s3ql}
to zero).

\end{itemize}


\section{Improving copy performance}
\label{tips:copy-performance}\label{tips:improving-copy-performance}
\begin{notice}{note}{Note:}
The following applies only when copying data \textbf{from} an S3QL file
system, \textbf{not} when copying data \textbf{to} an S3QL file system.
\end{notice}

If you want to copy a lot of smaller files \emph{from} an S3QL file system
(e.g. for a system restore) you will probably notice that the
performance is rather bad.

The reason for this is intrinsic to the way S3QL works. Whenever you
read a file, S3QL first has to retrieve this file over the network
from the storage backend. This takes a minimum amount of time (the
network latency), no matter how big or small the file is. So when you
copy lots of small files, 99\% of the time is actually spend waiting
for network data.

Theoretically, this problem is easy to solve: you just have to copy
several files at the same time. In practice, however, almost all unix
utilities (\code{cp}, \code{rsync}, \code{tar} and friends) insist on copying
data one file at a time. This makes a lot of sense when copying data
on the local hard disk, but in case of S3QL this is really
unfortunate.

The best workaround that has been found so far is to copy files by
starting several rsync processes at once and use exclusion rules to
make sure that they work on different sets of files.

For example, the following script will start 3 rsync instances. The
first instance handles all filenames starting with a-f, the second the
filenames from g-l and the third covers the rest. The \code{+ */} rule
ensures that every instance looks into all directories.

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{c}{\PYGZsh{}!/bin/bash}

\PYG{l}{RSYNC\PYGZus{}ARGS="-aHv /mnt/s3ql/ /home/restore/"}

\PYG{l}{rsync -f "+ */" -f "-! }\PYG{g+ge}{[a-f]}\PYG{l}{*" \PYGZdl{}RSYNC\PYGZus{}ARGS \&}
\PYG{l}{rsync -f "+ */" -f "-! }\PYG{g+ge}{[g-l]}\PYG{l}{*" \PYGZdl{}RSYNC\PYGZus{}ARGS \&}
\PYG{l}{rsync -f "+ */" -f "- }\PYG{g+ge}{[a-l]}\PYG{l}{*" \PYGZdl{}RSYNC\PYGZus{}ARGS \&}

\PYG{l}{wait}
\end{Verbatim}

The optimum number of parallel processes depends on your network
connection and the size of the files that you want to transfer.
However, starting about 10 processes seems to be a good compromise
that increases performance dramatically in almost all situations.

S3QL comes with a script named \code{pcp.py} in the \code{contrib} directory
that can be used to transfer files in parallel without having to write
an explicit script first. See the description of {\hyperref[contrib:pcp]{\emph{pcp.py}}} for
details.


\chapter{Known Issues}
\label{issues:known-issues}\label{issues::doc}\begin{itemize}
\item {} 
S3QL does not verify TLS/SSL server certificates, so a
man-in-the-middle attack is principally possible. See \href{http://code.google.com/p/s3ql/issues/detail?id=267}{issue 267} for more
details.

\item {} 
S3QL is rather slow when an application tries to write data in
unreasonably small chunks. If a 1 MB file is copied in chunks of 1
KB, this will take more than 10 times as long as when it's copied
with the (recommended) chunk size of 128 KB.

This is a limitation of the FUSE library (which does not yet support
write caching) which will hopefully be addressed in some future FUSE
version.

Most applications, including e.g. GNU \code{cp} and \code{rsync}, use
reasonably large buffers and are therefore not affected by this
problem and perform very efficient on S3QL file systems.

However, if you encounter unexpectedly slow performance with a
specific program, this might be due to the program using very small
write buffers. Although this is not really a bug in the program,
it might be worth to ask the program's authors for help.

\item {} 
S3QL always updates file and directory access times as if the \code{relatime}
mount option has been specified: the access time (``atime'') is only updated
if it is currently earlier than either the status change time
(``ctime'') or modification time (``mtime'').

\item {} 
S3QL directories always have an \code{st\_nlink} value of 1. This may confuse
programs that rely on directories having \code{st\_nlink} values of \emph{(2 +
number of sub directories)}.

Note that this is not a bug in S3QL. Including sub directories in
the \code{st\_nlink} value is a Unix convention, but by no means a
requirement. If an application blindly relies on this convention
being followed, then this is a bug in the application.

A prominent example are early versions of GNU find, which required
the \code{-{-}noleaf} option to work correctly on S3QL file systems. This
bug has already been fixed in recent find versions.

\item {} 
The \code{umount} and \code{fusermount -u} commands will \emph{not} block until all
data has been uploaded to the backend. (this is a FUSE limitation
that will hopefully be removed in the future, see \href{http://code.google.com/p/s3ql/issues/detail?id=159}{issue 159}). If you use
either command to unmount an S3QL file system, you have to take care
to explicitly wait for the \code{mount.s3ql} process to terminate before
you shut down or restart the system. Therefore it is generally not a
good idea to mount an S3QL file system in \code{/etc/fstab} (you should
use a dedicated init script instead).

\item {} 
S3QL relies on the backends not to run out of space. This is a given
for big storage providers like Amazon S3, but you may stumble upon
this if you store buckets e.g. on smaller servers or servies.

If there is no space left in the backend, attempts to write more
data into the S3QL file system will fail and the file system will be
in an inconsistent state and require a file system check (and you
should make sure to make space available in the backend before
running the check).

Unfortunately, there is no way to handle insufficient space in the
backend without leaving the file system inconsistent. Since
S3QL first writes data into the cache, it can no longer return an
error when it later turns out that the cache can not be committed to
the backend.

\end{itemize}


\chapter{Manpages}
\label{man/index:manpages}\label{man/index::doc}
The man pages are installed with S3QL on your system and can be viewed
with the \textbf{man} command. For reference, they are also included
here in the User's Guide.


\section{The \textbf{mkfs.s3ql} command}
\label{man/mkfs:the-mkfs-s3ql-command}\label{man/mkfs::doc}

\subsection{Synopsis}
\label{man/mkfs:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{mkfs.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}storage url\textgreater{}}
\end{Verbatim}


\subsection{Description}
\label{man/mkfs:description}
The \textbf{mkfs.s3ql} command creates a new file system in the location
specified by \emph{storage url}. The storage url depends on the backend
that is used. The S3QL User's Guide should be consulted for a
description of the available backends.

Unless you have specified the \code{-{-}plain} option, \code{mkfs.s3ql} will ask
you to enter an encryption password. This password will \emph{not} be read
from an authentication file specified with the \code{-{-}authfile}
option to prevent accidental creation of an encrypted bucket.


\subsection{Options}
\label{man/mkfs:options}
The \textbf{mkfs.s3ql} command accepts the following options.
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default:
\code{\textasciitilde{}/.s3ql)}
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file
(default: \code{\textasciitilde{}/.s3ql/authinfo2)}
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to
get debug messages from all modules. This option can
be specified multiple times.
\item [-{-}quiet]  
be really quiet
\item [-{-}version]  
just print program version and exit
\item [-L \textless{}name\textgreater{}]  
Filesystem label
\item [-{-}max-obj-size \textless{}size\textgreater{}]  
Maximum size of storage objects in KB. Files bigger
than this will be spread over multiple objects in the
storage backend. Default: 10240 KB.
\item [-{-}plain]  
Create unencrypted file system.
\item [-{-}force]  
Overwrite any existing data.
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/mkfs:exit-status}
\textbf{mkfs.s3ql} returns exit code 0 if the operation succeeded and 1 if some
error occured.


\subsection{See Also}
\label{man/mkfs:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, conventional locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{s3qladm} command}
\label{man/adm::doc}\label{man/adm:the-s3qladm-command}

\subsection{Synopsis}
\label{man/adm:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qladm }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}action\textgreater{}}\PYG{l}{ }\PYG{n+nv}{\textless{}storage url\textgreater{}}
\end{Verbatim}

where \code{action} may be either of \textbf{passphrase},
\textbf{upgrade}, \textbf{delete} or \textbf{download-metadata}.


\subsection{Description}
\label{man/adm:description}
The \textbf{s3qladm} command performs various operations on S3QL buckets.
The file system contained in the bucket \emph{must not be mounted} when
using \textbf{s3qladm} or things will go wrong badly.

The storage url depends on the backend that is used. The S3QL User's
Guide should be consulted for a description of the available backends.


\subsection{Options}
\label{man/adm:options}
The \textbf{s3qladm} command accepts the following options.
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get
debug messages from all modules. This option can be
specified multiple times.
\item [-{-}quiet]  
be really quiet
\item [-{-}log \textless{}target\textgreater{}]  
Write logging info into this file. File will be rotated
when it reaches 1 MB, and at most 5 old log files will be
kept. Specify \code{none} to disable logging. Default:
\code{none}
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file (default:
\code{\textasciitilde{}/.s3ql/authinfo2)}
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)}
\item [-{-}version]  
just print program version and exit
\end{optionlist}
\end{quote}

Hint: run \code{s3qladm \textless{}action\textgreater{} -{-}help} to get help on the additional
arguments that the different actions take.


\subsection{Actions}
\label{man/adm:actions}
The following actions may be specified:
\begin{description}
\item[{passphrase}] \leavevmode
Changes the encryption passphrase of the bucket.

\item[{upgrade}] \leavevmode
Upgrade the file system contained in the bucket to the newest revision.

\item[{delete}] \leavevmode
Delete the bucket and all its contents.

\item[{download-metadata}] \leavevmode
Interactively download backups of the file system metadata.

\end{description}


\subsection{Exit Status}
\label{man/adm:exit-status}
\textbf{s3qladm} returns exit code 0 if the operation succeeded and 1 if some
error occured.


\subsection{See Also}
\label{man/adm:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, conventional locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{mount.s3ql} command}
\label{man/mount::doc}\label{man/mount:the-mount-s3ql-command}

\subsection{Synopsis}
\label{man/mount:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{mount.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}storage url\textgreater{}}\PYG{l}{ }\PYG{n+nv}{\textless{}mount point\textgreater{}}
\end{Verbatim}


\subsection{Description}
\label{man/mount:description}
The \textbf{mount.s3ql} command mounts the S3QL file system stored in \emph{storage
url} in the directory \emph{mount point}. The storage url depends on the
backend that is used. The S3QL User's Guide should be consulted for a
description of the available backends.


\subsection{Options}
\label{man/mount:options}
The \textbf{mount.s3ql} command accepts the following options.
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}log \textless{}target\textgreater{}]  
Write logging info into this file. File will be
rotated when it reaches 1 MB, and at most 5 old log
files will be kept. Specify \code{none} to disable
logging. Default: \code{\textasciitilde{}/.s3ql/mount.log}
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default:
\code{\textasciitilde{}/.s3ql)}
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file
(default: \code{\textasciitilde{}/.s3ql/authinfo2)}
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to
get debug messages from all modules. This option can
be specified multiple times.
\item [-{-}quiet]  
be really quiet
\item [-{-}version]  
just print program version and exit
\item [-{-}cachesize \textless{}size\textgreater{}]  
Cache size in kb (default: 102400 (100 MB)). Should be
at least 10 times the maximum object size of the
filesystem, otherwise an object may be retrieved and
written several times during a single write() or
read() operation.
\item [-{-}max-cache-entries \textless{}num\textgreater{}]  
Maximum number of entries in cache (default: 768).
Each cache entry requires one file descriptor, so if
you increase this number you have to make sure that
your process file descriptor limit (as set with
\code{ulimit -n}) is high enough (at least the number of
cache entries + 100).
\item [-{-}min-obj-size \textless{}size\textgreater{}]  
Minimum size of storage objects in KB. Files smaller
than this may be combined into groups that are stored
as single objects in the storage backend. Default: 512
KB.
\item [-{-}allow-other]  
Normally, only the user who called \code{mount.s3ql} can
access the mount point. This user then also has full
access to it, independent of individual file
permissions. If the \code{-{-}allow-other} option is
specified, other users can access the mount point as
well and individual file permissions are taken into
account for all users.
\item [-{-}allow-root]  
Like \code{-{-}allow-other}, but restrict access to the
mounting user and the root user.
\item [-{-}fg]  
Do not daemonize, stay in foreground
\item [-{-}single]  
Run in single threaded mode. If you don't understand
this, then you don't need it.
\item [-{-}upstart]  
Stay in foreground and raise SIGSTOP once mountpoint
is up.
\item [-{-}profile]  
Create profiling information. If you don't understand
this, then you don't need it.
\item [-{-}compress \textless{}name\textgreater{}]  
Compression algorithm to use when storing new data.
Allowed values: \code{lzma}, \code{bzip2}, \code{zlib}, none.
(default: \code{lzma})
\item [-{-}metadata-upload-interval \textless{}seconds\textgreater{}]  
Interval in seconds between complete metadata uploads.
Set to 0 to disable. Default: 24h.
\item [-{-}threads \textless{}no\textgreater{}]  
Number of parallel upload threads to use (default:
auto).
\item [-{-}nfs]  
Support export of S3QL file systems over NFS (default:
False)
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/mount:exit-status}
\textbf{mount.s3ql} returns exit code 0 if the operation succeeded and 1 if some
error occured.


\subsection{See Also}
\label{man/mount:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, conventional locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{s3qlstat} command}
\label{man/stat:the-s3qlstat-command}\label{man/stat::doc}

\subsection{Synopsis}
\label{man/stat:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlstat }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}mountpoint\textgreater{}}
\end{Verbatim}


\subsection{Description}
\label{man/stat:description}
The \textbf{s3qlstat} command prints statistics about the S3QL file system mounted
at \code{mountpoint}.

\textbf{s3qlstat} can only be called by the user that mounted the file system
and (if the file system was mounted with \code{-{-}allow-other} or
\code{-{-}allow-root}) the root user. This limitation might be
removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}).


\subsection{Options}
\label{man/stat:options}
The \textbf{s3qlstat} command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug]  
activate debugging output
\item [-{-}quiet]  
be really quiet
\item [-{-}version]  
just print program version and exit
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/stat:exit-status}
\textbf{s3qlstat} returns exit code 0 if the operation succeeded and 1 if some
error occured.


\subsection{See Also}
\label{man/stat:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, conventional locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{s3qlctrl} command}
\label{man/ctrl:the-s3qlctrl-command}\label{man/ctrl::doc}

\subsection{Synopsis}
\label{man/ctrl:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlctrl }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}action\textgreater{}}\PYG{l}{ }\PYG{n+nv}{\textless{}mountpoint\textgreater{}}\PYG{l}{ ...}
\end{Verbatim}

where \code{action} may be either of \textbf{flushcache},
\textbf{upload-meta}, \textbf{cachesize} or
\textbf{log-metadata}.


\subsection{Description}
\label{man/ctrl:description}
The \textbf{s3qlctrl} command performs various actions on the S3QL file system mounted
in \code{mountpoint}.

\textbf{s3qlctrl} can only be called by the user that mounted the file system
and (if the file system was mounted with \code{-{-}allow-other} or
\code{-{-}allow-root}) the root user. This limitation might be
removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}).

The following actions may be specified:
\begin{description}
\item[{flushcache}] \leavevmode
Uploads all changed file data to the backend.

\item[{upload-meta}] \leavevmode
Upload metadata to the backend. All file system operations will
block while a snapshot of the metadata is prepared for upload.

\item[{cachesize}] \leavevmode
Changes the cache size of the file system. This action requires an
additional argument that specifies the new cache size in kB, so the
complete command line is:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlctrl }\PYG{g+ge}{[options]}\PYG{l}{ cachesize }\PYG{n+nv}{\textless{}mountpoint\textgreater{}}\PYG{l}{ }\PYG{n+nv}{\textless{}new-cache-size\textgreater{}}
\end{Verbatim}

\item[{log}] \leavevmode
Change the amount of information that is logged into
\code{\textasciitilde{}/.s3ql/mount.log} file. The complete syntax is:

\begin{Verbatim}[commandchars=@\[\]]
s3qlctrl @PYGZlb[]options@PYGZrb[] log @textless[]mountpoint@textgreater[] @textless[]level@textgreater[] @PYGZlb[]@textless[]module@textgreater[] @PYGZlb[]@textless[]module@textgreater[] ...@PYGZrb[]@PYGZrb[]
\end{Verbatim}

here \code{level} is the desired new log level and may be either of
\emph{debug}, \emph{info} or \emph{warn}. One or more \code{module} may only be
specified with the \emph{debug} level and allow to restrict the debug
output to just the listed modules.

\end{description}


\subsection{Options}
\label{man/ctrl:options}
The \textbf{s3qlctrl} command also accepts the following options, no matter
what specific action is being invoked:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug]  
activate debugging output
\item [-{-}quiet]  
be really quiet
\item [-{-}version]  
just print program version and exit
\end{optionlist}
\end{quote}

Hint: run \code{s3qlctrl \textless{}action\textgreater{} -{-}help} to get help on the additional
arguments that the different actions take.


\subsection{Exit Status}
\label{man/ctrl:exit-status}
\textbf{s3qlctrl} returns exit code 0 if the operation succeeded and 1 if some
error occured.


\subsection{See Also}
\label{man/ctrl:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, conventional locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{s3qlcp} command}
\label{man/cp:the-s3qlcp-command}\label{man/cp::doc}

\subsection{Synopsis}
\label{man/cp:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlcp }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}source-dir\textgreater{}}\PYG{l}{ }\PYG{n+nv}{\textless{}dest-dir\textgreater{}}
\end{Verbatim}


\subsection{Description}
\label{man/cp:description}
The \textbf{s3qlcp} command duplicates the directory tree \code{source-dir}
into \code{dest-dir} without physically copying the file contents.
Both source and destination must lie inside the same S3QL file system.

The replication will not take any additional space. Only if one of
directories is modified later on, the modified data will take
additional storage space.

\code{s3qlcp} can only be called by the user that mounted the file system
and (if the file system was mounted with \code{-{-}allow-other} or \code{-{-}allow-root})
the root user. This limitation might be removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}).

Note that:
\begin{itemize}
\item {} 
After the replication, both source and target directory will still
be completely ordinary directories. You can regard \code{\textless{}src\textgreater{}} as a
snapshot of \code{\textless{}target\textgreater{}} or vice versa. However, the most common
usage of \code{s3qlcp} is to regularly duplicate the same source
directory, say \code{documents}, to different target directories. For a
e.g. monthly replication, the target directories would typically be
named something like \code{documents\_January} for the replication in
January, \code{documents\_February} for the replication in February etc.
In this case it is clear that the target directories should be
regarded as snapshots of the source directory.

\item {} 
Exactly the same effect could be achieved by an ordinary copy
program like \code{cp -a}. However, this procedure would be orders of
magnitude slower, because \code{cp} would have to read every file
completely (so that S3QL had to fetch all the data over the network
from the backend) before writing them into the destination folder.

\end{itemize}


\subsubsection{Snapshotting vs Hardlinking}
\label{man/cp:snapshotting-vs-hardlinking}
Snapshot support in S3QL is inspired by the hardlinking feature that
is offered by programs like \href{http://www.samba.org/rsync}{rsync} or
\href{http://savannah.nongnu.org/projects/storebackup}{storeBackup}.
These programs can create a hardlink instead of copying a file if an
identical file already exists in the backup. However, using hardlinks
has two large disadvantages:
\begin{itemize}
\item {} 
backups and restores always have to be made with a special program
that takes care of the hardlinking. The backup must not be touched
by any other programs (they may make changes that inadvertently
affect other hardlinked files)

\item {} 
special care needs to be taken to handle files which are already
hardlinked (the restore program needs to know that the hardlink was
not just introduced by the backup program to safe space)

\end{itemize}

S3QL snapshots do not have these problems, and they can be used with
any backup program.


\subsection{Options}
\label{man/cp:options}
The \textbf{s3qlcp} command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug]  
activate debugging output
\item [-{-}quiet]  
be really quiet
\item [-{-}version]  
just print program version and exit
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/cp:exit-status}
\textbf{s3qlcp} returns exit code 0 if the operation succeeded and 1 if some
error occured.


\subsection{See Also}
\label{man/cp:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, conventional locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{s3qlrm} command}
\label{man/rm::doc}\label{man/rm:the-s3qlrm-command}

\subsection{Synopsis}
\label{man/rm:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qlrm }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}directory\textgreater{}}
\end{Verbatim}


\subsection{Description}
\label{man/rm:description}
The \textbf{s3qlrm} command recursively deletes files and directories on an
S3QL file system. Although \textbf{s3qlrm} is faster than using e.g.
\textbf{rm -r{}`}, the main reason for its existence is that it allows
you to delete immutable trees (which can be created with
\textbf{s3qllock}) as well.

Be warned that there is no additional confirmation. The directory will
be removed entirely and immediately.

\textbf{s3qlrm} can only be called by the user that mounted the file system
and (if the file system was mounted with \code{-{-}allow-other} or
\code{-{-}allow-root}) the root user. This limitation might be
removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}).


\subsection{Options}
\label{man/rm:options}
The \textbf{s3qlrm} command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug]  
activate debugging output
\item [-{-}quiet]  
be really quiet
\item [-{-}version]  
just print program version and exit
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/rm:exit-status}
\textbf{s3qlrm} returns exit code 0 if the operation succeeded and 1 if some
error occured.


\subsection{See Also}
\label{man/rm:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, conventional locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{s3qllock} command}
\label{man/lock:the-s3qllock-command}\label{man/lock::doc}

\subsection{Synopsis}
\label{man/lock:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{s3qllock }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}directory\textgreater{}}
\end{Verbatim}


\subsection{Description}
\label{man/lock:description}
The \textbf{s3qllock} command makes a directory tree in an S3QL file
system immutable. Immutable trees can no longer be changed in any way
whatsoever. You can not add new files or directories and you can not
change or delete existing files and directories. The only way to get
rid of an immutable tree is to use the \textbf{s3qlrm} command.

\textbf{s3qllock} can only be called by the user that mounted the file system
and (if the file system was mounted with \code{-{-}allow-other} or
\code{-{-}allow-root}) the root user. This limitation might be
removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}).


\subsection{Rationale}
\label{man/lock:rationale}
Immutability is a feature designed for backups. Traditionally, backups
have been made on external tape drives. Once a backup was made, the
tape drive was removed and locked somewhere in a shelf. This has the
great advantage that the contents of the backup are now permanently
fixed. Nothing (short of physical destruction) can change or delete
files in the backup.

In contrast, when backing up into an online storage system like S3QL,
all backups are available every time the file system is mounted.
Nothing prevents a file in an old backup from being changed again
later on. In the worst case, this may make your entire backup system
worthless. Imagine that your system gets infected by a nasty virus
that simply deletes all files it can find -- if the virus is active
while the backup file system is mounted, the virus will destroy all
your old backups as well!

Even if the possibility of a malicious virus or trojan horse is
excluded, being able to change a backup after it has been made is
generally not a good idea. A common S3QL use case is to keep the file
system mounted at all times and periodically create backups with
\textbf{rsync -a}. This allows every user to recover her files from a
backup without having to call the system administrator. However, this
also allows every user to accidentally change or delete files \emph{in} one
of the old backups.

Making a backup immutable protects you against all these problems.
Unless you happen to run into a virus that was specifically programmed
to attack S3QL file systems, backups can be neither deleted nor
changed after they have been made immutable.


\subsection{Options}
\label{man/lock:options}
The \textbf{s3qllock} command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug]  
activate debugging output
\item [-{-}quiet]  
be really quiet
\item [-{-}version]  
just print program version and exit
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/lock:exit-status}
\textbf{s3qllock} returns exit code 0 if the operation succeeded and 1 if some
error occured.


\subsection{See Also}
\label{man/lock:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, conventional locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{umount.s3ql} command}
\label{man/umount::doc}\label{man/umount:the-umount-s3ql-command}

\subsection{Synopsis}
\label{man/umount:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{umount.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}mount point\textgreater{}}
\end{Verbatim}


\subsection{Description}
\label{man/umount:description}
The \textbf{umount.s3ql} command unmounts the S3QL file system mounted in the
directory \emph{mount point} and blocks until all data has been uploaded to
the storage backend.

Only the user who mounted the file system with \textbf{mount.s3ql}
is able to unmount it with \textbf{umount.s3ql}. If you are root and want to
unmount an S3QL file system mounted by an ordinary user, you have to
use the \textbf{fusermount -u} or \textbf{umount} command instead.
Note that these commands do not block until all data has been
uploaded, so if you use them instead of \textbf{umount.s3ql} then
you should manually wait for the \textbf{mount.s3ql} process to
terminate before shutting down the system.


\subsection{Options}
\label{man/umount:options}
The \textbf{umount.s3ql} command accepts the following options.
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}debug]  
activate debugging output
\item [-{-}quiet]  
be really quiet
\item [-{-}version]  
just print program version and exit
\item [-{-}lazy, -z]  
Lazy umount. Detaches the file system immediately, even if
there are still open files. The data will be uploaded in the
background once all open files have been closed.
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/umount:exit-status}
\textbf{umount.s3ql} returns exit code 0 if the operation succeeded and 1 if some
error occured.


\subsection{See Also}
\label{man/umount:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, conventional locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{fsck.s3ql} command}
\label{man/fsck::doc}\label{man/fsck:the-fsck-s3ql-command}

\subsection{Synopsis}
\label{man/fsck:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{fsck.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}storage url\textgreater{}}
\end{Verbatim}


\subsection{Description}
\label{man/fsck:description}
The \textbf{mkfs.s3ql} command checks the new file system in the location
specified by \emph{storage url} for errors and attempts to repair any
problems. The storage url depends on the backend that is used. The
S3QL User's Guide should be consulted for a description of the
available backends.


\subsection{Options}
\label{man/fsck:options}
The \textbf{mkfs.s3ql} command accepts the following options.
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}log \textless{}target\textgreater{}]  
Write logging info into this file. File will be rotated
when it reaches 1 MB, and at most 5 old log files will be
kept. Specify \code{none} to disable logging. Default:
\code{\textasciitilde{}/.s3ql/fsck.log}
\item [-{-}cachedir \textless{}path\textgreater{}]  
Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)}
\item [-{-}authfile \textless{}path\textgreater{}]  
Read authentication credentials from this file (default:
\code{\textasciitilde{}/.s3ql/authinfo2)}
\item [-{-}debug \textless{}module\textgreater{}]  
activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get
debug messages from all modules. This option can be
specified multiple times.
\item [-{-}quiet]  
be really quiet
\item [-{-}version]  
just print program version and exit
\item [-{-}batch]  
If user input is required, exit without prompting.
\item [-{-}force]  
Force checking even if file system is marked clean.
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/fsck:exit-status}
\textbf{mkfs.s3ql} returns exit code 0 if the operation succeeded and 1 if some
error occured.


\subsection{See Also}
\label{man/fsck:see-also}
The S3QL homepage is at \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.

The full S3QL documentation should also be installed somewhere on your
system, conventional locations are \code{/usr/share/doc/s3ql} or
\code{/usr/local/doc/s3ql}.


\section{The \textbf{pcp} command}
\label{man/pcp:the-pcp-command}\label{man/pcp::doc}

\subsection{Synopsis}
\label{man/pcp:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{pcp }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}source\textgreater{}}\PYG{l}{ }\PYG{g+ge}{[\textless{}source\textgreater{} ...]}\PYG{l}{ }\PYG{n+nv}{\textless{}destination\textgreater{}}
\end{Verbatim}


\subsection{Description}
\label{man/pcp:description}
The \textbf{pcp} command is a is a wrapper that starts several
\textbf{sync} processes to copy directory trees in parallel. This is
allows much better copying performance on file system that have
relatively high latency when retrieving individual files like S3QL.

\textbf{Note}: Using this program only improves performance when copying
\emph{from} an S3QL file system. When copying \emph{to} an S3QL file system,
using \textbf{pcp} is more likely to \emph{decrease} performance.


\subsection{Options}
\label{man/pcp:options}
The \textbf{pcp} command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}quiet]  
be really quiet
\item [-{-}debug]  
activate debugging output
\item [-{-}version]  
just print program version and exit
\item [-a]  
Pass -aHAX option to rsync.
\item [-{-}processes \textless{}no\textgreater{}]  
Number of rsync processes to use (default: 10).
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/pcp:exit-status}
\textbf{pcp} returns exit code 0 if the operation succeeded and 1 if some
error occured.


\subsection{See Also}
\label{man/pcp:see-also}
\textbf{pcp} is shipped as part of S3QL, \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.


\section{The \textbf{expire\_backups} command}
\label{man/expire_backups::doc}\label{man/expire_backups:the-expire-backups-command}

\subsection{Synopsis}
\label{man/expire_backups:synopsis}
\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{expire\PYGZus{}backups }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\textless{}age\textgreater{}}\PYG{l}{ }\PYG{g+ge}{[\textless{}age\textgreater{} ...]}
\end{Verbatim}


\subsection{Description}
\label{man/expire_backups:description}
The \textbf{expire\_backups} command intelligently remove old backups that are no
longer needed.

To define what backups you want to keep for how long, you define a
number of \emph{age ranges}. \textbf{expire\_backups} ensures that you
will have at least one backup in each age range at all times. It will
keep exactly as many backups as are required for that and delete any
backups that become redundant.

Age ranges are specified by giving a list of range boundaries in terms
of backup cycles. Every time you create a new backup, the existing
backups age by one cycle.

Example: when \textbf{expire\_backups} is called with the age range
definition \code{1 3 7 14 31}, it will guarantee that you always have the
following backups available:
\begin{enumerate}
\item {} 
A backup that is 0 to 1 cycles old (i.e, the most recent backup)

\item {} 
A backup that is 1 to 3 cycles old

\item {} 
A backup that is 3 to 7 cycles old

\item {} 
A backup that is 7 to 14 cycles old

\item {} 
A backup that is 14 to 31 cycles old

\end{enumerate}

\begin{notice}{note}{Note:}
If you do backups in fixed intervals, then one cycle will be
equivalent to the backup interval. The advantage of specifying the
age ranges in terms of backup cycles rather than days or weeks is
that it allows you to gracefully handle irregular backup intervals.
Imagine that for some reason you do not turn on your computer for
one month. Now all your backups are at least a month old, and if you
had specified the above backup strategy in terms of absolute ages,
they would all be deleted! Specifying age ranges in terms of backup
cycles avoids these sort of problems.
\end{notice}

\textbf{expire\_backups} usage is simple. It requires backups to have
names of the forms \code{year-month-day\_hour:minute:seconds}
(\code{YYYY-MM-DD\_HH:mm:ss}) and works on all backups in the current
directory. So for the above backup strategy, the correct invocation
would be:

\begin{Verbatim}[commandchars=\\\{\}]
\PYG{l}{expire\PYGZus{}backups.py 1 3 7 14 31}
\end{Verbatim}

When storing your backups on an S3QL file system, you probably want to
specify the \code{-{-}use-s3qlrm} option as well. This tells
\textbf{expire\_backups} to use the {\hyperref[special:s3qlrm]{\emph{s3qlrm}}} command to
delete directories.

\textbf{expire\_backups} uses a ``state file'' to keep track which
backups are how many cycles old (since this cannot be inferred from
the dates contained in the directory names). The standard name for
this state file is \code{.expire\_backups.dat}. If this file gets
damaged or deleted, \textbf{expire\_backups} no longer knows the ages
of the backups and refuses to work. In this case you can use the
\code{-{-}reconstruct-state} option to try to reconstruct the state
from the backup dates. However, the accuracy of this reconstruction
depends strongly on how rigorous you have been with making backups (it
is only completely correct if the time between subsequent backups has
always been exactly the same), so it's generally a good idea not to
tamper with the state file.


\subsection{Options}
\label{man/expire_backups:options}
The \textbf{expire\_backups} command accepts the following options:
\begin{quote}
\begin{optionlist}{3cm}
\item [-{-}quiet]  
be really quiet
\item [-{-}debug]  
activate debugging output
\item [-{-}version]  
just print program version and exit
\item [-{-}state \textless{}file\textgreater{}]  
File to save state information in (default:
''.expire\_backups.dat'')
\item [-n]  
Dry run. Just show which backups would be deleted.
\item [-{-}reconstruct-state]  
Try to reconstruct a missing state file from backup
dates.
\item [-{-}use-s3qlrm]  
Use \code{s3qlrm} command to delete backups.
\end{optionlist}
\end{quote}


\subsection{Exit Status}
\label{man/expire_backups:exit-status}
\textbf{expire\_backups} returns exit code 0 if the operation succeeded and 1 if some
error occured.


\subsection{See Also}
\label{man/expire_backups:see-also}
\textbf{expire\_backups} is shipped as part of S3QL, \href{http://code.google.com/p/s3ql/}{http://code.google.com/p/s3ql/}.


\chapter{Further Resources / Getting Help}
\label{resources::doc}\label{resources:further-resources-getting-help}\label{resources:resources}
If you have questions or problems with S3QL that you weren't able to
resolve with this manual, you might want to consider the following other resources:
\begin{itemize}
\item {} 
The \href{http://code.google.com/p/s3ql/w/list}{S3QL Wiki}

\item {} 
The \href{http://code.google.com/p/s3ql/wiki/FAQ}{S3QL FAQ}

\item {} 
The \href{http://groups.google.com/group/s3ql}{S3QL Mailing List}. You
can subscribe by sending a mail to
\href{mailto:s3ql+subscribe@googlegroups.com}{s3ql+subscribe@googlegroups.com}.

\end{itemize}

Please report any bugs you may encounter in the \href{http://code.google.com/p/s3ql/issues/list}{Issue Tracker}.



\renewcommand{\indexname}{Index}
\printindex
\end{document}