diff options
Diffstat (limited to 'doc/latex/manual.tex')
| -rw-r--r-- | doc/latex/manual.tex | 603 |
1 files changed, 351 insertions, 252 deletions
diff --git a/doc/latex/manual.tex b/doc/latex/manual.tex index bc792d3..bc05afc 100644 --- a/doc/latex/manual.tex +++ b/doc/latex/manual.tex @@ -3,6 +3,7 @@ \documentclass[letterpaper,10pt,english]{sphinxmanual} \usepackage[utf8]{inputenc} \DeclareUnicodeCharacter{00A0}{\nobreakspace} +\usepackage{cmap} \usepackage[T1]{fontenc} \usepackage{babel} \usepackage{times} @@ -13,8 +14,8 @@ \title{S3QL Documentation} -\date{July 12, 2013} -\release{1.16} +\date{October 19, 2013} +\release{2.5} \author{Nikolaus Rath} \newcommand{\sphinxlogo}{} \renewcommand{\releasename}{Release} @@ -31,66 +32,66 @@ \PYG@it{\PYG@bf{\PYG@ff{#1}}}}}}} \def\PYG#1#2{\PYG@reset\PYG@toks#1+\relax+\PYG@do{#2}} -\expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@gu\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}} -\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.25,0.82}{##1}}} -\expandafter\def\csname PYG@tok@gs\endcsname{\let\PYG@bf=\textbf} -\expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.50,0.50,0.50}{##1}}} -\expandafter\def\csname PYG@tok@vg\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.82,0.44,0.00}{##1}}} -\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.38,0.00,0.88}{##1}}} -\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.31,0.50}{##1}}} -\expandafter\def\csname PYG@tok@cs\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.80,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@c1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@bp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} +\expandafter\def\csname PYG@tok@no\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.40}{##1}}} +\expandafter\def\csname PYG@tok@vc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.40,0.60}{##1}}} +\expandafter\def\csname PYG@tok@err\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.67,0.67}{\strut ##1}}} \expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit} -\expandafter\def\csname PYG@tok@vc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.19,0.38,0.56}{##1}}} -\expandafter\def\csname PYG@tok@il\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.82}{##1}}} -\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.50,0.50,0.50}{##1}}} -\expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.31,0.44,0.56}{##1}}} -\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}} -\expandafter\def\csname PYG@tok@gh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}} -\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@nl\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.56,0.44,0.00}{##1}}} -\expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}} -\expandafter\def\csname PYG@tok@no\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.19,0.38}{##1}}} -\expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.75}{##1}}} -\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} -\expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.69,0.00,0.38}{##1}}} -\expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.31,0.31,0.31}{##1}}} -\expandafter\def\csname PYG@tok@ne\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.94,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.38,0.69}{##1}}} -\expandafter\def\csname PYG@tok@si\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.88,0.88,0.88}{\strut ##1}}} -\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.19,0.19,0.69}{##1}}} -\expandafter\def\csname PYG@tok@nt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.00}{##1}}} -\expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.56,0.38,0.19}{##1}}} -\expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@gp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}} +\expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} +\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@nt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@sd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.27,0.13}{##1}}} +\expandafter\def\csname PYG@tok@se\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@mo\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.27,0.00,0.93}{##1}}} \expandafter\def\csname PYG@tok@sh\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@ow\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.82,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@bp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} -\expandafter\def\csname PYG@tok@c1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.50,0.50,0.50}{##1}}} -\expandafter\def\csname PYG@tok@kc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} -\expandafter\def\csname PYG@tok@c\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.50,0.50,0.50}{##1}}} -\expandafter\def\csname PYG@tok@mf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.38,0.00,0.88}{##1}}} -\expandafter\def\csname PYG@tok@err\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.94,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.94,0.63,0.63}{\strut ##1}}} -\expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} -\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.38,0.00}{##1}}} +\expandafter\def\csname PYG@tok@si\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.93,0.93,0.93}{\strut ##1}}} +\expandafter\def\csname PYG@tok@ne\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} \expandafter\def\csname PYG@tok@sr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,1.00}{\strut ##1}}} -\expandafter\def\csname PYG@tok@mo\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.25,0.00,0.88}{##1}}} -\expandafter\def\csname PYG@tok@mi\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.82}{##1}}} -\expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} -\expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.19,0.19,0.19}{##1}}} -\expandafter\def\csname PYG@tok@kr\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} +\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.67,0.40,0.00}{##1}}} +\expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.73,0.00,0.40}{##1}}} +\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} +\expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.80}{##1}}} +\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}} +\expandafter\def\csname PYG@tok@nl\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.60,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.33,0.47,0.60}{##1}}} \expandafter\def\csname PYG@tok@s\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@kp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.19,0.50}{##1}}} +\expandafter\def\csname PYG@tok@cs\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.80,0.00,0.00}{##1}}} \expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}} -\expandafter\def\csname PYG@tok@kt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.19,0.19,0.56}{##1}}} -\expandafter\def\csname PYG@tok@sc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.25,0.82}{##1}}} -\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}} -\expandafter\def\csname PYG@tok@se\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.38,0.38,0.38}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@sd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.82,0.25,0.13}{##1}}} +\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} +\expandafter\def\csname PYG@tok@gu\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}} +\expandafter\def\csname PYG@tok@gp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}} +\expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@gs\endcsname{\let\PYG@bf=\textbf} +\expandafter\def\csname PYG@tok@c\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@sc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} +\expandafter\def\csname PYG@tok@gh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}} +\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}} +\expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.20}{##1}}} +\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@ow\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.73}{##1}}} +\expandafter\def\csname PYG@tok@vg\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.87,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@mf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@kc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@kr\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.33,0.53}{##1}}} +\expandafter\def\csname PYG@tok@mi\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} +\expandafter\def\csname PYG@tok@kp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.53}{##1}}} +\expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.33,0.33,0.33}{##1}}} +\expandafter\def\csname PYG@tok@kt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.60}{##1}}} +\expandafter\def\csname PYG@tok@il\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} \def\PYGZbs{\char`\\} \def\PYGZus{\char`\_} @@ -103,6 +104,9 @@ \def\PYGZsh{\char`\#} \def\PYGZpc{\char`\%} \def\PYGZdl{\char`\$} +\def\PYGZhy{\char`\-} +\def\PYGZsq{\char`\'} +\def\PYGZdq{\char`\"} \def\PYGZti{\char`\~} % for compatibility with earlier versions \def\PYGZat{@} @@ -119,7 +123,7 @@ \chapter{About S3QL} -\label{about:about-s3ql}\label{about::doc}\label{about:s3ql-user-s-guide} +\label{about:about-s3ql}\label{about:s3ql-user-s-guide}\label{about::doc} S3QL is a file system that stores all its data online using storage services like \href{http://code.google.com/apis/storage/}{Google Storage}, \href{http://aws.amazon.com/s3AmazonS3}{Amazon S3} or \href{http://openstack.org/projects/storage/}{OpenStack}. S3QL effectively provides a hard disk of dynamic, infinite capacity that can be accessed from @@ -241,12 +245,16 @@ so under Linux you should actually use \emph{2.6.26 or newer whenever possible}. \item {} -\href{http://www.python.org/}{Python} 2.7.0 or newer (but not Python -3.x). Make sure to also install the development headers. +\href{http://www.python.org/}{Python} 3.3.0 or newer. Make sure to also +install the development headers. + +\item {} +The \href{https://pypi.python.org/pypi/distribute}{setuptools/distribute Python Module}. To check if this +module is installed, try to execute \code{python -c 'import setuptools'}. \item {} -The \href{http://pypi.python.org/pypi/pycryptopp}{PyCrypto++ Python Module}. To check if this module -is installed, try to execute \code{python -c 'import pycryptopp'}. +The \href{https://www.dlitz.net/software/pycrypto/}{PyCrypto Python Module}. To check if this +module is installed, try to execute \code{python -c 'import Crypto'}. \item {} \href{http://www.sqlite.org/}{SQLite} version 3.7.0 or newer. SQLite @@ -257,22 +265,16 @@ The \href{http://code.google.com/p/apsw/}{APSW Python Module}. To check which (if any) version of APWS is installed, run the command \begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{python -c 'import apsw; print apsw.apswversion()'} +\PYG{l}{python \PYGZhy{}c \PYGZsq{}import apsw; print(apsw.apswversion())\PYGZsq{}} \end{Verbatim} The printed version number should be at least 3.7.0. \item {} -The \href{http://pypi.python.org/pypi/pyliblzma}{PyLibLZMA Python module}. To check if this module -is installed, execute \code{python -c 'import lzma; print -lzma.\_\_version\_\_'}. This should print a version number. You need at -least version 0.5.3. - -\item {} The \href{http://code.google.com/p/python-llfuse/}{Python LLFUSE module}. To check if this module -is installed, execute \code{python -c 'import llfuse; print -llfuse.\_\_version\_\_'}. This should print a version number. You need at -least version 0.37. +is installed, execute \code{python -c 'import llfuse; +print(llfuse.\_\_version\_\_)'}. This should print a version number. You +need at least version 0.39. \end{itemize} @@ -288,10 +290,10 @@ Download S3QL from \href{http://code.google.com/p/s3ql/downloads/list}{http://co Unpack it into a folder of your choice \item {} -Run \code{python setup.py build} to build S3QL. +Run \code{python3 setup.py build\_ext -{-}inplace} to build S3QL. \item {} -Run \code{python setup.py test} to run a self-test. If this fails, ask +Run \code{python3 runtests.py tests} to run a self-test. If this fails, ask for help on the \href{http://groups.google.com/group/s3ql}{mailing list} or report a bug in the \href{http://code.google.com/p/s3ql/issues/list}{issue tracker}. @@ -304,10 +306,10 @@ You can run the S3QL commands from the \code{bin/} directory. \item {} You can install S3QL system-wide for all users. To do that, you -have to run \code{sudo python setup.py install}. +have to run \code{sudo python3 setup.py install}. \item {} -You can install S3QL into \code{\textasciitilde{}/.local} by executing \code{python +You can install S3QL into \code{\textasciitilde{}/.local} by executing \code{python3 setup.py install -{-}user}. In this case you should make sure that \code{\textasciitilde{}/.local/bin} is in your \code{\$PATH} environment variable. @@ -317,21 +319,93 @@ setup.py install -{-}user}. In this case you should make sure that \section{Development Version} \label{installation:development-version} If you have checked out the unstable development version from the -Mercurial repository, a bit more effort is required. You need to also -have \href{http://www.cython.org/}{Cython} (0.16 or newer) and \href{http://sphinx.pocoo.org/}{Sphinx} (1.1 or newer) installed, and -the necessary commands are: +Mercurial repository, a bit more effort is required. You'll also need: +\begin{itemize} +\item {} +Version 0.17 or newer of the \href{http://www.cython.org/}{Cython} compiler. + +\item {} +Version 1.1 or newer of the \href{http://sphinx.pocoo.org/}{Sphinx} document processor. + +\item {} +The \href{http://pytest.org/}{py.test} testing tool, version 2.3.3 or newer. + +\end{itemize} + +With these additional dependencies installed, S3QL can be build and +tested with + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{python3 setup.py build\PYGZus{}cython} +\PYG{l}{python3 setup.py build\PYGZus{}ext \PYGZhy{}\PYGZhy{}inplace} +\PYG{l}{py.test tests/} +\end{Verbatim} + +Note that when building from the Mercurial repository, building and +testing is done with several additional checks. This may cause +compilation and/or tests to fail even though there are no problems +with functionality. For example, when building from the Mercurial +repository, any use of functions that are scheduled for deprecation in +future Python version will cause tests to fail. If you would rather +just check for functionality, you can delete the \code{MANIFEST.in} +file. In that case, the build system will behave as it does for a +regular release. + +The HTML and PDF documentation can be generated with + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{python3 setup.py build\PYGZus{}sphinx} +\end{Verbatim} + +and S3QL can be installed as usual with + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{python3 setup.py install }\PYG{g+ge}{[\PYGZhy{}\PYGZhy{}user]} +\end{Verbatim} + + +\section{Running tests requiring remote servers} +\label{installation:running-tests-requiring-remote-servers} +By default, the \code{runtest.py} (or \code{py.test}) script skips all tests +that require connection to a remote storage backend. If you would like +to run these tests too (which is always a good idea), you have to +create additional entries in your \code{\textasciitilde{}/.s3ql/authinfo2} file that tell +S3QL what server and credentials to use for these tests. These entries +have the following form: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{g+ge}{[\PYGZlt{}BACKEND\PYGZgt{}\PYGZhy{}test]} +\PYG{l}{backend\PYGZhy{}login: }\PYG{n+nv}{\PYGZlt{}user\PYGZgt{}} +\PYG{l}{backend\PYGZhy{}password: }\PYG{n+nv}{\PYGZlt{}password\PYGZgt{}} +\PYG{l}{test\PYGZhy{}fs: }\PYG{n+nv}{\PYGZlt{}storage\PYGZhy{}url\PYGZgt{}} +\end{Verbatim} + +Here \emph{\textless{}BACKEND\textgreater{}} specifies the backend that you want to test +(e.g. \emph{s3}, \emph{s3c}, \emph{gs}, or \emph{swift}), \emph{\textless{}user\textgreater{}} and \emph{\textless{}password\textgreater{}} are +the backend authentication credentials, and \emph{\textless{}storage-url\textgreater{}} specifies +the full storage URL that will be used for testing. \textbf{Any existing +S3QL file system in this storage URL will be destroyed during +testing}. + +For example, to run tests that need connection to a Google Storage +server, you would add something like \begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{python setup.py build\PYGZus{}cython} -\PYG{l}{python setup.py build\PYGZus{}ext --inplace} -\PYG{l}{python setup.py build\PYGZus{}sphinx} -\PYG{l}{python setup.py test} -\PYG{l}{python setup.py install} +\PYG{g+ge}{[gs\PYGZhy{}test]} +\PYG{l}{backend\PYGZhy{}login: GOOGIGWLONT238MD7HZ4} +\PYG{l}{backend\PYGZhy{}password: rmEbstjscoeunt1249oes1298gauidbs3hl} +\PYG{l}{test\PYGZhy{}fs: gs://joes\PYGZhy{}gs\PYGZhy{}bucket/s3ql\PYGZus{}tests/} \end{Verbatim} +On the next run of \code{runtest.py} (or \code{py.test} when using the +development version), the additional tests will be run. If the tests +are still skipped, you can get more information about why tests are +being skipped by passing the \code{-rs} argument to +\code{runtest.py}/\code{py.test}. + \chapter{Storage Backends} -\label{backends::doc}\label{backends:id1}\label{backends:sphinx}\label{backends:storage-backends} +\label{backends:storage-backends}\label{backends:py-test}\label{backends:id1}\label{backends::doc} S3QL supports different \emph{backends} to store data at different service providers and using different protocols. A \emph{storage url} specifies a backend together with some backend-specific information and uniquely @@ -383,7 +457,7 @@ that will hold the S3QL file system, e.g. using the \href{https://console.aws.am Console}. For best performance, it is recommend to create the bucket in the geographically closest storage region, but not the US Standard region -(see below). +(see {\hyperref[durability:durability]{\emph{Important Rules to Avoid Losing Data}}} for the reason). The storage URL for accessing S3 buckets in S3QL has the form @@ -434,35 +508,46 @@ objects may cause subsequent data loss later in time (see the cloud storage module of OpenStack. Swift/OpenStack storage is offered by many different companies. -The storage URL for the OpenStack backend has the form +There are two different storage URL for the OpenStack backend that +make use of different authentication APIs. For legacy (v1) +authentication, the storage URL is \begin{Verbatim}[commandchars=\\\{\}] \PYG{l}{swift://}\PYG{n+nv}{\PYGZlt{}hostname\PYGZgt{}}\PYG{g+ge}{[:\PYGZlt{}port\PYGZgt{}]}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}container\PYGZgt{}}\PYG{g+ge}{[/\PYGZlt{}prefix\PYGZgt{}]} \end{Verbatim} -Note that the storage container must already exist. Most OpenStack -providers offer a web frontend that you can use to create storage -containers. \emph{prefix} can be an arbitrary prefix that will be prepended -to all object names used by S3QL. This allows you to store several -S3QL file systems in the same container. +for keystore (v2) authentication, the storage URL is -The OpenStack backend always uses HTTPS connections. Note, however, -that at this point S3QL does not verify the server certificate (cf. -\href{http://code.google.com/p/s3ql/issues/detail?id=267}{issue 267}). +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{swiftks://}\PYG{n+nv}{\PYGZlt{}hostname\PYGZgt{}}\PYG{g+ge}{[:\PYGZlt{}port\PYGZgt{}]}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}region\PYGZgt{}}\PYG{l}{:}\PYG{n+nv}{\PYGZlt{}container\PYGZgt{}}\PYG{g+ge}{[/\PYGZlt{}prefix\PYGZgt{}]} +\end{Verbatim} +Note that when using keystore authentication, you can (and have to) +specify the storage region of the container as well. -\section{RackSpace CloudFiles} -\label{backends:rackspace-cloudfiles}\label{backends:swift} -\href{http://www.rackspace.com/}{Rackspace} CloudFiles uses \href{http://www.openstack.org/}{OpenStack} internally, so it is possible to -just use the OpenStack/Swift backend (see above) with -\code{auth.api.rackspacecloud.com} as the host name and your rackspace -API key as the backend passphrase. However, in this case you are -restricted to using containers in the default storage region. +In both cases, \emph{hostname} name should be the name of the +authentication server. The storage container must already exist (most +OpenStack providers offer either a web frontend or a command line tool +for creating containers). \emph{prefix} can be an arbitrary prefix that +will be prepended to all object names used by S3QL, which can be used +to store multiple S3QL file systems in the same container. -To access containers in other storage regions, there is a special -\code{rackspace} backend that uses a storage URL of the form +When using legacy authentication, the backend login and password +correspond to the OpenStack username and API Access Key. When using +keystore authentication, the backend password is your regular +OpenStack password and the backend login combines you OpenStack +username and tenant name in the form \code{\textless{}tenant\textgreater{}:\textless{}user\textgreater{}}. If no tenant +is required, the OpenStack username alone may be used as backend +login. -The storage URL for the OpenStack backend has the form + +\section{Rackspace CloudFiles} +\label{backends:swift}\label{backends:rackspace-cloudfiles} +\href{http://www.rackspace.com/}{Rackspace} CloudFiles uses \href{http://www.openstack.org/}{OpenStack} internally, so it is possible to +just use the OpenStack/Swift backend (see above) with +\code{auth.api.rackspacecloud.com} as the host name. For convenince, +there is also a special \code{rackspace} backend that uses a storage URL +of the form \begin{Verbatim}[commandchars=\\\{\}] \PYG{l}{rackspace://}\PYG{n+nv}{\PYGZlt{}region\PYGZgt{}}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}container\PYGZgt{}}\PYG{g+ge}{[/\PYGZlt{}prefix\PYGZgt{}]} @@ -473,39 +558,24 @@ region. \emph{prefix} can be an arbitrary prefix that will be prepended to all object names used by S3QL and can be used to store several S3QL file systems in the same container. -The OpenStack backend always uses HTTPS connections. Note, however, -that at this point S3QL does not verify the server certificate (cf. -\href{http://code.google.com/p/s3ql/issues/detail?id=267}{issue 267}). - You can create a storage container for S3QL using the \href{https://mycloud.rackspace.com/}{Cloud Control Panel} (click on \emph{Files} in the topmost menu bar). -\begin{notice}{warning}{Warning:} -As of January 2012, RackSpace does not give any information about -data consistency or data durability on their web page. However, -RackSpace support agents (especially in the live chat) often claim -very high guarantees. Any such statement is wrong. As of 01/2012, -RackSpace CloudFiles does \emph{not} give \emph{any} durability or +\begin{notice}{note}{Note:} +As of January 2012, Rackspace does not give any durability or consistency guarantees (see {\hyperref[durability:durability]{\emph{Important Rules to Avoid Losing Data}}} for why this is -important). Why this fact is only acknowledged RackSpace's -technical engineers, and/or not communicated to their sales agents -is not known. +important). However, Rackspace support agents seem prone to claim +very high guarantees. Unless explicitly backed by their terms of +service, any such statement should thus be viewed with +suspicion. S3QL developers have also \href{http://www.rath.org/Tales\%20from\%20the\%20Rackspace\%20Support}{repeatedly experienced} +similar issues with the credibility and competence of the Rackspace +support. \end{notice} -You should note that opinions about RackSpace differ widely among S3QL -users and developers. On one hand, people praise RackSpace for their -backing of the (open source) OpenStack project. On the other hand, -their heavily advertised ``fanatical support'' is in practice often not -only \href{http://code.google.com/p/s3ql/issues/detail?id=243\#c5}{less than helpful}, but their -support agents also seem to be \href{http://code.google.com/p/s3ql/issues/detail?id=243\#c11}{downright incompetent}. However, -there are reports that the support quality increases dramatically once -you are a customer and use the ``Live Chat'' link when you are logged -into the control panel. - \section{S3 compatible} -\label{backends:s3-compatible}\label{backends:rackspace} +\label{backends:rackspace}\label{backends:s3-compatible} The S3 compatible backend allows S3QL to access any storage service that uses the same protocol as Amazon S3. The storage URL has the form @@ -524,7 +594,7 @@ the same bucket. S3QL is also able to store its data on the local file system. This can be used to backup data on external media, or to access external services that S3QL can not talk to directly (e.g., it is possible to -store data over SSH by first mounting the remote system using {\color{red}\bfseries{}{}`sshfs{}`\_} +store data over SSH by first mounting the remote system using \href{http://fuse.sourceforge.net/sshfs.html}{sshfs} and then using the local backend to store the data in the sshfs mountpoint). @@ -544,7 +614,7 @@ storage url \code{local:///home/john/s3ql}. \chapter{Important Rules to Avoid Losing Data} -\label{durability:important-rules-to-avoid-losing-data}\label{durability::doc}\label{durability:durability} +\label{durability:durability}\label{durability:important-rules-to-avoid-losing-data}\label{durability:sshfs}\label{durability::doc} Most S3QL backends store data in distributed storage systems. These systems differ from a traditional, local hard disk in several important ways. In order to avoid losing data, this section should be @@ -621,9 +691,9 @@ your storage provider. \begin{tabulary}{\linewidth}{|L|L|} \hline -\textbf{ +\textbf{\relax Storage Provider -} & \textbf{ +} & \textbf{\relax Consistency }\\\hline @@ -740,7 +810,7 @@ rise up to hours (\href{http://forums.aws.amazon.com/message.jspa?messageID=3847 \section{Data Durability} -\label{durability:backend-reliability}\label{durability:data-durability} +\label{durability:data-durability}\label{durability:backend-reliability} The durability of a storage service a measure of the average probability of a storage object to become corrupted over time. The lower the chance of data loss, the higher the durability. Storage @@ -832,7 +902,7 @@ over long periods of time. \chapter{File System Creation} -\label{mkfs::doc}\label{mkfs:file-system-creation} +\label{mkfs:file-system-creation}\label{mkfs::doc} A S3QL file system is created with the \code{mkfs.s3ql} command. It has the following syntax: @@ -861,11 +931,13 @@ get debug messages from all modules. This option can be specified multiple times. \item [-{-}quiet] be really quiet -\item [-{-}ssl] -Always use SSL connections when connecting to remote -servers. For backends that allow only encrypted -connections, S3QL uses SSL automatically, even if this -option is not set. +\item [-{-}no-ssl] +Do not use secure (ssl) connections when connecting to +remote servers. +\item [-{-}ssl-ca-path path] +File or directory or containing the trusted CA +certificates. If not specified, the defaults compiled +into the system's OpenSSL library are used. \item [-{-}version] just print program version and exit \item [-L \textless{}name\textgreater{}] @@ -896,7 +968,7 @@ file systems. The file system \emph{must not be mounted} when using The syntax is \begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{s3qladm }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}action\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage-url\PYGZgt{}} +\PYG{l}{s3qladm }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}action\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage\PYGZhy{}url\PYGZgt{}} \end{Verbatim} where \code{action} may be either of \textbf{passphrase}, @@ -907,8 +979,8 @@ matter what specific action is being invoked: \begin{quote} \begin{optionlist}{3cm} \item [-{-}debug \textless{}module\textgreater{}] -activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get -debug messages from all modules. This option can be +activate debugging output from \textless{}module\textgreater{}. Use \code{all} to +get debug messages from all modules. This option can be specified multiple times. \item [-{-}quiet] be really quiet @@ -916,15 +988,17 @@ be really quiet Write logging info into this file. File will be rotated when it reaches 1 MiB, and at most 5 old log files will be kept. Specify \code{none} to disable logging. Default: -\code{none} +\code{None} \item [-{-}authfile \textless{}path\textgreater{}] Read authentication credentials from this file (default: \code{\textasciitilde{}/.s3ql/authinfo2)} -\item [-{-}ssl] -Always use SSL connections when connecting to remote -servers. For backends that allow only encrypted -connections, S3QL uses SSL automatically, even if this -option is not set. +\item [-{-}no-ssl] +Do not use secure (ssl) connections when connecting to +remote servers. +\item [-{-}ssl-ca-path path] +File or directory or containing the trusted CA +certificates. If not specified, the defaults compiled +into the system's OpenSSL library are used. \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)} \item [-{-}version] @@ -985,7 +1059,7 @@ created backup copies. The command \begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{s3qladm download-metadata }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} +\PYG{l}{s3qladm download\PYGZhy{}metadata }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} \end{Verbatim} will give you a list of the available metadata backups and allow you @@ -1035,11 +1109,13 @@ get debug messages from all modules. This option can be specified multiple times. \item [-{-}quiet] be really quiet -\item [-{-}ssl] -Always use SSL connections when connecting to remote -servers. For backends that allow only encrypted -connections, S3QL uses SSL automatically, even if this -option is not set. +\item [-{-}no-ssl] +Do not use secure (ssl) connections when connecting to +remote servers. +\item [-{-}ssl-ca-path path] +File or directory or containing the trusted CA +certificates. If not specified, the defaults compiled +into the system's OpenSSL library are used. \item [-{-}version] just print program version and exit \item [-{-}cachesize \textless{}size\textgreater{}] @@ -1077,10 +1153,12 @@ is up. \item [-{-}profile] Create profiling information. If you don't understand this, then you don't need it. -\item [-{-}compress \textless{}name\textgreater{}] -Compression algorithm to use when storing new data. -Allowed values: \code{lzma}, \code{bzip2}, \code{zlib}, none. -(default: \code{lzma}) +\item [-{-}compress \textless{}algorithm-lvl\textgreater{}] +Compression algorithm and compression level to use +when storing new data. \emph{algorithm} may be any of +\code{lzma}, \code{bzip2}, \code{zlib}, or none. \emph{lvl} may be any +integer from 0 (fastest) to 9 (slowest). Default: +\code{lzma-6} \item [-{-}metadata-upload-interval \textless{}seconds\textgreater{}] Interval in seconds between complete metadata uploads. Set to 0 to disable. Default: 24h. @@ -1143,7 +1221,7 @@ number of file descriptors per process to about a thousand. If you specify a value for \code{-{-}max-cache-entries}, you should therefore make sure to also configure your system to increase the maximum number -of open file handles. This can be done temporarily with the \code{umask -n} +of open file handles. This can be done temporarily with the \code{ulimit -n} command. The method to permanently change this limit system-wide depends on your distribution. @@ -1169,46 +1247,46 @@ If your system is using upstart, an appropriate job can be defined as follows (and should be placed in \code{/etc/init/}): \begin{Verbatim}[commandchars=\\\{\},numbers=left,firstnumber=1,stepnumber=1] -\PYG{l}{description "S3QL Backup File System"} -\PYG{l}{author "Nikolaus Rath }\PYG{n+nv}{\PYGZlt{}Nikolaus@rath.org\PYGZgt{}}\PYG{l}{"} +\PYG{l}{description \PYGZdq{}S3QL Backup File System\PYGZdq{}} +\PYG{l}{author \PYGZdq{}Nikolaus Rath }\PYG{n+nv}{\PYGZlt{}Nikolaus@rath.org\PYGZgt{}}\PYG{l}{\PYGZdq{}} \PYG{l}{\PYGZsh{} This assumes that eth0 provides your internet connection} -\PYG{l}{start on (filesystem and net-device-up IFACE=eth0)} +\PYG{l}{start on (filesystem and net\PYGZhy{}device\PYGZhy{}up IFACE=eth0)} -\PYG{l}{\PYGZsh{} We can't use "stop on runlevel }\PYG{g+ge}{[016]}\PYG{l}{" because from that point on we} +\PYG{l}{\PYGZsh{} We can\PYGZsq{}t use \PYGZdq{}stop on runlevel }\PYG{g+ge}{[016]}\PYG{l}{\PYGZdq{} because from that point on we} \PYG{l}{\PYGZsh{} have only 10 seconds until the system shuts down completely.} \PYG{l}{stop on starting rc RUNLEVEL=}\PYG{g+ge}{[016]} \PYG{l}{\PYGZsh{} Time to wait before sending SIGKILL to the daemon and} -\PYG{l}{\PYGZsh{} pre-stop script} +\PYG{l}{\PYGZsh{} pre\PYGZhy{}stop script} \PYG{l}{kill timeout 300} -\PYG{l}{env STORAGE\PYGZus{}URL="s3://my-backup-bla"} -\PYG{l}{env MOUNTPOINT="/mnt/backup"} +\PYG{l}{env STORAGE\PYGZus{}URL=\PYGZdq{}s3://my\PYGZhy{}backup\PYGZhy{}bla\PYGZdq{}} +\PYG{l}{env MOUNTPOINT=\PYGZdq{}/mnt/backup\PYGZdq{}} -\PYG{l}{env USER="myusername"} -\PYG{l}{env AUTHFILE="/path/to/authinfo2"} +\PYG{l}{env USER=\PYGZdq{}myusername\PYGZdq{}} +\PYG{l}{env AUTHFILE=\PYGZdq{}/path/to/authinfo2\PYGZdq{}} \PYG{l}{expect stop} \PYG{l}{script} \PYG{l}{ \PYGZsh{} Redirect stdout and stderr into the system log} -\PYG{l}{ DIR=\PYGZdl{}(mktemp -d)} -\PYG{l}{ mkfifo "\PYGZdl{}DIR/LOG\PYGZus{}FIFO"} -\PYG{l}{ logger -t s3ql -p local0.info }\PYG{n+nv}{\PYGZlt{} "\PYGZdl{}DIR/LOG\PYGZus{}FIFO" \PYGZam{}} -\PYG{n+nv}{ exec \PYGZgt{}}\PYG{l}{ "\PYGZdl{}DIR/LOG\PYGZus{}FIFO"} +\PYG{l}{ DIR=\PYGZdl{}(mktemp \PYGZhy{}d)} +\PYG{l}{ mkfifo \PYGZdq{}\PYGZdl{}DIR/LOG\PYGZus{}FIFO\PYGZdq{}} +\PYG{l}{ logger \PYGZhy{}t s3ql \PYGZhy{}p local0.info }\PYG{n+nv}{\PYGZlt{} \PYGZdq{}\PYGZdl{}DIR/LOG\PYGZus{}FIFO\PYGZdq{} \PYGZam{}} +\PYG{n+nv}{ exec \PYGZgt{}}\PYG{l}{ \PYGZdq{}\PYGZdl{}DIR/LOG\PYGZus{}FIFO\PYGZdq{}} \PYG{l}{ exec 2\PYGZgt{}\PYGZam{}1} -\PYG{l}{ rm -rf "\PYGZdl{}DIR"} +\PYG{l}{ rm \PYGZhy{}rf \PYGZdq{}\PYGZdl{}DIR\PYGZdq{}} \PYG{l}{ \PYGZsh{} Check and mount file system} -\PYG{l}{ su -s /bin/sh -c 'exec "\PYGZdl{}0" "\PYGZdl{}@"' "\PYGZdl{}USER" -- \PYGZbs{}} -\PYG{l}{ fsck.s3ql --batch --authfile "\PYGZdl{}AUTHFILE" "\PYGZdl{}STORAGE\PYGZus{}URL"} -\PYG{l}{ exec su -s /bin/sh -c 'exec "\PYGZdl{}0" "\PYGZdl{}@"' "\PYGZdl{}USER" -- \PYGZbs{}} -\PYG{l}{ mount.s3ql --upstart --authfile "\PYGZdl{}AUTHFILE" "\PYGZdl{}STORAGE\PYGZus{}URL" "\PYGZdl{}MOUNTPOINT"} +\PYG{l}{ su \PYGZhy{}s /bin/sh \PYGZhy{}c \PYGZsq{}exec \PYGZdq{}\PYGZdl{}0\PYGZdq{} \PYGZdq{}\PYGZdl{}@\PYGZdq{}\PYGZsq{} \PYGZdq{}\PYGZdl{}USER\PYGZdq{} \PYGZhy{}\PYGZhy{} \PYGZbs{}} +\PYG{l}{ fsck.s3ql \PYGZhy{}\PYGZhy{}batch \PYGZhy{}\PYGZhy{}authfile \PYGZdq{}\PYGZdl{}AUTHFILE\PYGZdq{} \PYGZdq{}\PYGZdl{}STORAGE\PYGZus{}URL\PYGZdq{}} +\PYG{l}{ exec su \PYGZhy{}s /bin/sh \PYGZhy{}c \PYGZsq{}exec \PYGZdq{}\PYGZdl{}0\PYGZdq{} \PYGZdq{}\PYGZdl{}@\PYGZdq{}\PYGZsq{} \PYGZdq{}\PYGZdl{}USER\PYGZdq{} \PYGZhy{}\PYGZhy{} \PYGZbs{}} +\PYG{l}{ mount.s3ql \PYGZhy{}\PYGZhy{}upstart \PYGZhy{}\PYGZhy{}authfile \PYGZdq{}\PYGZdl{}AUTHFILE\PYGZdq{} \PYGZdq{}\PYGZdl{}STORAGE\PYGZus{}URL\PYGZdq{} \PYGZdq{}\PYGZdl{}MOUNTPOINT\PYGZdq{}} \PYG{l}{end script} -\PYG{l}{pre-stop script} -\PYG{l}{ su -s /bin/sh -c 'exec "\PYGZdl{}0" "\PYGZdl{}@"' "\PYGZdl{}USER" -- umount.s3ql "\PYGZdl{}MOUNTPOINT"} +\PYG{l}{pre\PYGZhy{}stop script} +\PYG{l}{ su \PYGZhy{}s /bin/sh \PYGZhy{}c \PYGZsq{}exec \PYGZdq{}\PYGZdl{}0\PYGZdq{} \PYGZdq{}\PYGZdl{}@\PYGZdq{}\PYGZsq{} \PYGZdq{}\PYGZdl{}USER\PYGZdq{} \PYGZhy{}\PYGZhy{} umount.s3ql \PYGZdq{}\PYGZdl{}MOUNTPOINT\PYGZdq{}} \PYG{l}{end script} \end{Verbatim} @@ -1349,7 +1427,7 @@ For example, to make the directory tree beneath the directory \code{2010-04-21} immutable, execute \begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{s3qllock 2010-04-21} +\PYG{l}{s3qllock 2010\PYGZhy{}04\PYGZhy{}21} \end{Verbatim} Immutability is a feature designed for backups. Traditionally, backups @@ -1384,7 +1462,7 @@ changed after they have been made immutable. \section{Fast Recursive Removal} -\label{special:s3qlrm}\label{special:fast-recursive-removal} +\label{special:fast-recursive-removal}\label{special:s3qlrm} The \code{s3qlrm} command can be used to recursively delete files and directories on an S3QL file system. Although \code{s3qlrm} is faster than using e.g. \code{rm -r}, the main reason for its existence is that it @@ -1400,7 +1478,7 @@ be removed entirely and immediately. \section{Runtime Configuration} -\label{special:runtime-configuration}\label{special:s3qlctrl} +\label{special:s3qlctrl}\label{special:runtime-configuration} The \code{s3qlctrl} can be used to control a mounted S3QL file system. Its syntax is @@ -1431,7 +1509,7 @@ Trigger a metadata upload. \chapter{Unmounting} -\label{umount::doc}\label{umount:unmounting} +\label{umount:unmounting}\label{umount::doc} To unmount an S3QL file system, use the command: \begin{Verbatim}[commandchars=\\\{\}] @@ -1495,16 +1573,18 @@ Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)} Read authentication credentials from this file (default: \code{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug \textless{}module\textgreater{}] -activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get -debug messages from all modules. This option can be +activate debugging output from \textless{}module\textgreater{}. Use \code{all} to +get debug messages from all modules. This option can be specified multiple times. \item [-{-}quiet] be really quiet -\item [-{-}ssl] -Always use SSL connections when connecting to remote -servers. For backends that allow only encrypted -connections, S3QL uses SSL automatically, even if this -option is not set. +\item [-{-}no-ssl] +Do not use secure (ssl) connections when connecting to +remote servers. +\item [-{-}ssl-ca-path path] +File or directory or containing the trusted CA +certificates. If not specified, the defaults compiled +into the system's OpenSSL library are used. \item [-{-}version] just print program version and exit \item [-{-}batch] @@ -1551,23 +1631,23 @@ For example, consider the following authentication file: \begin{Verbatim}[commandchars=\\\{\}] \PYG{g+ge}{[s3]} -\PYG{l}{storage-url: s3://} -\PYG{l}{backend-login: joe} -\PYG{l}{backend-password: notquitesecret} +\PYG{l}{storage\PYGZhy{}url: s3://} +\PYG{l}{backend\PYGZhy{}login: joe} +\PYG{l}{backend\PYGZhy{}password: notquitesecret} \PYG{g+ge}{[fs1]} -\PYG{l}{storage-url: s3://joes-first-bucket} -\PYG{l}{fs-passphrase: neitheristhis} +\PYG{l}{storage\PYGZhy{}url: s3://joes\PYGZhy{}first\PYGZhy{}bucket} +\PYG{l}{fs\PYGZhy{}passphrase: neitheristhis} \PYG{g+ge}{[fs2]} -\PYG{l}{storage-url: s3://joes-second-bucket} -\PYG{l}{fs-passphrase: swordfish} +\PYG{l}{storage\PYGZhy{}url: s3://joes\PYGZhy{}second\PYGZhy{}bucket} +\PYG{l}{fs\PYGZhy{}passphrase: swordfish} \PYG{g+ge}{[fs3]} -\PYG{l}{storage-url: s3://joes-second-bucket/with-prefix} -\PYG{l}{backend-login: bill} -\PYG{l}{backend-password: bi23ll} -\PYG{l}{fs-passphrase: ll23bi} +\PYG{l}{storage\PYGZhy{}url: s3://joes\PYGZhy{}second\PYGZhy{}bucket/with\PYGZhy{}prefix} +\PYG{l}{backend\PYGZhy{}login: bill} +\PYG{l}{backend\PYGZhy{}password: bi23ll} +\PYG{l}{fs\PYGZhy{}passphrase: ll23bi} \end{Verbatim} With this authentication file, S3QL would try to log in as ``joe'' @@ -1607,7 +1687,7 @@ migrate buckets to a different storage region or storage class \section{pcp.py} -\label{contrib:pcp-py}\label{contrib:pcp} +\label{contrib:pcp}\label{contrib:pcp-py} \code{pcp.py} is a wrapper program that starts several rsync processes to copy directory trees in parallel. This is important because transferring files in parallel significantly enhances performance when @@ -1619,7 +1699,7 @@ To recursively copy the directory \code{/mnt/home-backup} into you would execute \begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{pcp.py -a --processes=8 /mnt/home-backup/ /home/joe} +\PYG{l}{pcp.py \PYGZhy{}a \PYGZhy{}\PYGZhy{}processes=8 /mnt/home\PYGZhy{}backup/ /home/joe} \end{Verbatim} @@ -1823,11 +1903,11 @@ ensures that every instance looks into all directories. \begin{Verbatim}[commandchars=\\\{\}] \PYG{c}{\PYGZsh{}!/bin/bash} -\PYG{l}{RSYNC\PYGZus{}ARGS="-aHv /mnt/s3ql/ /home/restore/"} +\PYG{l}{RSYNC\PYGZus{}ARGS=\PYGZdq{}\PYGZhy{}aHv /mnt/s3ql/ /home/restore/\PYGZdq{}} -\PYG{l}{rsync -f "+ */" -f "-! }\PYG{g+ge}{[a-f]}\PYG{l}{*" \PYGZdl{}RSYNC\PYGZus{}ARGS \PYGZam{}} -\PYG{l}{rsync -f "+ */" -f "-! }\PYG{g+ge}{[g-l]}\PYG{l}{*" \PYGZdl{}RSYNC\PYGZus{}ARGS \PYGZam{}} -\PYG{l}{rsync -f "+ */" -f "- }\PYG{g+ge}{[a-l]}\PYG{l}{*" \PYGZdl{}RSYNC\PYGZus{}ARGS \PYGZam{}} +\PYG{l}{rsync \PYGZhy{}f \PYGZdq{}+ */\PYGZdq{} \PYGZhy{}f \PYGZdq{}\PYGZhy{}! }\PYG{g+ge}{[a\PYGZhy{}f]}\PYG{l}{*\PYGZdq{} \PYGZdl{}RSYNC\PYGZus{}ARGS \PYGZam{}} +\PYG{l}{rsync \PYGZhy{}f \PYGZdq{}+ */\PYGZdq{} \PYGZhy{}f \PYGZdq{}\PYGZhy{}! }\PYG{g+ge}{[g\PYGZhy{}l]}\PYG{l}{*\PYGZdq{} \PYGZdl{}RSYNC\PYGZus{}ARGS \PYGZam{}} +\PYG{l}{rsync \PYGZhy{}f \PYGZdq{}+ */\PYGZdq{} \PYGZhy{}f \PYGZdq{}\PYGZhy{} }\PYG{g+ge}{[a\PYGZhy{}l]}\PYG{l}{*\PYGZdq{} \PYGZdl{}RSYNC\PYGZus{}ARGS \PYGZam{}} \PYG{l}{wait} \end{Verbatim} @@ -1846,17 +1926,26 @@ details. \chapter{Known Issues} \label{issues:known-issues}\label{issues::doc}\begin{itemize} \item {} +S3QL de-duplicates data blocks based solely only on SHA256 +checksums, without doing a byte-by-byte comparison of the blocks. +Since it is possible for two data blocks to have the same checksum +despite having different contents, this can lead to problems. If two +such blocks are stored in an S3QL file system, the data in one block +will be lost and replaced by the data in the other block. However, +the chances of this occuring for any two blocks are about 1 in 10\textasciicircum{}77 +(2\textasciicircum{}256). For a file system that holds a total of 10\textasciicircum{}34 blocks, the +chances of a collision increase to about 1 in 10\textasciicircum{}9. Storing more +than 10\textasciicircum{}34 blocks (or about 10\textasciicircum{}25 TB with an (extremely small) block +size of 4 kB) is therefore not recommended. Being exceptionally +unlucky may also be a disadvantage. + +\item {} S3QL does not support Access Control Lists (ACLs). This is due to a bug in the FUSE library and will therefore hopefully be fixed at some point. See \href{http://code.google.com/p/s3ql/issues/detail?id=385}{issue 385} for more details. \item {} -S3QL 1.x does not verify TLS/SSL server certificates and is thus -vulnerable to man-in-the-middle attacks (S3QL versions after 2.0 -do not have this problem but require Python 3.3 or newer). - -\item {} S3QL is rather slow when an application tries to write data in unreasonably small chunks. If a 1 MiB file is copied in chunks of 1 KB, this will take more than 10 times as long as when it's copied @@ -1978,11 +2067,13 @@ get debug messages from all modules. This option can be specified multiple times. \item [-{-}quiet] be really quiet -\item [-{-}ssl] -Always use SSL connections when connecting to remote -servers. For backends that allow only encrypted -connections, S3QL uses SSL automatically, even if this -option is not set. +\item [-{-}no-ssl] +Do not use secure (ssl) connections when connecting to +remote servers. +\item [-{-}ssl-ca-path path] +File or directory or containing the trusted CA +certificates. If not specified, the defaults compiled +into the system's OpenSSL library are used. \item [-{-}version] just print program version and exit \item [-L \textless{}name\textgreater{}] @@ -2015,7 +2106,7 @@ system, common locations are \code{/usr/share/doc/s3ql} or \section{The \textbf{s3qladm} command} -\label{man/adm::doc}\label{man/adm:the-s3qladm-command} +\label{man/adm:the-s3qladm-command}\label{man/adm::doc} \subsection{Synopsis} \label{man/adm:synopsis} @@ -2043,8 +2134,8 @@ The \textbf{s3qladm} command accepts the following options. \begin{quote} \begin{optionlist}{3cm} \item [-{-}debug \textless{}module\textgreater{}] -activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get -debug messages from all modules. This option can be +activate debugging output from \textless{}module\textgreater{}. Use \code{all} to +get debug messages from all modules. This option can be specified multiple times. \item [-{-}quiet] be really quiet @@ -2052,15 +2143,17 @@ be really quiet Write logging info into this file. File will be rotated when it reaches 1 MiB, and at most 5 old log files will be kept. Specify \code{none} to disable logging. Default: -\code{none} +\code{None} \item [-{-}authfile \textless{}path\textgreater{}] Read authentication credentials from this file (default: \code{\textasciitilde{}/.s3ql/authinfo2)} -\item [-{-}ssl] -Always use SSL connections when connecting to remote -servers. For backends that allow only encrypted -connections, S3QL uses SSL automatically, even if this -option is not set. +\item [-{-}no-ssl] +Do not use secure (ssl) connections when connecting to +remote servers. +\item [-{-}ssl-ca-path path] +File or directory or containing the trusted CA +certificates. If not specified, the defaults compiled +into the system's OpenSSL library are used. \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)} \item [-{-}version] @@ -2107,7 +2200,7 @@ system, common locations are \code{/usr/share/doc/s3ql} or \section{The \textbf{mount.s3ql} command} -\label{man/mount::doc}\label{man/mount:the-mount-s3ql-command} +\label{man/mount:the-mount-s3ql-command}\label{man/mount::doc} \subsection{Synopsis} \label{man/mount:synopsis} @@ -2146,11 +2239,13 @@ get debug messages from all modules. This option can be specified multiple times. \item [-{-}quiet] be really quiet -\item [-{-}ssl] -Always use SSL connections when connecting to remote -servers. For backends that allow only encrypted -connections, S3QL uses SSL automatically, even if this -option is not set. +\item [-{-}no-ssl] +Do not use secure (ssl) connections when connecting to +remote servers. +\item [-{-}ssl-ca-path path] +File or directory or containing the trusted CA +certificates. If not specified, the defaults compiled +into the system's OpenSSL library are used. \item [-{-}version] just print program version and exit \item [-{-}cachesize \textless{}size\textgreater{}] @@ -2188,10 +2283,12 @@ is up. \item [-{-}profile] Create profiling information. If you don't understand this, then you don't need it. -\item [-{-}compress \textless{}name\textgreater{}] -Compression algorithm to use when storing new data. -Allowed values: \code{lzma}, \code{bzip2}, \code{zlib}, none. -(default: \code{lzma}) +\item [-{-}compress \textless{}algorithm-lvl\textgreater{}] +Compression algorithm and compression level to use +when storing new data. \emph{algorithm} may be any of +\code{lzma}, \code{bzip2}, \code{zlib}, or none. \emph{lvl} may be any +integer from 0 (fastest) to 9 (slowest). Default: +\code{lzma-6} \item [-{-}metadata-upload-interval \textless{}seconds\textgreater{}] Interval in seconds between complete metadata uploads. Set to 0 to disable. Default: 24h. @@ -2310,7 +2407,7 @@ additional argument that specifies the new cache size in KiB, so the complete command line is: \begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{s3qlctrl }\PYG{g+ge}{[options]}\PYG{l}{ cachesize }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}new-cache-size\PYGZgt{}} +\PYG{l}{s3qlctrl }\PYG{g+ge}{[options]}\PYG{l}{ cachesize }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}new\PYGZhy{}cache\PYGZhy{}size\PYGZgt{}} \end{Verbatim} \item[{log}] \leavevmode @@ -2369,7 +2466,7 @@ system, common locations are \code{/usr/share/doc/s3ql} or \subsection{Synopsis} \label{man/cp:synopsis} \begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{s3qlcp }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}source-dir\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}dest-dir\PYGZgt{}} +\PYG{l}{s3qlcp }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}source\PYGZhy{}dir\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}dest\PYGZhy{}dir\PYGZgt{}} \end{Verbatim} @@ -2468,7 +2565,7 @@ system, common locations are \code{/usr/share/doc/s3ql} or \section{The \textbf{s3qlrm} command} -\label{man/rm::doc}\label{man/rm:the-s3qlrm-command} +\label{man/rm:the-s3qlrm-command}\label{man/rm::doc} \subsection{Synopsis} \label{man/rm:synopsis} @@ -2612,7 +2709,7 @@ system, common locations are \code{/usr/share/doc/s3ql} or \section{The \textbf{umount.s3ql} command} -\label{man/umount::doc}\label{man/umount:the-umount-s3ql-command} +\label{man/umount:the-umount-s3ql-command}\label{man/umount::doc} \subsection{Synopsis} \label{man/umount:synopsis} @@ -2672,7 +2769,7 @@ system, common locations are \code{/usr/share/doc/s3ql} or \section{The \textbf{fsck.s3ql} command} -\label{man/fsck::doc}\label{man/fsck:the-fsck-s3ql-command} +\label{man/fsck:the-fsck-s3ql-command}\label{man/fsck::doc} \subsection{Synopsis} \label{man/fsck:synopsis} @@ -2706,16 +2803,18 @@ Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)} Read authentication credentials from this file (default: \code{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug \textless{}module\textgreater{}] -activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get -debug messages from all modules. This option can be +activate debugging output from \textless{}module\textgreater{}. Use \code{all} to +get debug messages from all modules. This option can be specified multiple times. \item [-{-}quiet] be really quiet -\item [-{-}ssl] -Always use SSL connections when connecting to remote -servers. For backends that allow only encrypted -connections, S3QL uses SSL automatically, even if this -option is not set. +\item [-{-}no-ssl] +Do not use secure (ssl) connections when connecting to +remote servers. +\item [-{-}ssl-ca-path path] +File or directory or containing the trusted CA +certificates. If not specified, the defaults compiled +into the system's OpenSSL library are used. \item [-{-}version] just print program version and exit \item [-{-}batch] @@ -2794,7 +2893,7 @@ error occured. \section{The \textbf{expire\_backups} command} -\label{man/expire_backups::doc}\label{man/expire_backups:the-expire-backups-command} +\label{man/expire_backups:the-expire-backups-command}\label{man/expire_backups::doc} \subsection{Synopsis} \label{man/expire_backups:synopsis} @@ -2917,7 +3016,7 @@ error occured. \chapter{Further Resources / Getting Help} -\label{resources::doc}\label{resources:further-resources-getting-help}\label{resources:resources} +\label{resources:resources}\label{resources:further-resources-getting-help}\label{resources::doc} If you have questions or problems with S3QL that you weren't able to resolve with this manual, you might want to consider the following other resources: \begin{itemize} @@ -2938,7 +3037,7 @@ Please report any bugs you may encounter in the \href{http://code.google.com/p/s \chapter{Implementation Details} -\label{impl_details:implementation-details}\label{impl_details::doc}\label{impl_details:impl-details} +\label{impl_details:impl-details}\label{impl_details:implementation-details}\label{impl_details::doc} This section provides some background information on how S3QL works internally. Reading this section is not necessary to use S3QL. |