\select@language {english}
\contentsline {chapter}{\numberline {1}S3QL}{1}{chapter.1}
\contentsline {section}{\numberline {1.1}Features}{1}{section.1.1}
\contentsline {section}{\numberline {1.2}Development Status}{2}{section.1.2}
\contentsline {section}{\numberline {1.3}Supported Platforms}{2}{section.1.3}
\contentsline {section}{\numberline {1.4}Contributing}{2}{section.1.4}
\contentsline {chapter}{\numberline {2}Installation}{3}{chapter.2}
\contentsline {section}{\numberline {2.1}Dependencies}{3}{section.2.1}
\contentsline {section}{\numberline {2.2}Installing S3QL}{3}{section.2.2}
\contentsline {section}{\numberline {2.3}Development Version}{4}{section.2.3}
\contentsline {section}{\numberline {2.4}Running tests requiring remote servers}{4}{section.2.4}
\contentsline {chapter}{\numberline {3}Storage Backends}{7}{chapter.3}
\contentsline {section}{\numberline {3.1}Google Storage}{7}{section.3.1}
\contentsline {section}{\numberline {3.2}Amazon S3}{8}{section.3.2}
\contentsline {section}{\numberline {3.3}OpenStack/Swift}{9}{section.3.3}
\contentsline {section}{\numberline {3.4}Rackspace CloudFiles}{10}{section.3.4}
\contentsline {section}{\numberline {3.5}S3 compatible}{10}{section.3.5}
\contentsline {section}{\numberline {3.6}Local}{11}{section.3.6}
\contentsline {chapter}{\numberline {4}Important Rules to Avoid Losing Data}{13}{chapter.4}
\contentsline {section}{\numberline {4.1}Rules in a Nutshell}{13}{section.4.1}
\contentsline {section}{\numberline {4.2}Consistency Window List}{14}{section.4.2}
\contentsline {section}{\numberline {4.3}Data Consistency}{14}{section.4.3}
\contentsline {section}{\numberline {4.4}Data Durability}{15}{section.4.4}
\contentsline {chapter}{\numberline {5}File System Creation}{17}{chapter.5}
\contentsline {chapter}{\numberline {6}Managing File Systems}{19}{chapter.6}
\contentsline {section}{\numberline {6.1}Changing the Passphrase}{19}{section.6.1}
\contentsline {section}{\numberline {6.2}Upgrading the file system}{20}{section.6.2}
\contentsline {section}{\numberline {6.3}Deleting a file system}{20}{section.6.3}
\contentsline {section}{\numberline {6.4}Restoring Metadata Backups}{20}{section.6.4}
\contentsline {chapter}{\numberline {7}Mounting}{21}{chapter.7}
\contentsline {section}{\numberline {7.1}Permission Checking}{22}{section.7.1}
\contentsline {section}{\numberline {7.2}Compression Algorithms}{22}{section.7.2}
\contentsline {section}{\numberline {7.3}Notes about Caching}{23}{section.7.3}
\contentsline {subsection}{\numberline {7.3.1}Maximum Number of Cache Entries}{23}{subsection.7.3.1}
\contentsline {subsection}{\numberline {7.3.2}Cache Flushing and Expiration}{23}{subsection.7.3.2}
\contentsline {section}{\numberline {7.4}NFS Support}{23}{section.7.4}
\contentsline {section}{\numberline {7.5}Failure Modes}{23}{section.7.5}
\contentsline {section}{\numberline {7.6}Automatic Mounting}{24}{section.7.6}
\contentsline {chapter}{\numberline {8}Advanced S3QL Features}{25}{chapter.8}
\contentsline {section}{\numberline {8.1}Snapshotting and Copy-on-Write}{25}{section.8.1}
\contentsline {subsection}{\numberline {8.1.1}Snapshotting vs Hardlinking}{25}{subsection.8.1.1}
\contentsline {section}{\numberline {8.2}Getting Statistics}{26}{section.8.2}
\contentsline {section}{\numberline {8.3}Immutable Trees}{26}{section.8.3}
\contentsline {section}{\numberline {8.4}Fast Recursive Removal}{27}{section.8.4}
\contentsline {section}{\numberline {8.5}Runtime Configuration}{27}{section.8.5}
\contentsline {chapter}{\numberline {9}Unmounting}{29}{chapter.9}
\contentsline {chapter}{\numberline {10}Checking for Errors}{31}{chapter.10}
\contentsline {section}{\numberline {10.1}Checking and repairing internal file system errors}{31}{section.10.1}
\contentsline {section}{\numberline {10.2}Detecting and handling backend data corruption}{32}{section.10.2}
\contentsline {chapter}{\numberline {11}Storing Backend Information}{35}{chapter.11}
\contentsline {chapter}{\numberline {12}Contributed Programs}{37}{chapter.12}
\contentsline {section}{\numberline {12.1}benchmark.py}{37}{section.12.1}
\contentsline {section}{\numberline {12.2}clone\_fs.py}{37}{section.12.2}
\contentsline {section}{\numberline {12.3}pcp.py}{37}{section.12.3}
\contentsline {section}{\numberline {12.4}s3ql\_backup.sh}{37}{section.12.4}
\contentsline {section}{\numberline {12.5}expire\_backups.py}{38}{section.12.5}
\contentsline {section}{\numberline {12.6}remove\_objects.py}{39}{section.12.6}
\contentsline {chapter}{\numberline {13}Tips \& Tricks}{41}{chapter.13}
\contentsline {section}{\numberline {13.1}SSH Backend}{41}{section.13.1}
\contentsline {section}{\numberline {13.2}Permanently mounted backup file system}{41}{section.13.2}
\contentsline {section}{\numberline {13.3}Improving copy performance}{41}{section.13.3}
\contentsline {chapter}{\numberline {14}Frequently Asked Questions}{43}{chapter.14}
\contentsline {section}{\numberline {14.1}What does \IeC {\textquotedblleft }python-apsw must be linked dynamically to sqlite3\IeC {\textquotedblright } mean?}{43}{section.14.1}
\contentsline {section}{\numberline {14.2}How can I improve the file system throughput?}{43}{section.14.2}
\contentsline {section}{\numberline {14.3}Why does \sphinxstyleliteralintitle {\sphinxupquote {df}} show 1 TB of free space?}{43}{section.14.3}
\contentsline {section}{\numberline {14.4}Which operating systems are supported?}{44}{section.14.4}
\contentsline {section}{\numberline {14.5}Is there a file size limit?}{44}{section.14.5}
\contentsline {section}{\numberline {14.6}Suppose I want to make a small change in a very large file. Will S3QL download and re-upload the entire file?}{44}{section.14.6}
\contentsline {section}{\numberline {14.7}I don\IeC {\textquoteright }t quite understand this de-duplication feature\IeC {\textellipsis }}{44}{section.14.7}
\contentsline {section}{\numberline {14.8}What does the \IeC {\textquotedblleft }Transport endpoint not connected\IeC {\textquotedblright } error mean?}{44}{section.14.8}
\contentsline {section}{\numberline {14.9}What does \IeC {\textquotedblleft }Backend reports that fs is still mounted elsewhere, aborting\IeC {\textquotedblright } mean?}{44}{section.14.9}
\contentsline {section}{\numberline {14.10}Can I access an S3QL file system on multiple computers simultaneously?}{45}{section.14.10}
\contentsline {section}{\numberline {14.11}What maximum object size should I use?}{45}{section.14.11}
\contentsline {section}{\numberline {14.12}Is there a way to make the cache persistent / access the file system offline?}{46}{section.14.12}
\contentsline {section}{\numberline {14.13}I would like to use S3QL with Hubic, but\IeC {\textellipsis }}{46}{section.14.13}
\contentsline {section}{\numberline {14.14}What\IeC {\textquoteright }s a reasonable metadata upload interval?}{46}{section.14.14}
\contentsline {chapter}{\numberline {15}Known Issues}{49}{chapter.15}
\contentsline {chapter}{\numberline {16}Manpages}{51}{chapter.16}
\contentsline {section}{\numberline {16.1}The \sphinxstyleliteralstrong {\sphinxupquote {mkfs.s3ql}} command}{51}{section.16.1}
\contentsline {subsection}{\numberline {16.1.1}Synopsis}{51}{subsection.16.1.1}
\contentsline {subsection}{\numberline {16.1.2}Description}{51}{subsection.16.1.2}
\contentsline {subsection}{\numberline {16.1.3}Options}{51}{subsection.16.1.3}
\contentsline {subsection}{\numberline {16.1.4}Exit Codes}{52}{subsection.16.1.4}
\contentsline {subsection}{\numberline {16.1.5}See Also}{52}{subsection.16.1.5}
\contentsline {section}{\numberline {16.2}The \sphinxstyleliteralstrong {\sphinxupquote {s3qladm}} command}{52}{section.16.2}
\contentsline {subsection}{\numberline {16.2.1}Synopsis}{52}{subsection.16.2.1}
\contentsline {subsection}{\numberline {16.2.2}Description}{53}{subsection.16.2.2}
\contentsline {subsection}{\numberline {16.2.3}Options}{53}{subsection.16.2.3}
\contentsline {subsection}{\numberline {16.2.4}Actions}{53}{subsection.16.2.4}
\contentsline {subsection}{\numberline {16.2.5}Exit Codes}{53}{subsection.16.2.5}
\contentsline {subsection}{\numberline {16.2.6}See Also}{54}{subsection.16.2.6}
\contentsline {section}{\numberline {16.3}The \sphinxstyleliteralstrong {\sphinxupquote {mount.s3ql}} command}{54}{section.16.3}
\contentsline {subsection}{\numberline {16.3.1}Synopsis}{54}{subsection.16.3.1}
\contentsline {subsection}{\numberline {16.3.2}Description}{54}{subsection.16.3.2}
\contentsline {subsection}{\numberline {16.3.3}Options}{54}{subsection.16.3.3}
\contentsline {subsection}{\numberline {16.3.4}Exit Codes}{55}{subsection.16.3.4}
\contentsline {subsection}{\numberline {16.3.5}See Also}{56}{subsection.16.3.5}
\contentsline {section}{\numberline {16.4}The \sphinxstyleliteralstrong {\sphinxupquote {s3qlstat}} command}{56}{section.16.4}
\contentsline {subsection}{\numberline {16.4.1}Synopsis}{56}{subsection.16.4.1}
\contentsline {subsection}{\numberline {16.4.2}Description}{56}{subsection.16.4.2}
\contentsline {subsection}{\numberline {16.4.3}Options}{57}{subsection.16.4.3}
\contentsline {subsection}{\numberline {16.4.4}Exit Codes}{57}{subsection.16.4.4}
\contentsline {subsection}{\numberline {16.4.5}See Also}{57}{subsection.16.4.5}
\contentsline {section}{\numberline {16.5}The \sphinxstyleliteralstrong {\sphinxupquote {s3qlctrl}} command}{57}{section.16.5}
\contentsline {subsection}{\numberline {16.5.1}Synopsis}{57}{subsection.16.5.1}
\contentsline {subsection}{\numberline {16.5.2}Description}{57}{subsection.16.5.2}
\contentsline {subsection}{\numberline {16.5.3}Options}{58}{subsection.16.5.3}
\contentsline {subsection}{\numberline {16.5.4}Exit Codes}{58}{subsection.16.5.4}
\contentsline {subsection}{\numberline {16.5.5}See Also}{58}{subsection.16.5.5}
\contentsline {section}{\numberline {16.6}The \sphinxstyleliteralstrong {\sphinxupquote {s3qlcp}} command}{58}{section.16.6}
\contentsline {subsection}{\numberline {16.6.1}Synopsis}{58}{subsection.16.6.1}
\contentsline {subsection}{\numberline {16.6.2}Description}{59}{subsection.16.6.2}
\contentsline {subsubsection}{Snapshotting vs Hardlinking}{59}{subsubsection*.22}
\contentsline {subsection}{\numberline {16.6.3}Options}{59}{subsection.16.6.3}
\contentsline {subsection}{\numberline {16.6.4}Exit Codes}{59}{subsection.16.6.4}
\contentsline {subsection}{\numberline {16.6.5}See Also}{60}{subsection.16.6.5}
\contentsline {section}{\numberline {16.7}The \sphinxstyleliteralstrong {\sphinxupquote {s3qlrm}} command}{60}{section.16.7}
\contentsline {subsection}{\numberline {16.7.1}Synopsis}{60}{subsection.16.7.1}
\contentsline {subsection}{\numberline {16.7.2}Description}{60}{subsection.16.7.2}
\contentsline {subsection}{\numberline {16.7.3}Options}{60}{subsection.16.7.3}
\contentsline {subsection}{\numberline {16.7.4}Exit Codes}{60}{subsection.16.7.4}
\contentsline {subsection}{\numberline {16.7.5}See Also}{61}{subsection.16.7.5}
\contentsline {section}{\numberline {16.8}The \sphinxstyleliteralstrong {\sphinxupquote {s3qllock}} command}{61}{section.16.8}
\contentsline {subsection}{\numberline {16.8.1}Synopsis}{61}{subsection.16.8.1}
\contentsline {subsection}{\numberline {16.8.2}Description}{61}{subsection.16.8.2}
\contentsline {subsection}{\numberline {16.8.3}Rationale}{61}{subsection.16.8.3}
\contentsline {subsection}{\numberline {16.8.4}Options}{61}{subsection.16.8.4}
\contentsline {subsection}{\numberline {16.8.5}Exit Codes}{62}{subsection.16.8.5}
\contentsline {subsection}{\numberline {16.8.6}See Also}{62}{subsection.16.8.6}
\contentsline {section}{\numberline {16.9}The \sphinxstyleliteralstrong {\sphinxupquote {umount.s3ql}} command}{62}{section.16.9}
\contentsline {subsection}{\numberline {16.9.1}Synopsis}{62}{subsection.16.9.1}
\contentsline {subsection}{\numberline {16.9.2}Description}{62}{subsection.16.9.2}
\contentsline {subsection}{\numberline {16.9.3}Options}{62}{subsection.16.9.3}
\contentsline {subsection}{\numberline {16.9.4}Exit Codes}{63}{subsection.16.9.4}
\contentsline {subsection}{\numberline {16.9.5}See Also}{63}{subsection.16.9.5}
\contentsline {section}{\numberline {16.10}The \sphinxstyleliteralstrong {\sphinxupquote {fsck.s3ql}} command}{63}{section.16.10}
\contentsline {subsection}{\numberline {16.10.1}Synopsis}{63}{subsection.16.10.1}
\contentsline {subsection}{\numberline {16.10.2}Description}{63}{subsection.16.10.2}
\contentsline {subsection}{\numberline {16.10.3}Options}{63}{subsection.16.10.3}
\contentsline {subsection}{\numberline {16.10.4}Exit Codes}{64}{subsection.16.10.4}
\contentsline {subsection}{\numberline {16.10.5}See Also}{65}{subsection.16.10.5}
\contentsline {section}{\numberline {16.11}The \sphinxstyleliteralstrong {\sphinxupquote {s3ql\_oauth\_client}} command}{65}{section.16.11}
\contentsline {subsection}{\numberline {16.11.1}Synopsis}{65}{subsection.16.11.1}
\contentsline {subsection}{\numberline {16.11.2}Description}{65}{subsection.16.11.2}
\contentsline {subsection}{\numberline {16.11.3}Options}{65}{subsection.16.11.3}
\contentsline {subsection}{\numberline {16.11.4}Exit Codes}{65}{subsection.16.11.4}
\contentsline {subsection}{\numberline {16.11.5}See Also}{66}{subsection.16.11.5}
\contentsline {section}{\numberline {16.12}The \sphinxstyleliteralstrong {\sphinxupquote {s3ql\_verify}} command}{66}{section.16.12}
\contentsline {subsection}{\numberline {16.12.1}Synopsis}{66}{subsection.16.12.1}
\contentsline {subsection}{\numberline {16.12.2}Description}{66}{subsection.16.12.2}
\contentsline {subsection}{\numberline {16.12.3}Options}{66}{subsection.16.12.3}
\contentsline {subsection}{\numberline {16.12.4}Exit Codes}{67}{subsection.16.12.4}
\contentsline {subsection}{\numberline {16.12.5}See Also}{67}{subsection.16.12.5}
\contentsline {section}{\numberline {16.13}The \sphinxstyleliteralstrong {\sphinxupquote {pcp}} command}{67}{section.16.13}
\contentsline {subsection}{\numberline {16.13.1}Synopsis}{67}{subsection.16.13.1}
\contentsline {subsection}{\numberline {16.13.2}Description}{67}{subsection.16.13.2}
\contentsline {subsection}{\numberline {16.13.3}Options}{68}{subsection.16.13.3}
\contentsline {subsection}{\numberline {16.13.4}Exit Codes}{68}{subsection.16.13.4}
\contentsline {subsection}{\numberline {16.13.5}See Also}{68}{subsection.16.13.5}
\contentsline {section}{\numberline {16.14}The \sphinxstyleliteralstrong {\sphinxupquote {expire\_backups}} command}{68}{section.16.14}
\contentsline {subsection}{\numberline {16.14.1}Synopsis}{68}{subsection.16.14.1}
\contentsline {subsection}{\numberline {16.14.2}Description}{68}{subsection.16.14.2}
\contentsline {subsection}{\numberline {16.14.3}Options}{69}{subsection.16.14.3}
\contentsline {subsection}{\numberline {16.14.4}Exit Codes}{70}{subsection.16.14.4}
\contentsline {subsection}{\numberline {16.14.5}See Also}{70}{subsection.16.14.5}
\contentsline {chapter}{\numberline {17}Further Resources / Getting Help}{71}{chapter.17}
\contentsline {chapter}{\numberline {18}Implementation Details}{73}{chapter.18}
\contentsline {section}{\numberline {18.1}Metadata Storage}{73}{section.18.1}
\contentsline {section}{\numberline {18.2}Data Storage}{73}{section.18.2}
\contentsline {section}{\numberline {18.3}Data De-Duplication}{74}{section.18.3}
\contentsline {section}{\numberline {18.4}Caching}{74}{section.18.4}
\contentsline {section}{\numberline {18.5}Eventual Consistency Handling}{74}{section.18.5}
\contentsline {section}{\numberline {18.6}Encryption}{74}{section.18.6}