blob: fa190f7a933c94d517baabccec438598b5d931d4 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
|
\babel@toc {english}{}
\contentsline {chapter}{\numberline {1}S3QL}{1}{chapter.1}%
\contentsline {section}{\numberline {1.1}Features}{1}{section.1.1}%
\contentsline {section}{\numberline {1.2}Development Status}{2}{section.1.2}%
\contentsline {section}{\numberline {1.3}Supported Platforms}{2}{section.1.3}%
\contentsline {section}{\numberline {1.4}Contributing}{2}{section.1.4}%
\contentsline {chapter}{\numberline {2}Installation}{3}{chapter.2}%
\contentsline {section}{\numberline {2.1}Dependencies}{3}{section.2.1}%
\contentsline {section}{\numberline {2.2}Installing S3QL}{4}{section.2.2}%
\contentsline {section}{\numberline {2.3}Development Version}{4}{section.2.3}%
\contentsline {section}{\numberline {2.4}Running tests requiring remote servers}{4}{section.2.4}%
\contentsline {chapter}{\numberline {3}Storage Backends}{7}{chapter.3}%
\contentsline {section}{\numberline {3.1}Google Storage}{7}{section.3.1}%
\contentsline {section}{\numberline {3.2}Amazon S3}{8}{section.3.2}%
\contentsline {section}{\numberline {3.3}OpenStack/Swift}{9}{section.3.3}%
\contentsline {section}{\numberline {3.4}Rackspace CloudFiles}{10}{section.3.4}%
\contentsline {section}{\numberline {3.5}S3 compatible}{10}{section.3.5}%
\contentsline {section}{\numberline {3.6}Local}{11}{section.3.6}%
\contentsline {chapter}{\numberline {4}Important Rules to Avoid Losing Data}{13}{chapter.4}%
\contentsline {section}{\numberline {4.1}Rules in a Nutshell}{13}{section.4.1}%
\contentsline {section}{\numberline {4.2}Consistency Window List}{14}{section.4.2}%
\contentsline {section}{\numberline {4.3}Data Consistency}{14}{section.4.3}%
\contentsline {section}{\numberline {4.4}Data Durability}{15}{section.4.4}%
\contentsline {chapter}{\numberline {5}File System Creation}{17}{chapter.5}%
\contentsline {chapter}{\numberline {6}Managing File Systems}{19}{chapter.6}%
\contentsline {section}{\numberline {6.1}Changing the Passphrase}{19}{section.6.1}%
\contentsline {section}{\numberline {6.2}Upgrading the file system}{20}{section.6.2}%
\contentsline {section}{\numberline {6.3}Deleting a file system}{20}{section.6.3}%
\contentsline {section}{\numberline {6.4}Restoring Metadata Backups}{20}{section.6.4}%
\contentsline {chapter}{\numberline {7}Mounting}{21}{chapter.7}%
\contentsline {section}{\numberline {7.1}Permission Checking}{22}{section.7.1}%
\contentsline {section}{\numberline {7.2}Compression Algorithms}{22}{section.7.2}%
\contentsline {section}{\numberline {7.3}Notes about Caching}{23}{section.7.3}%
\contentsline {subsection}{\numberline {7.3.1}Maximum Number of Cache Entries}{23}{subsection.7.3.1}%
\contentsline {subsection}{\numberline {7.3.2}Cache Flushing and Expiration}{23}{subsection.7.3.2}%
\contentsline {section}{\numberline {7.4}NFS Support}{23}{section.7.4}%
\contentsline {section}{\numberline {7.5}Failure Modes}{23}{section.7.5}%
\contentsline {section}{\numberline {7.6}Automatic Mounting}{24}{section.7.6}%
\contentsline {chapter}{\numberline {8}Advanced S3QL Features}{25}{chapter.8}%
\contentsline {section}{\numberline {8.1}Snapshotting and Copy-on-Write}{25}{section.8.1}%
\contentsline {subsection}{\numberline {8.1.1}Snapshotting vs Hardlinking}{25}{subsection.8.1.1}%
\contentsline {section}{\numberline {8.2}Getting Statistics}{26}{section.8.2}%
\contentsline {section}{\numberline {8.3}Immutable Trees}{26}{section.8.3}%
\contentsline {section}{\numberline {8.4}Fast Recursive Removal}{27}{section.8.4}%
\contentsline {section}{\numberline {8.5}Runtime Configuration}{27}{section.8.5}%
\contentsline {chapter}{\numberline {9}Unmounting}{29}{chapter.9}%
\contentsline {chapter}{\numberline {10}Checking for Errors}{31}{chapter.10}%
\contentsline {section}{\numberline {10.1}Checking and repairing internal file system errors}{31}{section.10.1}%
\contentsline {section}{\numberline {10.2}Detecting and handling backend data corruption}{32}{section.10.2}%
\contentsline {chapter}{\numberline {11}Storing Backend Information and Credentials}{35}{chapter.11}%
\contentsline {chapter}{\numberline {12}Contributed Programs}{37}{chapter.12}%
\contentsline {section}{\numberline {12.1}benchmark.py}{37}{section.12.1}%
\contentsline {section}{\numberline {12.2}clone\_fs.py}{37}{section.12.2}%
\contentsline {section}{\numberline {12.3}pcp.py}{37}{section.12.3}%
\contentsline {section}{\numberline {12.4}s3ql\_backup.sh}{37}{section.12.4}%
\contentsline {section}{\numberline {12.5}expire\_backups.py}{38}{section.12.5}%
\contentsline {section}{\numberline {12.6}remove\_objects.py}{39}{section.12.6}%
\contentsline {chapter}{\numberline {13}Tips \& Tricks}{41}{chapter.13}%
\contentsline {section}{\numberline {13.1}SSH Backend}{41}{section.13.1}%
\contentsline {section}{\numberline {13.2}Permanently mounted backup file system}{41}{section.13.2}%
\contentsline {section}{\numberline {13.3}Improving copy performance}{41}{section.13.3}%
\contentsline {chapter}{\numberline {14}Frequently Asked Questions}{43}{chapter.14}%
\contentsline {section}{\numberline {14.1}What does \IeC {\textquotedblleft }python-apsw must be linked dynamically to sqlite3\IeC {\textquotedblright } mean?}{43}{section.14.1}%
\contentsline {section}{\numberline {14.2}How can I improve the file system throughput?}{43}{section.14.2}%
\contentsline {section}{\numberline {14.3}Why does \sphinxstyleliteralintitle {\sphinxupquote {df}} show 1 TB of free space?}{43}{section.14.3}%
\contentsline {section}{\numberline {14.4}Which operating systems are supported?}{44}{section.14.4}%
\contentsline {section}{\numberline {14.5}Is there a file size limit?}{44}{section.14.5}%
\contentsline {section}{\numberline {14.6}Suppose I want to make a small change in a very large file. Will S3QL download and re-upload the entire file?}{44}{section.14.6}%
\contentsline {section}{\numberline {14.7}I don\IeC {\textquoteright }t quite understand this de-duplication feature\IeC {\textellipsis }}{44}{section.14.7}%
\contentsline {section}{\numberline {14.8}What does the \IeC {\textquotedblleft }Transport endpoint not connected\IeC {\textquotedblright } error mean?}{44}{section.14.8}%
\contentsline {section}{\numberline {14.9}What does \IeC {\textquotedblleft }Backend reports that fs is still mounted elsewhere, aborting\IeC {\textquotedblright } mean?}{44}{section.14.9}%
\contentsline {section}{\numberline {14.10}Can I access an S3QL file system on multiple computers simultaneously?}{45}{section.14.10}%
\contentsline {section}{\numberline {14.11}What maximum object size should I use?}{45}{section.14.11}%
\contentsline {section}{\numberline {14.12}Is there a way to make the cache persistent / access the file system offline?}{46}{section.14.12}%
\contentsline {section}{\numberline {14.13}I would like to use S3QL with Hubic, but\IeC {\textellipsis }}{46}{section.14.13}%
\contentsline {section}{\numberline {14.14}What\IeC {\textquoteright }s a reasonable metadata upload interval?}{46}{section.14.14}%
\contentsline {chapter}{\numberline {15}Known Issues}{49}{chapter.15}%
\contentsline {chapter}{\numberline {16}Manpages}{51}{chapter.16}%
\contentsline {section}{\numberline {16.1}The \sphinxstyleliteralstrong {\sphinxupquote {mkfs.s3ql}} command}{51}{section.16.1}%
\contentsline {subsection}{\numberline {16.1.1}Synopsis}{51}{subsection.16.1.1}%
\contentsline {subsection}{\numberline {16.1.2}Description}{51}{subsection.16.1.2}%
\contentsline {subsection}{\numberline {16.1.3}Options}{51}{subsection.16.1.3}%
\contentsline {subsection}{\numberline {16.1.4}Exit Codes}{52}{subsection.16.1.4}%
\contentsline {subsection}{\numberline {16.1.5}See Also}{52}{subsection.16.1.5}%
\contentsline {section}{\numberline {16.2}The \sphinxstyleliteralstrong {\sphinxupquote {s3qladm}} command}{52}{section.16.2}%
\contentsline {subsection}{\numberline {16.2.1}Synopsis}{52}{subsection.16.2.1}%
\contentsline {subsection}{\numberline {16.2.2}Description}{53}{subsection.16.2.2}%
\contentsline {subsection}{\numberline {16.2.3}Options}{53}{subsection.16.2.3}%
\contentsline {subsection}{\numberline {16.2.4}Actions}{53}{subsection.16.2.4}%
\contentsline {subsection}{\numberline {16.2.5}Exit Codes}{53}{subsection.16.2.5}%
\contentsline {subsection}{\numberline {16.2.6}See Also}{54}{subsection.16.2.6}%
\contentsline {section}{\numberline {16.3}The \sphinxstyleliteralstrong {\sphinxupquote {mount.s3ql}} command}{54}{section.16.3}%
\contentsline {subsection}{\numberline {16.3.1}Synopsis}{54}{subsection.16.3.1}%
\contentsline {subsection}{\numberline {16.3.2}Description}{54}{subsection.16.3.2}%
\contentsline {subsection}{\numberline {16.3.3}Options}{54}{subsection.16.3.3}%
\contentsline {subsection}{\numberline {16.3.4}Exit Codes}{55}{subsection.16.3.4}%
\contentsline {subsection}{\numberline {16.3.5}See Also}{56}{subsection.16.3.5}%
\contentsline {section}{\numberline {16.4}The \sphinxstyleliteralstrong {\sphinxupquote {s3qlstat}} command}{56}{section.16.4}%
\contentsline {subsection}{\numberline {16.4.1}Synopsis}{56}{subsection.16.4.1}%
\contentsline {subsection}{\numberline {16.4.2}Description}{57}{subsection.16.4.2}%
\contentsline {subsection}{\numberline {16.4.3}Options}{57}{subsection.16.4.3}%
\contentsline {subsection}{\numberline {16.4.4}Exit Codes}{57}{subsection.16.4.4}%
\contentsline {subsection}{\numberline {16.4.5}See Also}{57}{subsection.16.4.5}%
\contentsline {section}{\numberline {16.5}The \sphinxstyleliteralstrong {\sphinxupquote {s3qlctrl}} command}{57}{section.16.5}%
\contentsline {subsection}{\numberline {16.5.1}Synopsis}{57}{subsection.16.5.1}%
\contentsline {subsection}{\numberline {16.5.2}Description}{58}{subsection.16.5.2}%
\contentsline {subsection}{\numberline {16.5.3}Options}{58}{subsection.16.5.3}%
\contentsline {subsection}{\numberline {16.5.4}Exit Codes}{58}{subsection.16.5.4}%
\contentsline {subsection}{\numberline {16.5.5}See Also}{59}{subsection.16.5.5}%
\contentsline {section}{\numberline {16.6}The \sphinxstyleliteralstrong {\sphinxupquote {s3qlcp}} command}{59}{section.16.6}%
\contentsline {subsection}{\numberline {16.6.1}Synopsis}{59}{subsection.16.6.1}%
\contentsline {subsection}{\numberline {16.6.2}Description}{59}{subsection.16.6.2}%
\contentsline {subsubsection}{Snapshotting vs Hardlinking}{59}{subsubsection*.28}%
\contentsline {subsection}{\numberline {16.6.3}Options}{60}{subsection.16.6.3}%
\contentsline {subsection}{\numberline {16.6.4}Exit Codes}{60}{subsection.16.6.4}%
\contentsline {subsection}{\numberline {16.6.5}See Also}{60}{subsection.16.6.5}%
\contentsline {section}{\numberline {16.7}The \sphinxstyleliteralstrong {\sphinxupquote {s3qlrm}} command}{60}{section.16.7}%
\contentsline {subsection}{\numberline {16.7.1}Synopsis}{60}{subsection.16.7.1}%
\contentsline {subsection}{\numberline {16.7.2}Description}{60}{subsection.16.7.2}%
\contentsline {subsection}{\numberline {16.7.3}Options}{61}{subsection.16.7.3}%
\contentsline {subsection}{\numberline {16.7.4}Exit Codes}{61}{subsection.16.7.4}%
\contentsline {subsection}{\numberline {16.7.5}See Also}{61}{subsection.16.7.5}%
\contentsline {section}{\numberline {16.8}The \sphinxstyleliteralstrong {\sphinxupquote {s3qllock}} command}{61}{section.16.8}%
\contentsline {subsection}{\numberline {16.8.1}Synopsis}{61}{subsection.16.8.1}%
\contentsline {subsection}{\numberline {16.8.2}Description}{61}{subsection.16.8.2}%
\contentsline {subsection}{\numberline {16.8.3}Rationale}{62}{subsection.16.8.3}%
\contentsline {subsection}{\numberline {16.8.4}Options}{62}{subsection.16.8.4}%
\contentsline {subsection}{\numberline {16.8.5}Exit Codes}{62}{subsection.16.8.5}%
\contentsline {subsection}{\numberline {16.8.6}See Also}{62}{subsection.16.8.6}%
\contentsline {section}{\numberline {16.9}The \sphinxstyleliteralstrong {\sphinxupquote {umount.s3ql}} command}{63}{section.16.9}%
\contentsline {subsection}{\numberline {16.9.1}Synopsis}{63}{subsection.16.9.1}%
\contentsline {subsection}{\numberline {16.9.2}Description}{63}{subsection.16.9.2}%
\contentsline {subsection}{\numberline {16.9.3}Options}{63}{subsection.16.9.3}%
\contentsline {subsection}{\numberline {16.9.4}Exit Codes}{63}{subsection.16.9.4}%
\contentsline {subsection}{\numberline {16.9.5}See Also}{64}{subsection.16.9.5}%
\contentsline {section}{\numberline {16.10}The \sphinxstyleliteralstrong {\sphinxupquote {fsck.s3ql}} command}{64}{section.16.10}%
\contentsline {subsection}{\numberline {16.10.1}Synopsis}{64}{subsection.16.10.1}%
\contentsline {subsection}{\numberline {16.10.2}Description}{64}{subsection.16.10.2}%
\contentsline {subsection}{\numberline {16.10.3}Options}{64}{subsection.16.10.3}%
\contentsline {subsection}{\numberline {16.10.4}Exit Codes}{65}{subsection.16.10.4}%
\contentsline {subsection}{\numberline {16.10.5}See Also}{65}{subsection.16.10.5}%
\contentsline {section}{\numberline {16.11}The \sphinxstyleliteralstrong {\sphinxupquote {s3ql\_oauth\_client}} command}{66}{section.16.11}%
\contentsline {subsection}{\numberline {16.11.1}Synopsis}{66}{subsection.16.11.1}%
\contentsline {subsection}{\numberline {16.11.2}Description}{66}{subsection.16.11.2}%
\contentsline {subsection}{\numberline {16.11.3}Options}{66}{subsection.16.11.3}%
\contentsline {subsection}{\numberline {16.11.4}Exit Codes}{66}{subsection.16.11.4}%
\contentsline {subsection}{\numberline {16.11.5}See Also}{66}{subsection.16.11.5}%
\contentsline {section}{\numberline {16.12}The \sphinxstyleliteralstrong {\sphinxupquote {s3ql\_verify}} command}{66}{section.16.12}%
\contentsline {subsection}{\numberline {16.12.1}Synopsis}{66}{subsection.16.12.1}%
\contentsline {subsection}{\numberline {16.12.2}Description}{67}{subsection.16.12.2}%
\contentsline {subsection}{\numberline {16.12.3}Options}{67}{subsection.16.12.3}%
\contentsline {subsection}{\numberline {16.12.4}Exit Codes}{67}{subsection.16.12.4}%
\contentsline {subsection}{\numberline {16.12.5}See Also}{68}{subsection.16.12.5}%
\contentsline {section}{\numberline {16.13}The \sphinxstyleliteralstrong {\sphinxupquote {pcp}} command}{68}{section.16.13}%
\contentsline {subsection}{\numberline {16.13.1}Synopsis}{68}{subsection.16.13.1}%
\contentsline {subsection}{\numberline {16.13.2}Description}{68}{subsection.16.13.2}%
\contentsline {subsection}{\numberline {16.13.3}Options}{68}{subsection.16.13.3}%
\contentsline {subsection}{\numberline {16.13.4}Exit Codes}{69}{subsection.16.13.4}%
\contentsline {subsection}{\numberline {16.13.5}See Also}{69}{subsection.16.13.5}%
\contentsline {section}{\numberline {16.14}The \sphinxstyleliteralstrong {\sphinxupquote {expire\_backups}} command}{69}{section.16.14}%
\contentsline {subsection}{\numberline {16.14.1}Synopsis}{69}{subsection.16.14.1}%
\contentsline {subsection}{\numberline {16.14.2}Description}{69}{subsection.16.14.2}%
\contentsline {subsection}{\numberline {16.14.3}Options}{70}{subsection.16.14.3}%
\contentsline {subsection}{\numberline {16.14.4}Exit Codes}{70}{subsection.16.14.4}%
\contentsline {subsection}{\numberline {16.14.5}See Also}{71}{subsection.16.14.5}%
\contentsline {chapter}{\numberline {17}Further Resources / Getting Help}{73}{chapter.17}%
\contentsline {chapter}{\numberline {18}Implementation Details}{75}{chapter.18}%
\contentsline {section}{\numberline {18.1}Metadata Storage}{75}{section.18.1}%
\contentsline {section}{\numberline {18.2}Data Storage}{75}{section.18.2}%
\contentsline {section}{\numberline {18.3}Data De-Duplication}{76}{section.18.3}%
\contentsline {section}{\numberline {18.4}Caching}{76}{section.18.4}%
\contentsline {section}{\numberline {18.5}Eventual Consistency Handling}{76}{section.18.5}%
\contentsline {section}{\numberline {18.6}Encryption}{76}{section.18.6}%
|
