blob: e6f652e90e8537e48be57b1768ddd9a43015ab19 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
|
\select@language {english}
\contentsline {chapter}{\numberline {1}About S3QL}{1}{chapter.1}
\contentsline {section}{\numberline {1.1}Features}{1}{section.1.1}
\contentsline {section}{\numberline {1.2}Development Status}{2}{section.1.2}
\contentsline {chapter}{\numberline {2}Installation}{3}{chapter.2}
\contentsline {section}{\numberline {2.1}Dependencies}{3}{section.2.1}
\contentsline {section}{\numberline {2.2}Installing S3QL}{3}{section.2.2}
\contentsline {section}{\numberline {2.3}Development Version}{4}{section.2.3}
\contentsline {section}{\numberline {2.4}Running tests requiring remote servers}{4}{section.2.4}
\contentsline {chapter}{\numberline {3}Storage Backends}{7}{chapter.3}
\contentsline {section}{\numberline {3.1}Google Storage}{7}{section.3.1}
\contentsline {section}{\numberline {3.2}Amazon S3}{7}{section.3.2}
\contentsline {subsection}{\numberline {3.2.1}Reduced Redundancy Storage (RRS)}{8}{subsection.3.2.1}
\contentsline {section}{\numberline {3.3}OpenStack/Swift}{8}{section.3.3}
\contentsline {section}{\numberline {3.4}Rackspace CloudFiles}{8}{section.3.4}
\contentsline {section}{\numberline {3.5}S3 compatible}{9}{section.3.5}
\contentsline {section}{\numberline {3.6}Local}{9}{section.3.6}
\contentsline {chapter}{\numberline {4}Important Rules to Avoid Losing Data}{11}{chapter.4}
\contentsline {section}{\numberline {4.1}Rules in a Nutshell}{11}{section.4.1}
\contentsline {section}{\numberline {4.2}Consistency Window List}{12}{section.4.2}
\contentsline {section}{\numberline {4.3}Data Consistency}{12}{section.4.3}
\contentsline {section}{\numberline {4.4}Data Durability}{13}{section.4.4}
\contentsline {chapter}{\numberline {5}File System Creation}{15}{chapter.5}
\contentsline {chapter}{\numberline {6}Managing File Systems}{17}{chapter.6}
\contentsline {section}{\numberline {6.1}Changing the Passphrase}{17}{section.6.1}
\contentsline {section}{\numberline {6.2}Upgrading the file system}{18}{section.6.2}
\contentsline {section}{\numberline {6.3}Deleting a file system}{18}{section.6.3}
\contentsline {section}{\numberline {6.4}Restoring Metadata Backups}{18}{section.6.4}
\contentsline {chapter}{\numberline {7}Mounting}{19}{chapter.7}
\contentsline {section}{\numberline {7.1}Compression Algorithms}{20}{section.7.1}
\contentsline {section}{\numberline {7.2}Notes about Caching}{20}{section.7.2}
\contentsline {subsection}{\numberline {7.2.1}Maximum Number of Cache Entries}{21}{subsection.7.2.1}
\contentsline {subsection}{\numberline {7.2.2}Cache Flushing and Expiration}{21}{subsection.7.2.2}
\contentsline {section}{\numberline {7.3}Automatic Mounting}{21}{section.7.3}
\contentsline {chapter}{\numberline {8}Advanced S3QL Features}{23}{chapter.8}
\contentsline {section}{\numberline {8.1}Snapshotting and Copy-on-Write}{23}{section.8.1}
\contentsline {subsection}{\numberline {8.1.1}Snapshotting vs Hardlinking}{23}{subsection.8.1.1}
\contentsline {section}{\numberline {8.2}Getting Statistics}{24}{section.8.2}
\contentsline {section}{\numberline {8.3}Immutable Trees}{24}{section.8.3}
\contentsline {section}{\numberline {8.4}Fast Recursive Removal}{24}{section.8.4}
\contentsline {section}{\numberline {8.5}Runtime Configuration}{25}{section.8.5}
\contentsline {chapter}{\numberline {9}Unmounting}{27}{chapter.9}
\contentsline {chapter}{\numberline {10}Checking for Errors}{29}{chapter.10}
\contentsline {chapter}{\numberline {11}Storing Authentication Information}{31}{chapter.11}
\contentsline {chapter}{\numberline {12}Contributed Programs}{33}{chapter.12}
\contentsline {section}{\numberline {12.1}benchmark.py}{33}{section.12.1}
\contentsline {section}{\numberline {12.2}s3\_copy.py}{33}{section.12.2}
\contentsline {section}{\numberline {12.3}pcp.py}{33}{section.12.3}
\contentsline {section}{\numberline {12.4}s3\_backup.sh}{33}{section.12.4}
\contentsline {section}{\numberline {12.5}expire\_backups.py}{34}{section.12.5}
\contentsline {section}{\numberline {12.6}s3ql\_upstart.conf}{35}{section.12.6}
\contentsline {chapter}{\numberline {13}Tips \& Tricks}{37}{chapter.13}
\contentsline {section}{\numberline {13.1}SSH Backend}{37}{section.13.1}
\contentsline {section}{\numberline {13.2}Permanently mounted backup file system}{37}{section.13.2}
\contentsline {section}{\numberline {13.3}Improving copy performance}{37}{section.13.3}
\contentsline {chapter}{\numberline {14}Known Issues}{39}{chapter.14}
\contentsline {chapter}{\numberline {15}Manpages}{41}{chapter.15}
\contentsline {section}{\numberline {15.1}The \textbf {mkfs.s3ql} command}{41}{section.15.1}
\contentsline {subsection}{\numberline {15.1.1}Synopsis}{41}{subsection.15.1.1}
\contentsline {subsection}{\numberline {15.1.2}Description}{41}{subsection.15.1.2}
\contentsline {subsection}{\numberline {15.1.3}Options}{41}{subsection.15.1.3}
\contentsline {subsection}{\numberline {15.1.4}Exit Status}{42}{subsection.15.1.4}
\contentsline {subsection}{\numberline {15.1.5}See Also}{42}{subsection.15.1.5}
\contentsline {section}{\numberline {15.2}The \textbf {s3qladm} command}{42}{section.15.2}
\contentsline {subsection}{\numberline {15.2.1}Synopsis}{42}{subsection.15.2.1}
\contentsline {subsection}{\numberline {15.2.2}Description}{42}{subsection.15.2.2}
\contentsline {subsection}{\numberline {15.2.3}Options}{42}{subsection.15.2.3}
\contentsline {subsection}{\numberline {15.2.4}Actions}{43}{subsection.15.2.4}
\contentsline {subsection}{\numberline {15.2.5}Exit Status}{43}{subsection.15.2.5}
\contentsline {subsection}{\numberline {15.2.6}See Also}{43}{subsection.15.2.6}
\contentsline {section}{\numberline {15.3}The \textbf {mount.s3ql} command}{43}{section.15.3}
\contentsline {subsection}{\numberline {15.3.1}Synopsis}{43}{subsection.15.3.1}
\contentsline {subsection}{\numberline {15.3.2}Description}{43}{subsection.15.3.2}
\contentsline {subsection}{\numberline {15.3.3}Options}{44}{subsection.15.3.3}
\contentsline {subsection}{\numberline {15.3.4}Exit Status}{45}{subsection.15.3.4}
\contentsline {subsection}{\numberline {15.3.5}See Also}{45}{subsection.15.3.5}
\contentsline {section}{\numberline {15.4}The \textbf {s3qlstat} command}{45}{section.15.4}
\contentsline {subsection}{\numberline {15.4.1}Synopsis}{45}{subsection.15.4.1}
\contentsline {subsection}{\numberline {15.4.2}Description}{45}{subsection.15.4.2}
\contentsline {subsection}{\numberline {15.4.3}Options}{45}{subsection.15.4.3}
\contentsline {subsection}{\numberline {15.4.4}Exit Status}{45}{subsection.15.4.4}
\contentsline {subsection}{\numberline {15.4.5}See Also}{46}{subsection.15.4.5}
\contentsline {section}{\numberline {15.5}The \textbf {s3qlctrl} command}{46}{section.15.5}
\contentsline {subsection}{\numberline {15.5.1}Synopsis}{46}{subsection.15.5.1}
\contentsline {subsection}{\numberline {15.5.2}Description}{46}{subsection.15.5.2}
\contentsline {subsection}{\numberline {15.5.3}Options}{46}{subsection.15.5.3}
\contentsline {subsection}{\numberline {15.5.4}Exit Status}{47}{subsection.15.5.4}
\contentsline {subsection}{\numberline {15.5.5}See Also}{47}{subsection.15.5.5}
\contentsline {section}{\numberline {15.6}The \textbf {s3qlcp} command}{47}{section.15.6}
\contentsline {subsection}{\numberline {15.6.1}Synopsis}{47}{subsection.15.6.1}
\contentsline {subsection}{\numberline {15.6.2}Description}{47}{subsection.15.6.2}
\contentsline {subsubsection}{Snapshotting vs Hardlinking}{47}{subsubsection*.3}
\contentsline {subsection}{\numberline {15.6.3}Options}{48}{subsection.15.6.3}
\contentsline {subsection}{\numberline {15.6.4}Exit Status}{48}{subsection.15.6.4}
\contentsline {subsection}{\numberline {15.6.5}See Also}{48}{subsection.15.6.5}
\contentsline {section}{\numberline {15.7}The \textbf {s3qlrm} command}{48}{section.15.7}
\contentsline {subsection}{\numberline {15.7.1}Synopsis}{48}{subsection.15.7.1}
\contentsline {subsection}{\numberline {15.7.2}Description}{48}{subsection.15.7.2}
\contentsline {subsection}{\numberline {15.7.3}Options}{48}{subsection.15.7.3}
\contentsline {subsection}{\numberline {15.7.4}Exit Status}{49}{subsection.15.7.4}
\contentsline {subsection}{\numberline {15.7.5}See Also}{49}{subsection.15.7.5}
\contentsline {section}{\numberline {15.8}The \textbf {s3qllock} command}{49}{section.15.8}
\contentsline {subsection}{\numberline {15.8.1}Synopsis}{49}{subsection.15.8.1}
\contentsline {subsection}{\numberline {15.8.2}Description}{49}{subsection.15.8.2}
\contentsline {subsection}{\numberline {15.8.3}Rationale}{49}{subsection.15.8.3}
\contentsline {subsection}{\numberline {15.8.4}Options}{50}{subsection.15.8.4}
\contentsline {subsection}{\numberline {15.8.5}Exit Status}{50}{subsection.15.8.5}
\contentsline {subsection}{\numberline {15.8.6}See Also}{50}{subsection.15.8.6}
\contentsline {section}{\numberline {15.9}The \textbf {umount.s3ql} command}{50}{section.15.9}
\contentsline {subsection}{\numberline {15.9.1}Synopsis}{50}{subsection.15.9.1}
\contentsline {subsection}{\numberline {15.9.2}Description}{50}{subsection.15.9.2}
\contentsline {subsection}{\numberline {15.9.3}Options}{50}{subsection.15.9.3}
\contentsline {subsection}{\numberline {15.9.4}Exit Status}{51}{subsection.15.9.4}
\contentsline {subsection}{\numberline {15.9.5}See Also}{51}{subsection.15.9.5}
\contentsline {section}{\numberline {15.10}The \textbf {fsck.s3ql} command}{51}{section.15.10}
\contentsline {subsection}{\numberline {15.10.1}Synopsis}{51}{subsection.15.10.1}
\contentsline {subsection}{\numberline {15.10.2}Description}{51}{subsection.15.10.2}
\contentsline {subsection}{\numberline {15.10.3}Options}{51}{subsection.15.10.3}
\contentsline {subsection}{\numberline {15.10.4}Exit Status}{52}{subsection.15.10.4}
\contentsline {subsection}{\numberline {15.10.5}See Also}{52}{subsection.15.10.5}
\contentsline {section}{\numberline {15.11}The \textbf {pcp} command}{52}{section.15.11}
\contentsline {subsection}{\numberline {15.11.1}Synopsis}{52}{subsection.15.11.1}
\contentsline {subsection}{\numberline {15.11.2}Description}{52}{subsection.15.11.2}
\contentsline {subsection}{\numberline {15.11.3}Options}{52}{subsection.15.11.3}
\contentsline {subsection}{\numberline {15.11.4}Exit Status}{52}{subsection.15.11.4}
\contentsline {subsection}{\numberline {15.11.5}See Also}{52}{subsection.15.11.5}
\contentsline {section}{\numberline {15.12}The \textbf {expire\_backups} command}{53}{section.15.12}
\contentsline {subsection}{\numberline {15.12.1}Synopsis}{53}{subsection.15.12.1}
\contentsline {subsection}{\numberline {15.12.2}Description}{53}{subsection.15.12.2}
\contentsline {subsection}{\numberline {15.12.3}Options}{53}{subsection.15.12.3}
\contentsline {subsection}{\numberline {15.12.4}Exit Status}{54}{subsection.15.12.4}
\contentsline {subsection}{\numberline {15.12.5}See Also}{54}{subsection.15.12.5}
\contentsline {chapter}{\numberline {16}Further Resources / Getting Help}{55}{chapter.16}
\contentsline {chapter}{\numberline {17}Implementation Details}{57}{chapter.17}
\contentsline {section}{\numberline {17.1}Metadata Storage}{57}{section.17.1}
\contentsline {section}{\numberline {17.2}Data Storage}{57}{section.17.2}
\contentsline {section}{\numberline {17.3}Data De-Duplication}{57}{section.17.3}
\contentsline {section}{\numberline {17.4}Caching}{58}{section.17.4}
\contentsline {section}{\numberline {17.5}Eventual Consistency Handling}{58}{section.17.5}
\contentsline {section}{\numberline {17.6}Encryption}{58}{section.17.6}
|
