diff options
| author | Nikolaus Rath <Nikolaus@rath.org> | 2016-03-09 10:08:24 -0800 |
|---|---|---|
| committer | Nikolaus Rath <Nikolaus@rath.org> | 2016-03-09 10:08:24 -0800 |
| commit | 538edbe3d8a600a9ee4ff4a7822abbc79cbf00e1 (patch) | |
| tree | 21cd3a3ec21703fc89ffde47c88f23b2f4a3e785 | |
| parent | 5b62499deb280253671a3c468644335be9e55f04 (diff) | |
Import s3ql_1.2.orig.tar.bz2
182 files changed, 9102 insertions, 13373 deletions
diff --git a/Changes.txt b/Changes.txt index 02c5446..a86f89f 100644 --- a/Changes.txt +++ b/Changes.txt @@ -1,9 +1,110 @@ +2011-09-28, S3QL 1.2 + + * Fixed a database problem that was responsible for file + system access becomer slower and slower the more data + was stored in the file system. + + * Fixed a race condition that could cause applications to get just + zero bytes when trying to read from a file that has just been + copied with s3qlcp. + +2011-09-20, S3QL 1.1.4 + + * Fixed a typo that caused errors when trying to remove any blocks + that have been committed to the backend. + + * Improved accuracy of s3qlstat during active file transfers + (but still not 100% accurate). + + * Fixed some theoretical deadlocks. + + * contrib/benchmark.py is now working again and also takes into + account the throughput from userspace to S3QL. + +2011-09-18, S3QL 1.1.3 + + * Fixed a race condition in the local backend that resulted in + errors of the form "[Errno 17] File exists: [bucket path]". + + * Added Google Storage backend. + + * Added backend for general, S3 compatible storage services. + + * Fixed a bug that caused S3QL to complain about the backend having + lost objects when trying to download an object before its upload + was completed. This could happen because locking was done based on + inode and block number rather than object id, and the + de-duplication feature can reuse an object for another inode + before the upload is completed. + + * Fixed a data corruption bug. If a data block was changed while + being uploaded to the backend, and a second, identical data block + was flushed while the upload was in progress, but before the first + block was changed, the second data block was linked to the + *modified* data. This happened because changes to an object in + transit were only checked for after the upload completed, leaving + a window in which the contents of the upload object did not agree + with its stored hash. + + This problem can be detected by verifying the hash values of all + stored data blocks. This procedure will automatically be done when + the file system is updated to the newest revision, but may take a + longer time since every object needs to be downloaded and checked. + +2011-09-08, S3QL 1.1.2 + + * The modules for communicating with the different storage providers + have been completely rewritten, resulting in improved performance, + more features and better maintainability. + + * S3 buckets can now be used with arbitrary prefixes, allowing to + store more than one S3QL file system in a bucket. + + * The format of the --authinfo file has changed. See the + documentation for details of the new format. To avoid breaking + backwards compatibility, the default file is now called authinfo2. + + * Network errors are now handled much more consistently. + + * There is a new --nfs option for mount.s3ql that needs to be used + when the S3QL file system will be exported over NFS. + + * The local backend now stores data and metadata in the same file, + so it needs only half as many files as before. + + * The --homedir option has been replaced by the more finely grained + --authfile, --cachedir and --log options. + + * S3QL can now log directly to syslog. + + * The sftp storage backend has been dropped. The recommended way to + use S3QL over ssh is to use sshfs + (http://fuse.sourceforge.net/sshfs.html) with S3QL's local + backend. + + * fsck now checks if all indices have been created. This avoids + a huge performance problem when mount.s3ql was interrupted + after downloading metadata, but before creating the indices. + +2011-07-23, S3QL 1.1 (development version) + + * Restructured metadata. This should also significantly reduce the + size of the SQLite database file. + + * Fixed license typo in file header comments, license + is GNU GPL Version 3, not LGPL. + + * Fixed problem with fsck.s3ql generating extraordinary long + filenames in /lost+found and then crashing. + + * When called as root, use umount rather than fusermount for + compatibility with FUSE4BSD. + 2011-05-20, S3QL 1.0.1 * Disabled WAL mode again for now because of unexpected problems with s3qlcp, s3qllock and s3qlrm (performance going down orders of magnitude, and *very* large *.db-wal file in ~/.s3ql). - 2011-05-13, S3QL 1.0 @@ -0,0 +1,687 @@ +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License Version 3 as +published by the Free Software Foundation. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +For reference, the full text of the GNU General Public +License Version 3 is included below: + + + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<http://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<http://www.gnu.org/philosophy/why-not-lgpl.html>. @@ -1,11 +1,11 @@ Metadata-Version: 1.1 Name: s3ql -Version: 1.0.1 +Version: 1.2 Summary: a full-featured file system for online data storage Home-page: http://code.google.com/p/s3ql/ Author: Nikolaus Rath Author-email: Nikolaus@rath.org -License: LGPL +License: GPLv3 Download-URL: http://code.google.com/p/s3ql/downloads/list Description: .. -*- mode: rst -*- @@ -13,13 +13,15 @@ Description: .. -*- mode: rst -*- About S3QL ============ - S3QL is a file system that stores all its data online. It supports - `Amazon S3 <http://aws.amazon.com/s3 Amazon S3>`_ as well as arbitrary - SFTP servers and effectively provides you with a hard disk of dynamic, - infinite capacity that can be accessed from any computer with internet - access. + S3QL is a file system that stores all its data online using storage + services like `Google Storage + <http://code.google.com/apis/storage/>`_, `Amazon S3 + <http://aws.amazon.com/s3 Amazon S3>`_ or `OpenStack + <http://openstack.org/projects/storage/>`_. S3QL effectively provides + a hard disk of dynamic, infinite capacity that can be accessed from + any computer with internet access. - S3QL is providing a standard, full featured UNIX file system that is + S3QL is a standard conforming, full featured UNIX file system that is conceptually indistinguishable from any local file system. Furthermore, S3QL has additional features like compression, encryption, data de-duplication, immutable trees and snapshotting @@ -106,7 +108,7 @@ Platform: Linux Classifier: Development Status :: 4 - Beta Classifier: Environment :: No Input/Output (Daemon) Classifier: Environment :: Console -Classifier: License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL) +Classifier: License :: OSI Approved :: GNU Library or Lesser General Public License (GPLv3) Classifier: Topic :: Internet Classifier: Operating System :: POSIX Classifier: Topic :: System :: Archiving diff --git a/bin/fsck.s3ql b/bin/fsck.s3ql index ddc4fb6..f296555 100755 --- a/bin/fsck.s3ql +++ b/bin/fsck.s3ql @@ -4,7 +4,7 @@ fsck.s3ql - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import unicode_literals, division, print_function diff --git a/bin/mkfs.s3ql b/bin/mkfs.s3ql index ed85b95..d0d311c 100755 --- a/bin/mkfs.s3ql +++ b/bin/mkfs.s3ql @@ -4,7 +4,7 @@ mkfs.s3ql - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import unicode_literals, division, print_function diff --git a/bin/mount.s3ql b/bin/mount.s3ql index f292af6..52eb540 100755 --- a/bin/mount.s3ql +++ b/bin/mount.s3ql @@ -4,7 +4,7 @@ mount.s3ql - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import unicode_literals, division, print_function diff --git a/bin/s3qladm b/bin/s3qladm index a877af2..b5bb8b1 100755 --- a/bin/s3qladm +++ b/bin/s3qladm @@ -4,7 +4,7 @@ s3qladm - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2010 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import unicode_literals, division, print_function @@ -4,7 +4,7 @@ s3qlcp - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import unicode_literals, division, print_function diff --git a/bin/s3qlctrl b/bin/s3qlctrl index 6b2535a..1c765eb 100755 --- a/bin/s3qlctrl +++ b/bin/s3qlctrl @@ -4,7 +4,7 @@ s3qlctrl - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2010 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import unicode_literals, division, print_function diff --git a/bin/s3qllock b/bin/s3qllock index 69b18ae..e3e7716 100755 --- a/bin/s3qllock +++ b/bin/s3qllock @@ -4,7 +4,7 @@ s3qllock - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import unicode_literals, division, print_function @@ -4,7 +4,7 @@ s3qlrm - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import unicode_literals, division, print_function diff --git a/bin/s3qlstat b/bin/s3qlstat index df7a5bf..083f5c1 100755 --- a/bin/s3qlstat +++ b/bin/s3qlstat @@ -4,7 +4,7 @@ s3qlstat - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import unicode_literals, division, print_function diff --git a/bin/umount.s3ql b/bin/umount.s3ql index 9db3e70..f2df19c 100755 --- a/bin/umount.s3ql +++ b/bin/umount.s3ql @@ -4,7 +4,7 @@ umount.s3ql - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import unicode_literals, division, print_function diff --git a/contrib/benchmark.py b/contrib/benchmark.py index 6dcd71a..150f69a 100755 --- a/contrib/benchmark.py +++ b/contrib/benchmark.py @@ -8,18 +8,20 @@ algorithm that maximizes throughput. --- Copyright (C) 2010 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import - +import argparse +import atexit +import logging +import os import sys +import tempfile +import math import time -import os -import logging -import lzma -import zlib -import bz2 +import subprocess +import shutil # We are running from the S3QL source directory, make sure # that we use modules from this directory @@ -28,10 +30,10 @@ if (os.path.exists(os.path.join(basedir, 'setup.py')) and os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): sys.path = [os.path.join(basedir, 'src')] + sys.path -from s3ql.backends.common import compress_encrypt_fh -from s3ql.common import (get_backend, QuietError,setup_logging) +from s3ql.backends.common import get_bucket, BetterBucket +from s3ql.backends.local import Bucket +from s3ql.common import setup_logging from s3ql.parse_args import ArgumentParser -import argparse log = logging.getLogger('benchmark') @@ -39,24 +41,17 @@ def parse_args(args): '''Parse command line''' parser = ArgumentParser( - usage="%prog [options] <storage-url> <test-file>\n" - "%prog --help", - description="Transfers and compresses the test file and gives a recommendation " - "for the compression algorithm to use.") + description='Measure S3QL write performance, uplink bandwidth and ' + 'compression speed and determine limiting factor.') - parser.add_homedir() + parser.add_authfile() parser.add_quiet() parser.add_debug() parser.add_version() - parser.add_ssl() - parser.add_storage_url() parser.add_argument('file', metavar='<file>', type=argparse.FileType(mode='rb'), help='File to transfer') - parser.add_argument("--compression-threads", action="store", type=int, - default=1, metavar='<no>', - help='Number of parallel compression and encryption threads ' - 'to use (default: %(default)s).') + parser.add_cachedir() return parser.parse_args(args) @@ -64,72 +59,110 @@ def main(args=None): if args is None: args = sys.argv[1:] + args = ['local:///home/nikratio/tmp/bucket', '/home/nikratio/tmp/testfile0'] + options = parse_args(args) setup_logging(options) - with get_backend(options.storage_url, options.homedir, - options.ssl) as (conn, bucketname): - - if not bucketname in conn: - raise QuietError("Bucket does not exist.") - bucket = conn.get_bucket(bucketname) - - ifh = options.testfile - ofh = open('/dev/null', 'r+b') - size = os.fstat(ifh.fileno()).st_size / 1024 - log.info('Test file size: %.2f MB', (size / 1024)) - - log.info('Compressing with LZMA...') - stamp = time.time() - compress_encrypt_fh(ifh, ofh, 'foobar', 'nonce', - lzma.LZMACompressor(options={ 'level': 7 })) - seconds = time.time() - stamp - lzma_speed = size / seconds - log.info('done. LZMA Compression Speed: %.2f KB per second', lzma_speed) - - log.info('Compressing with BZip2...') - ifh.seek(0) - stamp = time.time() - compress_encrypt_fh(ifh, ofh, 'foobar', 'nonce', - bz2.BZ2Compressor(9)) - seconds = time.time() - stamp - bzip2_speed = size / seconds - log.info('done. Bzip2 Compression Speed: %.2f KB per second', bzip2_speed) - - log.info('Compressing with zlib...') - ifh.seek(0) - stamp = time.time() - compress_encrypt_fh(ifh, ofh, 'foobar', 'nonce', - zlib.compressobj(9)) - seconds = time.time() - stamp - zlib_speed = size / seconds - log.info('done. zlib Compression Speed: %.2f KB per second', zlib_speed) - - log.info('Transferring to backend...') - ifh.seek(0) - stamp = time.time() - bucket.raw_store(options.testfile, ifh, dict()) - seconds = time.time() - stamp - net_speed = size / seconds - log.info('done. Network Uplink Speed: %.2f KB per second', net_speed) - - - print('Assuming mount.s3ql will be called with --compression-threads %d' - % options.compression_threads) - lzma_speed *= options.compression_threads - bzip2_speed *= options.compression_threads - zlib_speed *= options.compression_threads + size = 100*1024*1024 # KB + log.info('Measuring throughput to cache...') + bucket_dir = tempfile.mkdtemp() + mnt_dir = tempfile.mkdtemp() + atexit.register(shutil.rmtree, bucket_dir) + atexit.register(shutil.rmtree, mnt_dir) + subprocess.check_call(['mkfs.s3ql', '--plain', 'local://%s' % bucket_dir, + '--quiet', '--cachedir', options.cachedir]) + subprocess.check_call(['mount.s3ql', '--threads', '1', '--quiet', + '--cachesize', '%d' % (2 * size/1024), '--log', + '%s/mount.log' % bucket_dir, '--cachedir', options.cachedir, + 'local://%s' % bucket_dir, mnt_dir]) + with open('/dev/urandom', 'rb', 0) as src: + with open('%s/bigfile' % mnt_dir, 'wb', 0) as dst: + stamp = time.time() + copied = 0 + while copied < size: + buf = src.read(256*1024) + dst.write(buf) + copied += len(buf) + fuse_speed = copied / (time.time() - stamp) + os.unlink('%s/bigfile' % mnt_dir) + subprocess.check_call(['umount.s3ql', mnt_dir]) + log.info('Cache throughput: %.2f KB/sec', fuse_speed / 1024) + + # Upload random data to prevent effects of compression + # on the network layer + log.info('Measuring raw backend throughput..') + bucket = get_bucket(options, plain=True) + with bucket.open_write('s3ql_testdata') as dst: + with open('/dev/urandom', 'rb', 0) as src: + stamp = time.time() + copied = 0 + while copied < size: + buf = src.read(256*1024) + dst.write(buf) + copied += len(buf) + upload_speed = copied / (time.time() - stamp) + log.info('Backend throughput: %.2f KB/sec', upload_speed / 1024) + bucket.delete('s3ql_testdata') - if lzma_speed > net_speed: - print('You should use LZMA compression.') - elif bzip2_speed > net_speed: - print('You should use BZip2 compression.') - elif zlib_speed > net_speed: - print('You should use zlib compression.') - else: - print('You should use zlib compression, but even that is not fast\n' - 'enough to saturate your network connection.') - + src = options.file + size = os.fstat(options.file.fileno()).st_size + log.info('Test file size: %.2f MB', (size / 1024**2)) + + times = dict() + out_sizes = dict() + for alg in ('lzma', 'bzip2', 'zlib'): + log.info('compressing with %s...', alg) + bucket = BetterBucket('pass', alg, Bucket(bucket_dir, None, None)) + with bucket.open_write('s3ql_testdata') as dst: + src.seek(0) + stamp = time.time() + while True: + buf = src.read(256*1024) + if not buf: + break + dst.write(buf) + times[alg] = time.time() - stamp + out_sizes[alg] = dst.compr_size + log.info('%s compression speed: %.2f KB/sec (in)', alg, size/times[alg]/1024) + log.info('%s compression speed: %.2f KB/sec (out)', alg, + out_sizes[alg] / times[alg] / 1024) + + print('') + + req = dict() + for alg in ('lzma', 'bzip2', 'zlib'): + backend_req = math.ceil(upload_speed * times[alg] / out_sizes[alg]) + fuse_req = math.ceil(fuse_speed * times[alg] / size) + req[alg] = min(backend_req, fuse_req) + print('When using %s compression, incoming writes can keep up to %d threads\n' + 'busy. The backend can handle data from up to %d threads. Therefore,\n' + 'the maximum achievable throughput is %.2f KB/sec with %d threads.\n' + % (alg, fuse_req, backend_req, min(upload_speed, fuse_speed)/1024, req[alg])) + + print('All numbers assume that the test file is representative and that', + 'there are enough processor cores to run all threads in parallel.', + 'To compensate for network latency, you should start about twice as', + 'many upload threads as you need for optimal performance.\n', sep='\n') + + cores = os.sysconf('SC_NPROCESSORS_ONLN') + best_size = None + max_threads = cores + while best_size is None: + for alg in out_sizes: + if req[alg] > max_threads: + continue + if best_size is None or out_sizes[alg] < best_size: + best_size = out_sizes[alg] + best_alg = alg + threads = req[alg] + + max_threads = min(req.itervalues()) + + print('This system appears to have %d cores, so best performance with maximum\n' + 'compression ratio would be achieved by using %s compression with %d\n' + 'upload threads.' % (cores, best_alg, + 2 * threads if cores >= threads else 2 * cores)) if __name__ == '__main__': main(sys.argv[1:]) diff --git a/contrib/expire_backups.1 b/contrib/expire_backups.1 index a6e2b3f..188191f 100644 --- a/contrib/expire_backups.1 +++ b/contrib/expire_backups.1 @@ -1,4 +1,4 @@ -.TH "EXPIRE_BACKUPS" "1" "May 20, 2011" "1.0.1" "S3QL" +.TH "EXPIRE_BACKUPS" "1" "September 28, 2011" "1.2" "S3QL" .SH NAME expire_backups \- Intelligently expire old backups . @@ -57,23 +57,17 @@ definition \fB1 3 7 14 31\fP, it will guarantee that you always have the following backups available: .INDENT 0.0 .IP 1. 3 -. A backup that is 0 to 1 cycles old (i.e, the most recent backup) .IP 2. 3 -. A backup that is 1 to 3 cycles old .IP 3. 3 -. A backup that is 3 to 7 cycles old .IP 4. 3 -. A backup that is 7 to 14 cycles old .IP 5. 3 -. A backup that is 14 to 31 cycles old .UNINDENT .IP Note -. If you do backups in fixed intervals, then one cycle will be equivalent to the backup interval. The advantage of specifying the age ranges in terms of backup cycles rather than days or weeks is @@ -122,33 +116,26 @@ The \fBexpire_backups\fP command accepts the following options: .INDENT 0.0 .TP .B \-\-quiet -. be really quiet .TP .B \-\-debug -. activate debugging output .TP .B \-\-version -. just print program version and exit .TP .BI \-\-state \ <file> -. File to save state information in (default: ".expire_backups.dat") .TP .B \-n -. Dry run. Just show which backups would be deleted. .TP .B \-\-reconstruct\-state -. Try to reconstruct a missing state file from backup dates. .TP .B \-\-use\-s3qlrm -. Use \fBs3qlrm\fP command to delete backups. .UNINDENT .UNINDENT diff --git a/contrib/expire_backups.py b/contrib/expire_backups.py index 971e21b..20b78a7 100755 --- a/contrib/expire_backups.py +++ b/contrib/expire_backups.py @@ -4,7 +4,7 @@ expire_backups.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import diff --git a/contrib/make_dummy.py b/contrib/make_dummy.py index e2b4396..ddd3414 100755 --- a/contrib/make_dummy.py +++ b/contrib/make_dummy.py @@ -9,7 +9,7 @@ will be copied and all files contain just \0 bytes. --- Copyright (C) 2010 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' @@ -42,7 +42,7 @@ def parse_args(args): 'contain a file system with the same structure, but all files' 'will just contain \\0 bytes.') - parser.add_homedir() + parser.add_authfile() parser.add_quiet() parser.add_debug_modules() parser.add_version() @@ -67,7 +67,7 @@ def main(args=None): options = parse_args(args) setup_logging(options) - with get_backend(options.src, options.homedir, + with get_backend(options.src, options.authfile, options.ssl) as (src_conn, src_name): if not src_name in src_conn: @@ -75,11 +75,12 @@ def main(args=None): src_bucket = src_conn.get_bucket(src_name) try: - unlock_bucket(options.homedir, options.src, src_bucket) + unlock_bucket(options.authfile, options.src, src_bucket) except ChecksumError: raise QuietError('Checksum error - incorrect password?') - with get_backend(options.dest, options.homedir) as (dest_conn, dest_name): + with get_backend(options.dest, options.authfile, + options.ssl) as (dest_conn, dest_name): if dest_name in dest_conn: raise QuietError("Bucket already exists!\n" diff --git a/contrib/pcp.1 b/contrib/pcp.1 index 9dc75ec..61bab55 100644 --- a/contrib/pcp.1 +++ b/contrib/pcp.1 @@ -1,4 +1,4 @@ -.TH "PCP" "1" "May 20, 2011" "1.0.1" "S3QL" +.TH "PCP" "1" "September 28, 2011" "1.2" "S3QL" .SH NAME pcp \- Recursive, parallel copy of directory trees . @@ -43,6 +43,10 @@ The \fBpcp\fP command is a is a wrapper that starts several \fBsync\fP processes to copy directory trees in parallel. This is allows much better copying performance on file system that have relatively high latency when retrieving individual files like S3QL. +.sp +\fBNote\fP: Using this program only improves performance when copying +\fIfrom\fP an S3QL file system. When copying \fIto\fP an S3QL file system, +using \fBpcp\fP is more likely to \fIdecrease\fP performance. .SH OPTIONS .sp The \fBpcp\fP command accepts the following options: @@ -51,23 +55,18 @@ The \fBpcp\fP command accepts the following options: .INDENT 0.0 .TP .B \-\-quiet -. be really quiet .TP .B \-\-debug -. activate debugging output .TP .B \-\-version -. just print program version and exit .TP .B \-a -. Pass \-aHAX option to rsync. .TP .BI \-\-processes \ <no> -. Number of rsync processes to use (default: 10). .UNINDENT .UNINDENT diff --git a/contrib/pcp.py b/contrib/pcp.py index fa015dc..17fd6d0 100755 --- a/contrib/pcp.py +++ b/contrib/pcp.py @@ -7,7 +7,7 @@ Parallel, recursive copy of directory trees. --- Copyright (C) 2010 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import diff --git a/doc/html/.buildinfo b/doc/html/.buildinfo index 7d05252..19a9de7 100644 --- a/doc/html/.buildinfo +++ b/doc/html/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: 85f6a556ea11bab9efd893c7b6e319a1 +config: 511abf8af0910e88e7d158947974ebc2 tags: fbb0d17656682115ca4d033fb2f83ba1 diff --git a/doc/html/_sources/about.txt b/doc/html/_sources/about.txt index bef3684..d3fc8a5 100644 --- a/doc/html/_sources/about.txt +++ b/doc/html/_sources/about.txt @@ -4,13 +4,15 @@ About S3QL ============ -S3QL is a file system that stores all its data online. It supports -`Amazon S3 <http://aws.amazon.com/s3 Amazon S3>`_ as well as arbitrary -SFTP servers and effectively provides you with a hard disk of dynamic, -infinite capacity that can be accessed from any computer with internet -access. - -S3QL is providing a standard, full featured UNIX file system that is +S3QL is a file system that stores all its data online using storage +services like `Google Storage +<http://code.google.com/apis/storage/>`_, `Amazon S3 +<http://aws.amazon.com/s3 Amazon S3>`_ or `OpenStack +<http://openstack.org/projects/storage/>`_. S3QL effectively provides +a hard disk of dynamic, infinite capacity that can be accessed from +any computer with internet access. + +S3QL is a standard conforming, full featured UNIX file system that is conceptually indistinguishable from any local file system. Furthermore, S3QL has additional features like compression, encryption, data de-duplication, immutable trees and snapshotting diff --git a/doc/html/_sources/adm.txt b/doc/html/_sources/adm.txt index 3e50f64..34fc7b4 100644 --- a/doc/html/_sources/adm.txt +++ b/doc/html/_sources/adm.txt @@ -14,7 +14,7 @@ The syntax is :: s3qladm [options] <action> <storage-url> where :var:`action` may be either of :program:`passphrase`, -:program:`upgrade`, :program:`delete` or :program:`download-metadata`. +:program:`upgrade`, :program:`clear` or :program:`download-metadata`. The :program:`s3qladm` accepts the following general options, no matter what specific action is being invoked: @@ -55,7 +55,7 @@ Deleting a file system A file system can be deleted with:: - s3qladm delete <storage url> + s3qladm clear <storage url> This physically deletes all the data and file system structures. diff --git a/doc/html/_sources/backends.txt b/doc/html/_sources/backends.txt index 480ff90..5bc9521 100644 --- a/doc/html/_sources/backends.txt +++ b/doc/html/_sources/backends.txt @@ -1,292 +1,205 @@ .. -*- mode: rst -*- +.. _storage_backends: + ================== Storage Backends ================== -S3QL can use different protocols to store the file system data. -Independent of the backend that you use, the place where your file -system data is being stored is called a *bucket*. (This is mostly for -historical reasons, since initially S3QL supported only the Amazon S3 -backend). - - -On Backend Reliability -====================== - -S3QL has been designed for use with a storage backend where data loss -is so infrequent that it can be completely neglected (e.g. the Amazon -S3 backend). If you decide to use a less reliable backend, you should -keep the following warning in mind and read this section carefully. - -.. WARNING:: - - S3QL is not able to compensate for any failures of the backend. In - particular, it is not able reconstruct any data that has been lost - or corrupted by the backend. The persistence and durability of data - stored in an S3QL file system is limited and determined by the - backend alone. - - -On the plus side, if a backend looses or corrupts some of the stored -data, S3QL *will* detect the problem. Missing data will be detected -when running `fsck.s3ql` or when attempting to access the data in the -mounted file system. In the later case you will get an IO Error, and -on unmounting S3QL will warn you that the file system is damaged and -you need to run `fsck.s3ql`. - -`fsck.s3ql` will report all the affected files and move them into the -`/lost+found` directory of the file system. - -You should be aware that, because of S3QL's data de-duplication -feature, the consequences of a data loss in the backend can be -significantly more severe than you may expect. More concretely, a data -loss in the backend at time *x* may cause data that is written *after* -time *x* to be lost as well. What may happen is this: - -#. You store an important file in the S3QL file system. -#. The backend looses the data blocks of this file. As long as you - do not access the file or run `fsck.s3ql`, S3QL - is not aware that the data has been lost by the backend. -#. You save an additional copy of the important file in a different - location on the same S3QL file system. -#. S3QL detects that the contents of the new file are identical to the - data blocks that have been stored earlier. Since at this point S3QL - is not aware that these blocks have been lost by the backend, it - does not save another copy of the file contents in the backend but - relies on the (presumably) existing blocks instead. -#. Therefore, even though you saved another copy, you still do not - have a backup of the important file (since both copies refer to the - same data blocks that have been lost by the backend). - -As one can see, this effect becomes the less important the more often -one runs `fsck.s3ql`, since `fsck.s3ql` will make S3QL aware of any -blocks that the backend may have lost. Figuratively, this establishes -a "checkpoint": data loss in the backend that occurred before running -`fsck.s3ql` can not affect any file system operations performed after -running `fsck.s3ql`. - - -Nevertheless, (as said at the beginning) the recommended way to use -S3QL is in combination with a sufficiently reliable storage backend. -In that case none of the above will ever be a concern. - - -The `authinfo` file -=================== - -Most backends first try to read the file `~/.s3ql/authinfo` to determine -the username and password for connecting to the remote host. If this -fails, both username and password are read from the terminal. - -The `authinfo` file has to contain entries of the form :: - - backend <backend> machine <host> login <user> password <password> - -So to use the login `joe` with password `jibbadup` when using the FTP -backend to connect to the host `backups.joesdomain.com`, you would -specify :: - - backend ftp machine backups.joesdomain.com login joe password jibbadup +The following backends are currently available in S3QL: + +Google Storage +============== + +`Google Storage <http://code.google.com/apis/storage/>`_ is an online +storage service offered by Google. It is the most feature-rich service +supported by S3QL and S3QL offers the best performance when used with +the Google Storage backend. + +To use the Google Storage backend, you need to have (or sign up for) a +Google account, and then `activate Google Storage +<http://code.google.com/apis/storage/docs/signup.html>`_ for your +account. The account is free, you will pay only for the amount of +storage and traffic that you actually use. Once you have created the +account, make sure to `activate legacy access +<http://code.google.com/apis/storage/docs/reference/v1/apiversion1.html#enabling>`_. + +To create a Google Storage bucket, you can use e.g. the `Google +Storage Manager +<https://sandbox.google.com/storage/>`_. The +storage URL for accessing the bucket in S3QL is then :: + + gs://<bucketname>/<prefix> + +Here *bucketname* is the name of the bucket, and *prefix* can be +an arbitrary prefix that will be prepended to all object names used by +S3QL. This allows you to store several S3QL file systems in the same +Google Storage bucket. + +Note that the backend login and password for accessing your Google +Storage bucket are not your Google account name and password, but the +*Google Storage developer access key* and *Google Storage developer +secret* that you can manage with the `Google Storage key management +tool +<https://code.google.com/apis/console/#:storage:legacy>`_. + +If you would like S3QL to connect using HTTPS instead of standard +HTTP, start the storage url with ``gss://`` instead of ``gs://``. Note +that at this point S3QL does not perform any server certificate +validation (see `issue 267 +<http://code.google.com/p/s3ql/issues/detail?id=267>`_). + + +Amazon S3 +========= + +`Amazon S3 <http://aws.amazon.com/s3>`_ is the online storage service +offered by `Amazon Web Services (AWS) <http://aws.amazon.com/>`_. To +use the S3 backend, you first need to sign up for an AWS account. The +account is free, you will pay only for the amount of storage and +traffic that you actually use. After that, you need to create a bucket +that will hold the S3QL file system, e.g. using the `AWS Management +Console <https://console.aws.amazon.com/s3/home>`_. For best +performance, it is recommend to create the bucket in the +geographically closest storage region, but not the US Standard +region (see below). + +The storage URL for accessing S3 buckets in S3QL has the form :: + + s3://<bucketname>/<prefix> + +Here *bucketname* is the name of the bucket, and *prefix* can be +an arbitrary prefix that will be prepended to all object names used by +S3QL. This allows you to store several S3QL file systems in the same +S3 bucket. + +Note that the backend login and password for accessing S3 are not the +user id and password that you use to log into the Amazon Webpage, but +the *AWS access key id* and *AWS secret access key* shown under `My +Account/Access Identifiers +<https://aws-portal.amazon.com/gp/aws/developer/account/index.html?ie=UTF8&action=access-key>`_. - -Consistency Guarantees -====================== - -The different backends provide different types of *consistency -guarantees*. Informally, a consistency guarantee tells you how fast -the backend will apply changes to the stored data. +If you would like S3QL to connect using HTTPS instead of standard +HTTP, start the storage url with ``s3s://`` instead of ``s3://``. Note +that, as of May 2011, Amazon S3 is faster when accessed using a +standard HTTP connection, and that S3QL does not perform any server +certificate validation (see `issue 267 +<http://code.google.com/p/s3ql/issues/detail?id=267>`_). -S3QL defines the following three levels: -* **Read-after-Write Consistency.** This is the strongest consistency - guarantee. If a backend offers read-after-write consistency, it - guarantees that as soon as you have committed any changes to the - backend, subsequent requests will take into account these changes. +Reduced Redundancy Storage (RRS) +-------------------------------- -* **Read-after-Create Consistency.** If a backend provides only - read-after-create consistency, only the creation of a new object is - guaranteed to be taken into account for subsequent requests. This - means that, for example, if you overwrite data in an existing - object, subsequent requests may still return the old data for a - certain period of time. +S3QL does not allow the use of `reduced redundancy storage +<http://aws.amazon.com/s3/#protecting>`_. The reason for that is a +combination of three factors: -* **Eventual consistency.** This is the lowest consistency level. - Basically, any changes that you make to the backend may not be - visible for a certain amount of time after the change has been made. - However, you are guaranteed that no change will be lost. All changes - will *eventually* become visible. - - . +* RRS has a relatively low reliability, on average you loose one + out of every ten-thousand objects a year. So you can expect to + occasionally loose some data. +* When `fsck.s3ql` asks S3 for a list of the stored objects, this list + includes even those objects that have been lost. Therefore + `fsck.s3ql` *can not detect lost objects* and lost data will only + become apparent when you try to actually read from a file whose data + has been lost. This is a (very unfortunate) peculiarity of Amazon + S3. -As long as your backend provides read-after-write or read-after-create -consistency, you do not have to worry about consistency guarantees at -all. However, if you plan to use a backend with only eventual -consistency, you have to be a bit careful in some situations. - - -.. _eventual_consistency: - -Dealing with Eventual Consistency ---------------------------------- +* Due to the data de-duplication feature of S3QL, unnoticed lost + objects may cause subsequent data loss later in time (see + :ref:`backend_reliability` for details). -.. NOTE:: - The following applies only to storage backends that do not provide - read-after-create or read-after-write consistency. Currently, - this is only the Amazon S3 backend *if used with the US-Standard - storage region*. If you use a different storage backend, or the S3 - backend with a different storage region, this section does not apply - to you. +Potential issues when using the US Standard storage region +---------------------------------------------------------- -While the file system is mounted, S3QL is able to automatically handle -all issues related to the weak eventual consistency guarantee. -However, some issues may arise during the mount process and when the -file system is checked. +In the US Standard storage region, Amazon S3 does not guarantee read +after create consistency. This means that after a new object has been +stored, requests to read this object may still fail for a little +while. While the file system is mounted, S3QL is able to automatically +handle all issues related to this so-called eventual consistency. +However, problems may arise during the mount process and when the file +system is checked: Suppose that you mount the file system, store some new data, delete -some old data and unmount it again. Now remember that eventual -consistency means that there is no guarantee that these changes will -be visible immediately. At least in theory it is therefore possible -that if you mount the file system again, S3QL does not see any of the -changes that you have done and presents you an "old version" of the -file system without them. Even worse, if you notice the problem and -unmount the file system, S3QL will upload the old status (which S3QL -necessarily has to consider as current) and thereby permanently -override the newer version (even though this change may not become -immediately visible either). - -The same problem applies when checking the file system. If the backend +some old data and unmount it again. Now there is no guarantee that +these changes will be visible immediately. At least in theory it is +therefore possible that if you mount the file system again, S3QL +does not see any of the changes that you have done and presents you +an "old version" of the file system without them. Even worse, if you +notice the problem and unmount the file system, S3QL will upload the +old status (which S3QL necessarily has to consider as current) and +thereby permanently override the newer version (even though this +change may not become immediately visible either). + +The same problem applies when checking the file system. If S3 provides S3QL with only partially updated data, S3QL has no way to find out if this a real consistency problem that needs to be fixed or if it is only a temporary problem that will resolve itself automatically (because there are still changes that have not become visible yet). -While this may seem to be a rather big problem, the likelihood of it -to occur is rather low. In practice, most storage providers rarely -need more than a few seconds to apply incoming changes, so to trigger -this problem one would have to unmount and remount the file system in -a very short time window. Many people therefore make sure that they -wait a few minutes between successive mounts (or file system checks) -and decide that the remaining risk is negligible. - -Nevertheless, the eventual consistency guarantee does not impose an -upper limit on the time that it may take for change to become visible. -Therefore there is no "totally safe" waiting time that would totally -eliminate this problem; a theoretical possibility always remains. - - - -The Amazon S3 Backend -===================== - -To store your file system in an Amazon S3 bucket, use a storage URL of -the form `s3://<bucketname>`. Bucket names must conform to the `S3 -Bucket Name Restrictions`_. - -The S3 backend offers exceptionally strong reliability guarantees. As -of August 2010, Amazon guarantees a durability of 99.999999999% per -year. In other words, if you store a thousand million objects then on -average you would loose less than one object in a hundred years. - -The Amazon S3 backend provides read-after-create consistency for the -EU, Asia-Pacific and US-West storage regions. *For the US-Standard -storage region, Amazon S3 provides only eventual consistency* (please -refer to :ref:`eventual_consistency` for information about -what this entails). - -When connecting to Amazon S3, S3QL uses an unencrypted HTTP -connection, so if you want your data to stay confidential, you have -to create the S3QL file system with encryption (this is also the default). - -When reading the authentication information for the S3 backend from -the `authinfo` file, the `host` field is ignored, i.e. the first entry -with `s3` as a backend will be used. For example :: - - backend s3 machine any login myAWSaccessKeyId password myAwsSecretAccessKey - -Note that the bucket names come from a global pool, so chances are -that your favorite name has already been taken by another S3 user. -Usually a longer bucket name containing some random numbers, like -`19283712_yourname_s3ql`, will work better. - -If you do not already have one, you need to obtain an Amazon S3 -account from `Amazon AWS <http://aws.amazon.com/>`_. The account is -free, you will pay only for the amount of storage that you actually -use. - -Note that the login and password for accessing S3 are not the user id -and password that you use to log into the Amazon Webpage, but the "AWS -access key id" and "AWS secret access key" shown under `My -Account/Access Identifiers -<https://aws-portal.amazon.com/gp/aws/developer/account/index.html?ie=UTF8&action=access-key>`_. - -.. _`S3 Bucket Name Restrictions`: http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/BucketRestrictions.html - -.. NOTE:: - - S3QL also allows you to use `reduced redundancy storage - <http://aws.amazon.com/s3/#protecting>`_ by using ``s3rr://`` - instead of ``s3://`` in the storage url. However, this not - recommended. The reason is a combination of three factors: - - * RRS has a relatively low reliability, on average you loose one - out of every ten-thousand objects a year. So you can expect to - occasionally loose some data. - - * When `fsck.s3ql` asks Amazon S3 for a list of the stored objects, - this list includes even those objects that have been lost. - Therefore `fsck.s3ql` *can not detect lost objects* and lost data - will only become apparent when you try to actually read from a - file whose data has been lost. This is a (very unfortunate) - peculiarity of Amazon S3. +The likelihood of this to happen is rather low. In practice, most +objects are ready for retrieval just a few seconds after they have +been stored, so to trigger this problem one would have to unmount and +remount the file system in a very short time window. However, since S3 +does not place any upper limit on the length of this window, it is +recommended to not place S3QL buckets in the US Standard storage +region. As of May 2011, all other storage regions provide stronger +consistency guarantees that completely eliminate any of the described +problems. - * Due to the data de-duplication feature of S3QL, unnoticed lost - objects may cause subsequent data loss later in time (see `On - Backend Reliability`_ for details). - In other words, you should really only store an S3QL file system - using RRS if you know exactly what you are getting into. - +S3 compatible +============= +S3QL is also able to access other, S3 compatible storage services for +which no specific backend exists. Note that when accessing such +services, only the lowest common denominator of available features can +be used, so it is generally recommended to use a service specific +backend instead. -The Local Backend -================= +The storage URL for accessing an arbitrary S3 compatible storage +service is :: -The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -`local://<path>`. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. `local:///var/archive`. + s3c://<hostname>:<port>/<bucketname>/<prefix> -The local backend provides read-after-write consistency. +or :: -The SFTP Backend -================ + s3cs://<hostname>:<port>/<bucketname>/<prefix> -The SFTP backend uses the SFTP protocol, which is a file transfer -protocol similar to ftp, but uses an encrypted SSH connection. -It provides read-after-write consistency. +to use HTTPS connections. Note, however, that at this point S3QL does +not verify the server certificate (cf. `issue 267 +<http://code.google.com/p/s3ql/issues/detail?id=267>`_). -Note that the SFTP backend is rather slow and has not been tested -as extensively as the S3 and Local backends. -The storage URL for SFTP connections has the form :: +Local +===== - sftp://<host>[:port]/<path> +S3QL is also able to store its data on the local file system. This can +be used to backup data on external media, or to access external +services that S3QL can not talk to directly (e.g., it is possible to +store data over SSH by first mounting the remote system using +`sshfs`_, then using the local backend to store the data in the sshfs +mountpoint). -The SFTP backend will always ask you for a password if you haven't -defined one in `~/.s3ql/authinfo`. However, public key authentication -is tried first and the password will only be used if the public key -authentication fails. +The storage URL for local storage is :: -The public and private keys will be read from the standard files in -`~/.ssh/`. Note that S3QL will refuse to connect to a computer with -unknown host key; to add the key to your local keyring you have to -establish a connection to that computer with the standard SSH command -line programs first. + local://<path> + +Note that you have to write three consecutive slashes to specify an +absolute path, e.g. `local:///var/archive`. Also, relative paths will +automatically be converted to absolute paths before the authentication +file is read, i.e. if you are in the `/home/john` directory and try to +mount `local://bucket`, the corresponding section in the +authentication file must match the storage url +`local:///home/john/bucket`. +SSH/SFTP +======== +Previous versions of S3QL included an SSH/SFTP backend. With newer +S3QL versions, it is recommended to instead combine the local backend +with `sshfs <http://fuse.sourceforge.net/sshfs.html>`_ (cf. :ref:`ssh_tipp`). diff --git a/doc/html/_sources/contrib.txt b/doc/html/_sources/contrib.txt index 3ee2323..7cbe997 100644 --- a/doc/html/_sources/contrib.txt +++ b/doc/html/_sources/contrib.txt @@ -14,8 +14,10 @@ the `contrib` directory of the source distribution or in benchmark.py ============ -This program measures your uplink bandwidth and compression speed and -recommends a compression algorithm for optimal throughput. +This program measures S3QL write performance, uplink bandwidth and +compression speed to determine the limiting factor. It also gives +recommendation for compression algorithm and number of upload threads +to achieve maximum performance. s3_copy.py diff --git a/doc/html/_sources/general.txt b/doc/html/_sources/general.txt new file mode 100644 index 0000000..e95ab8e --- /dev/null +++ b/doc/html/_sources/general.txt @@ -0,0 +1,149 @@ +.. -*- mode: rst -*- + +=================== +General Information +=================== + +Terminology +=========== + +S3QL can store data at different service providers and using different +protocols. The term *backend* refers to both the part of S3QL that +implements communication with a specific storage service and the +storage service itself. Most backends can hold more than one S3QL file +system and thus require some additional information that specifies the +file system location within the backend. This location is called a +*bucket* (for historical reasons). + +Many S3QL commands expect a *storage url* as a parameter. A storage +url specifies both the backend and the bucket and thus uniquely +identifies an S3QL file system. The form of the storage url depends on +the backend and is described together with the +:ref:`storage_backends`. + +.. _bucket_pw: + +Storing Authentication Information +================================== + +Normally, S3QL reads username and password for the backend as well as +an encryption passphrase for the bucket from the terminal. Most +commands also accept an :cmdopt:`--authfile` parameter that can be +used to read this information from a file instead. + +The authentication file consists of sections, led by a ``[section]`` +header and followed by ``name: value`` entries. The section headers +themselves are not used by S3QL but have to be unique within the file. + +In each section, the following entries can be defined: + +:storage-url: + Specifies the storage url to which this section applies. If a + storage url starts with the value of this entry, the section is + considered applicable. + +:backend-login: + Specifies the username to use for authentication with the backend. + +:backend-password: + Specifies the password to use for authentication with the backend. + +:bucket-passphrase: + Specifies the passphrase to use to decrypt the bucket (if it is + encrypted). + + +When reading the authentication file, S3QL considers every applicable +section in order and uses the last value that it found for each entry. +For example, consider the following authentication file:: + + [s3] + storage-url: s3:// + backend-login: joe + backend-password: notquitesecret + + [bucket1] + storage-url: s3://joes-first-bucket + bucket-passphrase: neitheristhis + + [bucket2] + storage-url: s3://joes-second-bucket + bucket-passphrase: swordfish + + [bucket3] + storage-url: s3://joes-second-bucket/with-prefix + backend-login: bill + backend-password: bi23ll + bucket-passphrase: ll23bi + +With this authentication file, S3QL would try to log in as "joe" +whenever the s3 backend is used, except when accessing a storage url +that begins with "s3://joes-second-bucket/with-prefix". In that case, +the last section becomes active and S3QL would use the "bill" +credentials. Furthermore, bucket encryption passphrases will be used +for storage urls that start with "s3://joes-first-bucket" or +"s3://joes-second-bucket". + +The authentication file is parsed by the `Python ConfigParser +module <http://docs.python.org/library/configparser.html>`_. + +.. _backend_reliability: + +On Backend Reliability +====================== + +S3QL has been designed for use with a storage backend where data loss +is so infrequent that it can be completely neglected (e.g. the Amazon +S3 backend). If you decide to use a less reliable backend, you should +keep the following warning in mind and read this section carefully. + +.. WARNING:: + + S3QL is not able to compensate for any failures of the backend. In + particular, it is not able reconstruct any data that has been lost + or corrupted by the backend. The persistence and durability of data + stored in an S3QL file system is limited and determined by the + backend alone. + + +On the plus side, if a backend looses or corrupts some of the stored +data, S3QL *will* detect the problem. Missing data will be detected +when running `fsck.s3ql` or when attempting to access the data in the +mounted file system. In the later case you will get an IO Error, and +on unmounting S3QL will warn you that the file system is damaged and +you need to run `fsck.s3ql`. + +`fsck.s3ql` will report all the affected files and move them into the +`/lost+found` directory of the file system. + +You should be aware that, because of S3QL's data de-duplication +feature, the consequences of a data loss in the backend can be +significantly more severe than you may expect. More concretely, a data +loss in the backend at time *x* may cause data that is written *after* +time *x* to be lost as well. What may happen is this: + +#. You store an important file in the S3QL file system. +#. The backend looses the data blocks of this file. As long as you + do not access the file or run `fsck.s3ql`, S3QL + is not aware that the data has been lost by the backend. +#. You save an additional copy of the important file in a different + location on the same S3QL file system. +#. S3QL detects that the contents of the new file are identical to the + data blocks that have been stored earlier. Since at this point S3QL + is not aware that these blocks have been lost by the backend, it + does not save another copy of the file contents in the backend but + relies on the (presumably) existing blocks instead. +#. Therefore, even though you saved another copy, you still do not + have a backup of the important file (since both copies refer to the + same data blocks that have been lost by the backend). + +As one can see, this effect becomes the less important the more often +one runs `fsck.s3ql`, since `fsck.s3ql` will make S3QL aware of any +blocks that the backend may have lost. Figuratively, this establishes +a "checkpoint": data loss in the backend that occurred before running +`fsck.s3ql` can not affect any file system operations performed after +running `fsck.s3ql`. + +Nevertheless, the recommended way to use S3QL is in combination with a +sufficiently reliable storage backend. In that case none of the above +will ever be a concern. diff --git a/doc/html/_sources/index.txt b/doc/html/_sources/index.txt index f3b5b72..960a63a 100644 --- a/doc/html/_sources/index.txt +++ b/doc/html/_sources/index.txt @@ -9,6 +9,7 @@ about installation + general backends mkfs adm diff --git a/doc/html/_sources/installation.txt b/doc/html/_sources/installation.txt index b57325e..e153de0 100644 --- a/doc/html/_sources/installation.txt +++ b/doc/html/_sources/installation.txt @@ -27,13 +27,11 @@ running S3QL. Generally, you should first check if your distribution already provides a suitable packages and only install from source if that is not the case. -* Kernel version 2.6.9 or newer. Starting with kernel 2.6.26 - you will get significantly better write performance, so you should - actually use *2.6.26 or newer whenever possible*. - -* The `FUSE Library <http://fuse.sourceforge.net/>`_ should already be - installed on your system. However, you have to make sure that you - have at least version 2.8.0. +* Kernel: Linux 2.6.9 or newer or FreeBSD with `FUSE4BSD + <http://www.freshports.org/sysutils/fusefs-kmod/>`_. Starting with + kernel 2.6.26 you will get significantly better write performance, + so under Linux you should actually use *2.6.26 or newer whenever + possible*. * The `PyCrypto++ Python Module <http://pypi.python.org/pypi/pycryptopp>`_. To check if this module @@ -69,12 +67,6 @@ that is not the case. this module. If you are upgrading from such a version, make sure to completely remove the old S3QL version first. -* If you want to use the SFTP backend, then you also need the - `Paramiko Python Module <http://www.lag.net/paramiko/>`_. To check - if this module is installed, try to execute `python -c 'import - paramiko'`. - - .. _inst-s3ql: Installing S3QL diff --git a/doc/html/_sources/issues.txt b/doc/html/_sources/issues.txt index 29b76ce..ac2cb8c 100644 --- a/doc/html/_sources/issues.txt +++ b/doc/html/_sources/issues.txt @@ -4,6 +4,11 @@ Known Issues ============ +* S3QL does not verify TLS/SSL server certificates, so a + man-in-the-middle attack is principally possible. See `issue 267 + <http://code.google.com/p/s3ql/issues/detail?id=267>`_ for more + details. + * S3QL is rather slow when an application tries to write data in unreasonably small chunks. If a 1 MB file is copied in chunks of 1 KB, this will take more than 10 times as long as when it's copied @@ -74,7 +79,7 @@ Known Issues * S3QL relies on the backends not to run out of space. This is a given for big storage providers like Amazon S3, but you may stumble upon - this if you store buckets e.g. on a small sftp server. + this if you store buckets e.g. on smaller servers or servies. If there is no space left in the backend, attempts to write more data into the S3QL file system will fail and the file system will be diff --git a/doc/html/_sources/man/adm.txt b/doc/html/_sources/man/adm.txt index c23865e..9010cd8 100644 --- a/doc/html/_sources/man/adm.txt +++ b/doc/html/_sources/man/adm.txt @@ -23,8 +23,8 @@ The |command| command performs various operations on S3QL buckets. The file system contained in the bucket *must not be mounted* when using |command| or things will go wrong badly. -.. include:: ../include/backends.rst - +The storage url depends on the backend that is used. The S3QL User's +Guide should be consulted for a description of the available backends. Options ======= @@ -52,15 +52,6 @@ download-metadata Interactively download backups of the file system metadata. -Files -===== - -Authentication data for backends and bucket encryption passphrases are -read from :file:`authinfo` in :file:`~/.s3ql` or the directory -specified with :cmdopt:`--homedir`. Log files are placed in the same -directory. - - .. include:: ../include/postman.rst .. |command| replace:: :program:`s3qladm` diff --git a/doc/html/_sources/man/cp.txt b/doc/html/_sources/man/cp.txt index d0cbb41..a397fe9 100644 --- a/doc/html/_sources/man/cp.txt +++ b/doc/html/_sources/man/cp.txt @@ -40,7 +40,7 @@ Note that: usage of `s3qlcp` is to regularly duplicate the same source directory, say `documents`, to different target directories. For a e.g. monthly replication, the target directories would typically be - named something like `documents_Januray` for the replication in + named something like `documents_January` for the replication in January, `documents_February` for the replication in February etc. In this case it is clear that the target directories should be regarded as snapshots of the source directory. @@ -51,12 +51,6 @@ Note that: completely (so that S3QL had to fetch all the data over the network from the backend) before writing them into the destination folder. -* Before starting with the replication, S3QL has to flush the local - cache. So if you just copied lots of new data into the file system - that has not yet been uploaded, replication will take longer than - usual. - - Snapshotting vs Hardlinking --------------------------- diff --git a/doc/html/_sources/man/ctrl.txt b/doc/html/_sources/man/ctrl.txt index 4afa33b..8173162 100644 --- a/doc/html/_sources/man/ctrl.txt +++ b/doc/html/_sources/man/ctrl.txt @@ -24,6 +24,12 @@ Description The |command| command performs various actions on the S3QL file system mounted in :var:`mountpoint`. +|command| can only be called by the user that mounted the file system +and (if the file system was mounted with :cmdopt:`--allow-other` or +:cmdopt:`--allow-root`) the root user. This limitation might be +removed in the future (see `issue 155 +<http://code.google.com/p/s3ql/issues/detail?id=155>`_). + The following actions may be specified: flushcache diff --git a/doc/html/_sources/man/fsck.txt b/doc/html/_sources/man/fsck.txt index ef6ed2d..8f901c7 100644 --- a/doc/html/_sources/man/fsck.txt +++ b/doc/html/_sources/man/fsck.txt @@ -18,10 +18,9 @@ Description The |command| command checks the new file system in the location specified by *storage url* for errors and attempts to repair any -problems. - -.. include:: ../include/backends.rst - +problems. The storage url depends on the backend that is used. The +S3QL User's Guide should be consulted for a description of the +available backends. Options ======= @@ -31,14 +30,6 @@ The |command| command accepts the following options. .. pipeinclude:: ../../bin/fsck.s3ql --help :start-after: show this help message and exit -Files -===== - -Authentication data for backends and bucket encryption passphrases are -read from :file:`authinfo` in :file:`~/.s3ql` or the directory -specified with :cmdopt:`--homedir`. Log files are placed in the same -directory. - .. include:: ../include/postman.rst .. |command| replace:: :command:`mkfs.s3ql` diff --git a/doc/html/_sources/man/lock.txt b/doc/html/_sources/man/lock.txt index f17bf32..19c3e3a 100644 --- a/doc/html/_sources/man/lock.txt +++ b/doc/html/_sources/man/lock.txt @@ -23,6 +23,12 @@ whatsoever. You can not add new files or directories and you can not change or delete existing files and directories. The only way to get rid of an immutable tree is to use the :program:`s3qlrm` command. +|command| can only be called by the user that mounted the file system +and (if the file system was mounted with :cmdopt:`--allow-other` or +:cmdopt:`--allow-root`) the root user. This limitation might be +removed in the future (see `issue 155 +<http://code.google.com/p/s3ql/issues/detail?id=155>`_). + Rationale ========= diff --git a/doc/html/_sources/man/mkfs.txt b/doc/html/_sources/man/mkfs.txt index c61270a..3a61717 100644 --- a/doc/html/_sources/man/mkfs.txt +++ b/doc/html/_sources/man/mkfs.txt @@ -17,10 +17,15 @@ Description .. include:: ../include/about.rst The |command| command creates a new file system in the location -specified by *storage url*. +specified by *storage url*. The storage url depends on the backend +that is used. The S3QL User's Guide should be consulted for a +description of the available backends. + +Unless you have specified the `--plain` option, `mkfs.s3ql` will ask +you to enter an encryption password. This password will *not* be read +from an authentication file specified with the :cmdopt:`--authfile` +option to prevent accidental creation of an encrypted bucket. -.. include:: ../include/backends.rst - Options ======= @@ -30,13 +35,6 @@ The |command| command accepts the following options. .. pipeinclude:: ../../bin/mkfs.s3ql --help :start-after: show this help message and exit -Files -===== - -Authentication data for backends and bucket encryption passphrases are -read from :file:`authinfo` in :file:`~/.s3ql` or the directory -specified with :cmdopt:`--homedir`. Log files are placed in the same -directory. .. include:: ../include/postman.rst diff --git a/doc/html/_sources/man/mount.txt b/doc/html/_sources/man/mount.txt index 3905c03..fb10a11 100644 --- a/doc/html/_sources/man/mount.txt +++ b/doc/html/_sources/man/mount.txt @@ -19,10 +19,10 @@ Description .. include:: ../include/about.rst The |command| command mounts the S3QL file system stored in *storage -url* in the directory *mount point*. +url* in the directory *mount point*. The storage url depends on the +backend that is used. The S3QL User's Guide should be consulted for a +description of the available backends. -.. include:: ../include/backends.rst - Options ======= @@ -33,14 +33,6 @@ The |command| command accepts the following options. :start-after: show this help message and exit -Files -===== - -Authentication data for backends and bucket encryption passphrases are -read from :file:`authinfo` in :file:`~/.s3ql` or the directory -specified with :cmdopt:`--homedir`. Log files are placed in the same -directory. - .. include:: ../include/postman.rst diff --git a/doc/html/_sources/man/pcp.txt b/doc/html/_sources/man/pcp.txt index cd7a66c..c7b3bef 100644 --- a/doc/html/_sources/man/pcp.txt +++ b/doc/html/_sources/man/pcp.txt @@ -21,6 +21,10 @@ The |command| command is a is a wrapper that starts several allows much better copying performance on file system that have relatively high latency when retrieving individual files like S3QL. +**Note**: Using this program only improves performance when copying +*from* an S3QL file system. When copying *to* an S3QL file system, +using |command| is more likely to *decrease* performance. + Options ======= diff --git a/doc/html/_sources/man/rm.txt b/doc/html/_sources/man/rm.txt index 0832e27..dda1eda 100644 --- a/doc/html/_sources/man/rm.txt +++ b/doc/html/_sources/man/rm.txt @@ -25,6 +25,12 @@ you to delete immutable trees (which can be created with Be warned that there is no additional confirmation. The directory will be removed entirely and immediately. + +|command| can only be called by the user that mounted the file system +and (if the file system was mounted with :cmdopt:`--allow-other` or +:cmdopt:`--allow-root`) the root user. This limitation might be +removed in the future (see `issue 155 +<http://code.google.com/p/s3ql/issues/detail?id=155>`_). Options diff --git a/doc/html/_sources/mkfs.txt b/doc/html/_sources/mkfs.txt index 0b9fa97..3eceb0a 100644 --- a/doc/html/_sources/mkfs.txt +++ b/doc/html/_sources/mkfs.txt @@ -14,7 +14,8 @@ This command accepts the following options: .. pipeinclude:: ../bin/mkfs.s3ql --help :start-after: show this help message and exit -Unless you have specified the `--plain` option, `mkfs.s3ql` will ask you -to enter an encryption password. If you do not want to enter this -password every time that you mount the file system, you can store it -in the `~/.s3ql/authinfo` file, see :ref:`bucket_pw`. +Unless you have specified the `--plain` option, `mkfs.s3ql` will ask +you to enter an encryption password. This password will *not* be read +from an authentication file specified with the :cmdopt:`--authfile` +option to prevent accidental creation of an encrypted bucket. + diff --git a/doc/html/_sources/mount.txt b/doc/html/_sources/mount.txt index 609c4a4..4f220cd 100644 --- a/doc/html/_sources/mount.txt +++ b/doc/html/_sources/mount.txt @@ -22,33 +22,6 @@ This command accepts the following options: .. pipeinclude:: ../bin/mount.s3ql --help :start-after: show this help message and exit -.. _bucket_pw: - -Storing Encryption Passwords -============================ - -If you are trying to mount an encrypted bucket, `mount.s3ql` will first -try to read the password from the `.s3ql/authinfo` file (the same file -that is used to read the backend authentication data) and prompt the -user to enter the password only if this fails. - -The `authinfo` entries to specify bucket passwords are of the form :: - - storage-url <storage-url> password <password> - -So to always use the password `topsecret` when mounting `s3://joes_bucket`, -the entry would be :: - - storage-url s3://joes_bucket password topsecret - -.. NOTE:: - - If you are using the local backend, the storage url will - always be converted to an absolute path. So if you are in the - `/home/john` directory and try to mount `local://bucket`, the matching - `authinfo` entry has to have a storage url of - `local:///home/john/bucket`. - Compression Algorithms ====================== @@ -83,9 +56,8 @@ Parallel Compression ==================== If you are running S3QL on a system with multiple cores, you might -want to set ``--compression-threads`` to a value bigger than one. This -will instruct S3QL to compress and encrypt several blocks at the same -time. +want to set the ``--threads`` value larger than one. This will +instruct S3QL to compress and encrypt several blocks at the same time. If you want to do this in combination with using the LZMA compression algorithm, you should keep an eye on memory usage though. Every diff --git a/doc/html/_sources/special.txt b/doc/html/_sources/special.txt index c5acade..a217a93 100644 --- a/doc/html/_sources/special.txt +++ b/doc/html/_sources/special.txt @@ -11,8 +11,8 @@ Snapshotting and Copy-on-Write ============================== The command `s3qlcp` can be used to duplicate a directory tree without -physically copying the file contents. This is possible due to the data -de-duplication feature of S3QL. +physically copying the file contents. This is made possible by the +data de-duplication feature of S3QL. The syntax of `s3qlcp` is:: diff --git a/doc/html/_sources/tips.txt b/doc/html/_sources/tips.txt index b857f75..705bc20 100644 --- a/doc/html/_sources/tips.txt +++ b/doc/html/_sources/tips.txt @@ -4,8 +4,21 @@ Tips & Tricks ============= +.. _ssh_tipp: -.. _copy_performance: +SSH Backend +=========== + +By combining S3QL's local backend with `sshfs +<http://fuse.sourceforge.net/sshfs.html>`_, it is possible to store an +S3QL file system on arbitrary SSH servers: first mount the remote +target directory into the local filesystem, :: + + sshfs user@my.server.com:/mnt/s3ql /mnt/sshfs + +and then give the mountpoint to S3QL as a local destination:: + + mount.s3ql local:///mnt/sshfs/mybucket /mnt/s3ql Permanently mounted backup file system @@ -28,11 +41,16 @@ If you decide to do so, you should make sure to :cmdopt:`--metadata-upload-interval` option of :program:`mount.s3ql` to zero). - +.. _copy_performance: Improving copy performance ========================== +.. NOTE:: + + The following applies only when copying data **from** an S3QL file + system, **not** when copying data **to** an S3QL file system. + If you want to copy a lot of smaller files *from* an S3QL file system (e.g. for a system restore) you will probably notice that the performance is rather bad. @@ -44,6 +62,7 @@ network latency), no matter how big or small the file is. So when you copy lots of small files, 99% of the time is actually spend waiting for network data. + Theoretically, this problem is easy to solve: you just have to copy several files at the same time. In practice, however, almost all unix utilities (``cp``, ``rsync``, ``tar`` and friends) insist on copying diff --git a/doc/html/about.html b/doc/html/about.html index f2af2ae..e50b15b 100644 --- a/doc/html/about.html +++ b/doc/html/about.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>About S3QL — S3QL 1.0.1 documentation</title> + <title>About S3QL — S3QL 1.2 documentation</title> <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="_static/underscore.js"></script> <script type="text/javascript" src="_static/doctools.js"></script> <link rel="author" title="About these documents" href="#" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="index.html" /> <link rel="next" title="Installation" href="installation.html" /> <link rel="prev" title="S3QL User’s Guide" href="index.html" /> </head> @@ -40,7 +40,7 @@ <li class="right" > <a href="index.html" title="S3QL User’s Guide" accesskey="P">previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -53,6 +53,7 @@ </ul> </li> <li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="adm.html">Managing Buckets</a></li> @@ -91,12 +92,11 @@ <div class="section" id="about-s3ql"> <h1>About S3QL<a class="headerlink" href="#about-s3ql" title="Permalink to this headline">¶</a></h1> -<p>S3QL is a file system that stores all its data online. It supports -<a class="reference external" href="http://aws.amazon.com/s3AmazonS3">Amazon S3</a> as well as arbitrary -SFTP servers and effectively provides you with a hard disk of dynamic, -infinite capacity that can be accessed from any computer with internet -access.</p> -<p>S3QL is providing a standard, full featured UNIX file system that is +<p>S3QL is a file system that stores all its data online using storage +services like <a class="reference external" href="http://code.google.com/apis/storage/">Google Storage</a>, <a class="reference external" href="http://aws.amazon.com/s3AmazonS3">Amazon S3</a> or <a class="reference external" href="http://openstack.org/projects/storage/">OpenStack</a>. S3QL effectively provides +a hard disk of dynamic, infinite capacity that can be accessed from +any computer with internet access.</p> +<p>S3QL is a standard conforming, full featured UNIX file system that is conceptually indistinguishable from any local file system. Furthermore, S3QL has additional features like compression, encryption, data de-duplication, immutable trees and snapshotting @@ -185,7 +185,7 @@ will, although being inconvenient, not endanger any stored data.</p> <li class="right" > <a href="index.html" title="S3QL User’s Guide" >previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/html/adm.html b/doc/html/adm.html index cda4433..64bc811 100644 --- a/doc/html/adm.html +++ b/doc/html/adm.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>Managing Buckets — S3QL 1.0.1 documentation</title> + <title>Managing Buckets — S3QL 1.2 documentation</title> <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="_static/underscore.js"></script> <script type="text/javascript" src="_static/doctools.js"></script> <link rel="author" title="About these documents" href="about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="index.html" /> <link rel="next" title="Mounting" href="mount.html" /> <link rel="prev" title="File System Creation" href="mkfs.html" /> </head> @@ -40,7 +40,7 @@ <li class="right" > <a href="mkfs.html" title="File System Creation" accesskey="P">previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -49,6 +49,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> <li class="toctree-l1 current"><a class="current reference internal" href="">Managing Buckets</a><ul> @@ -93,15 +94,15 @@ <div class="section" id="managing-buckets"> <h1>Managing Buckets<a class="headerlink" href="#managing-buckets" title="Permalink to this headline">¶</a></h1> -<p>The <tt class=" docutils literal"><span class="pre">s3qladm</span></tt> command performs various operations on S3QL buckets. +<p>The <tt class="docutils literal"><span class="pre">s3qladm</span></tt> command performs various operations on S3QL buckets. The file system contained in the bucket <em>must not be mounted</em> when -using <tt class=" docutils literal"><span class="pre">s3qladm</span></tt> or things will go wrong badly.</p> +using <tt class="docutils literal"><span class="pre">s3qladm</span></tt> or things will go wrong badly.</p> <p>The syntax is</p> <div class="highlight-commandline"><div class="highlight"><pre><span class="l">s3qladm </span><span class="ge">[options]</span><span class="l"> </span><span class="nv"><action></span><span class="l"> </span><span class="nv"><storage-url></span><span class="l"></span> </pre></div> </div> <p>where <tt class="var docutils literal"><span class="pre">action</span></tt> may be either of <strong class="program">passphrase</strong>, -<strong class="program">upgrade</strong>, <strong class="program">delete</strong> or <strong class="program">download-metadata</strong>.</p> +<strong class="program">upgrade</strong>, <strong class="program">clear</strong> or <strong class="program">download-metadata</strong>.</p> <p>The <strong class="program">s3qladm</strong> accepts the following general options, no matter what specific action is being invoked:</p> <blockquote> @@ -112,35 +113,38 @@ matter what specific action is being invoked:</p> <tr><td class="option-group" colspan="2"> <kbd><span class="option">--debug <var><module></var></span></kbd></td> </tr> -<tr><td> </td><td>activate debugging output from <module>. Use <tt class=" docutils literal"><span class="pre">all</span></tt> to get +<tr><td> </td><td>activate debugging output from <module>. Use <tt class="docutils literal"><span class="pre">all</span></tt> to get debug messages from all modules. This option can be specified multiple times.</td></tr> <tr><td class="option-group"> <kbd><span class="option">--quiet</span></kbd></td> <td>be really quiet</td></tr> +<tr><td class="option-group"> +<kbd><span class="option">--log <var><target></var></span></kbd></td> +<td>Write logging info into this file. File will be rotated +when it reaches 1 MB, and at most 5 old log files will be +kept. Specify <tt class="docutils literal"><span class="pre">none</span></tt> to disable logging. Default: +<tt class="docutils literal"><span class="pre">none</span></tt></td></tr> +<tr><td class="option-group" colspan="2"> +<kbd><span class="option">--authfile <var><path></var></span></kbd></td> +</tr> +<tr><td> </td><td>Read authentication credentials from this file (default: +<tt class="docutils literal"><span class="pre">~/.s3ql/authinfo2)</span></tt></td></tr> <tr><td class="option-group" colspan="2"> -<kbd><span class="option">--homedir <var><path></var></span></kbd></td> +<kbd><span class="option">--cachedir <var><path></var></span></kbd></td> </tr> -<tr><td> </td><td>Directory for log files, cache and authentication info. -(default: <tt class=" docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> +<tr><td> </td><td>Store cached data in this directory (default: <tt class="docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> <tr><td class="option-group"> <kbd><span class="option">--version</span></kbd></td> <td>just print program version and exit</td></tr> -<tr><td class="option-group"> -<kbd><span class="option">--ssl</span></kbd></td> -<td>Use SSL when connecting to remote servers. This option is -not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext -even for unencrypted file systems.</td></tr> </tbody> </table> </div></blockquote> -<p>Hint: run <tt class=" docutils literal"><span class="pre">s3qladm</span> <span class="pre"><action></span> <span class="pre">--help</span></tt> to get help on the additional +<p>Hint: run <tt class="docutils literal"><span class="pre">s3qladm</span> <span class="pre"><action></span> <span class="pre">--help</span></tt> to get help on the additional arguments that the different actions take.</p> <div class="section" id="changing-the-passphrase"> <h2>Changing the Passphrase<a class="headerlink" href="#changing-the-passphrase" title="Permalink to this headline">¶</a></h2> -<p>To change the passphrase a bucket, use the <tt class=" docutils literal"><span class="pre">s3qladm</span></tt> command:</p> +<p>To change the passphrase a bucket, use the <tt class="docutils literal"><span class="pre">s3qladm</span></tt> command:</p> <div class="highlight-commandline"><div class="highlight"><pre><span class="l">s3qladm passphrase </span><span class="nv"><storage url></span><span class="l"></span> </pre></div> </div> @@ -164,7 +168,7 @@ execute</p> <div class="section" id="deleting-a-file-system"> <h2>Deleting a file system<a class="headerlink" href="#deleting-a-file-system" title="Permalink to this headline">¶</a></h2> <p>A file system can be deleted with:</p> -<div class="highlight-commandline"><div class="highlight"><pre><span class="l">s3qladm delete </span><span class="nv"><storage url></span><span class="l"></span> +<div class="highlight-commandline"><div class="highlight"><pre><span class="l">s3qladm clear </span><span class="nv"><storage url></span><span class="l"></span> </pre></div> </div> <p>This physically deletes all the data and file system structures.</p> @@ -206,7 +210,7 @@ for help on the mailing list first (see <a class="reference internal" href="reso <li class="right" > <a href="mkfs.html" title="File System Creation" >previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/html/backends.html b/doc/html/backends.html index e906681..d810f27 100644 --- a/doc/html/backends.html +++ b/doc/html/backends.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>Storage Backends — S3QL 1.0.1 documentation</title> + <title>Storage Backends — S3QL 1.2 documentation</title> <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,9 +26,9 @@ <script type="text/javascript" src="_static/underscore.js"></script> <script type="text/javascript" src="_static/doctools.js"></script> <link rel="author" title="About these documents" href="about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="index.html" /> <link rel="next" title="File System Creation" href="mkfs.html" /> - <link rel="prev" title="Installation" href="installation.html" /> + <link rel="prev" title="General Information" href="general.html" /> </head> <body> <div class="related"> @@ -38,9 +38,9 @@ <a href="mkfs.html" title="File System Creation" accesskey="N">next</a></li> <li class="right" > - <a href="installation.html" title="Installation" + <a href="general.html" title="General Information" accesskey="P">previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -49,13 +49,13 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a></li> <li class="toctree-l1 current"><a class="current reference internal" href="">Storage Backends</a><ul> -<li class="toctree-l2"><a class="reference internal" href="#on-backend-reliability">On Backend Reliability</a></li> -<li class="toctree-l2"><a class="reference internal" href="#the-authinfo-file">The <tt class=" docutils literal"><span class="pre">authinfo</span></tt> file</a></li> -<li class="toctree-l2"><a class="reference internal" href="#consistency-guarantees">Consistency Guarantees</a></li> -<li class="toctree-l2"><a class="reference internal" href="#the-amazon-s3-backend">The Amazon S3 Backend</a></li> -<li class="toctree-l2"><a class="reference internal" href="#the-local-backend">The Local Backend</a></li> -<li class="toctree-l2"><a class="reference internal" href="#the-sftp-backend">The SFTP Backend</a></li> +<li class="toctree-l2"><a class="reference internal" href="#google-storage">Google Storage</a></li> +<li class="toctree-l2"><a class="reference internal" href="#amazon-s3">Amazon S3</a></li> +<li class="toctree-l2"><a class="reference internal" href="#s3-compatible">S3 compatible</a></li> +<li class="toctree-l2"><a class="reference internal" href="#local">Local</a></li> +<li class="toctree-l2"><a class="reference internal" href="#ssh-sftp">SSH/SFTP</a></li> </ul> </li> <li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> @@ -94,243 +94,167 @@ <div class="body"> <div class="section" id="storage-backends"> -<h1>Storage Backends<a class="headerlink" href="#storage-backends" title="Permalink to this headline">¶</a></h1> -<p>S3QL can use different protocols to store the file system data. -Independent of the backend that you use, the place where your file -system data is being stored is called a <em>bucket</em>. (This is mostly for -historical reasons, since initially S3QL supported only the Amazon S3 -backend).</p> -<div class="section" id="on-backend-reliability"> -<h2>On Backend Reliability<a class="headerlink" href="#on-backend-reliability" title="Permalink to this headline">¶</a></h2> -<p>S3QL has been designed for use with a storage backend where data loss -is so infrequent that it can be completely neglected (e.g. the Amazon -S3 backend). If you decide to use a less reliable backend, you should -keep the following warning in mind and read this section carefully.</p> -<div class="admonition warning"> -<p class="first admonition-title">Warning</p> -<p class="last">S3QL is not able to compensate for any failures of the backend. In -particular, it is not able reconstruct any data that has been lost -or corrupted by the backend. The persistence and durability of data -stored in an S3QL file system is limited and determined by the -backend alone.</p> -</div> -<p>On the plus side, if a backend looses or corrupts some of the stored -data, S3QL <em>will</em> detect the problem. Missing data will be detected -when running <tt class=" docutils literal"><span class="pre">fsck.s3ql</span></tt> or when attempting to access the data in the -mounted file system. In the later case you will get an IO Error, and -on unmounting S3QL will warn you that the file system is damaged and -you need to run <tt class=" docutils literal"><span class="pre">fsck.s3ql</span></tt>.</p> -<p><tt class=" docutils literal"><span class="pre">fsck.s3ql</span></tt> will report all the affected files and move them into the -<tt class=" docutils literal"><span class="pre">/lost+found</span></tt> directory of the file system.</p> -<p>You should be aware that, because of S3QL’s data de-duplication -feature, the consequences of a data loss in the backend can be -significantly more severe than you may expect. More concretely, a data -loss in the backend at time <em>x</em> may cause data that is written <em>after</em> -time <em>x</em> to be lost as well. What may happen is this:</p> -<ol class="arabic simple"> -<li>You store an important file in the S3QL file system.</li> -<li>The backend looses the data blocks of this file. As long as you -do not access the file or run <tt class=" docutils literal"><span class="pre">fsck.s3ql</span></tt>, S3QL -is not aware that the data has been lost by the backend.</li> -<li>You save an additional copy of the important file in a different -location on the same S3QL file system.</li> -<li>S3QL detects that the contents of the new file are identical to the -data blocks that have been stored earlier. Since at this point S3QL -is not aware that these blocks have been lost by the backend, it -does not save another copy of the file contents in the backend but -relies on the (presumably) existing blocks instead.</li> -<li>Therefore, even though you saved another copy, you still do not -have a backup of the important file (since both copies refer to the -same data blocks that have been lost by the backend).</li> -</ol> -<p>As one can see, this effect becomes the less important the more often -one runs <tt class=" docutils literal"><span class="pre">fsck.s3ql</span></tt>, since <tt class=" docutils literal"><span class="pre">fsck.s3ql</span></tt> will make S3QL aware of any -blocks that the backend may have lost. Figuratively, this establishes -a “checkpoint”: data loss in the backend that occurred before running -<tt class=" docutils literal"><span class="pre">fsck.s3ql</span></tt> can not affect any file system operations performed after -running <tt class=" docutils literal"><span class="pre">fsck.s3ql</span></tt>.</p> -<p>Nevertheless, (as said at the beginning) the recommended way to use -S3QL is in combination with a sufficiently reliable storage backend. -In that case none of the above will ever be a concern.</p> -</div> -<div class="section" id="the-authinfo-file"> -<h2>The <tt class=" docutils literal"><span class="pre">authinfo</span></tt> file<a class="headerlink" href="#the-authinfo-file" title="Permalink to this headline">¶</a></h2> -<p>Most backends first try to read the file <tt class=" docutils literal"><span class="pre">~/.s3ql/authinfo</span></tt> to determine -the username and password for connecting to the remote host. If this -fails, both username and password are read from the terminal.</p> -<p>The <tt class=" docutils literal"><span class="pre">authinfo</span></tt> file has to contain entries of the form</p> -<div class="highlight-commandline"><div class="highlight"><pre><span class="l">backend </span><span class="nv"><backend></span><span class="l"> machine </span><span class="nv"><host></span><span class="l"> login </span><span class="nv"><user></span><span class="l"> password </span><span class="nv"><password></span><span class="l"></span> +<span id="id1"></span><h1>Storage Backends<a class="headerlink" href="#storage-backends" title="Permalink to this headline">¶</a></h1> +<p>The following backends are currently available in S3QL:</p> +<div class="section" id="google-storage"> +<h2>Google Storage<a class="headerlink" href="#google-storage" title="Permalink to this headline">¶</a></h2> +<p><a class="reference external" href="http://code.google.com/apis/storage/">Google Storage</a> is an online +storage service offered by Google. It is the most feature-rich service +supported by S3QL and S3QL offers the best performance when used with +the Google Storage backend.</p> +<p>To use the Google Storage backend, you need to have (or sign up for) a +Google account, and then <a class="reference external" href="http://code.google.com/apis/storage/docs/signup.html">activate Google Storage</a> for your +account. The account is free, you will pay only for the amount of +storage and traffic that you actually use. Once you have created the +account, make sure to <a class="reference external" href="http://code.google.com/apis/storage/docs/reference/v1/apiversion1.html#enabling">activate legacy access</a>.</p> +<p>To create a Google Storage bucket, you can use e.g. the <a class="reference external" href="https://sandbox.google.com/storage/">Google +Storage Manager</a>. The +storage URL for accessing the bucket in S3QL is then</p> +<div class="highlight-commandline"><div class="highlight"><pre><span class="l">gs://</span><span class="nv"><bucketname></span><span class="l">/</span><span class="nv"><prefix></span><span class="l"></span> </pre></div> </div> -<p>So to use the login <tt class=" docutils literal"><span class="pre">joe</span></tt> with password <tt class=" docutils literal"><span class="pre">jibbadup</span></tt> when using the FTP -backend to connect to the host <tt class=" docutils literal"><span class="pre">backups.joesdomain.com</span></tt>, you would -specify</p> -<div class="highlight-commandline"><div class="highlight"><pre><span class="l">backend ftp machine backups.joesdomain.com login joe password jibbadup</span> -</pre></div> +<p>Here <em>bucketname</em> is the name of the bucket, and <em>prefix</em> can be +an arbitrary prefix that will be prepended to all object names used by +S3QL. This allows you to store several S3QL file systems in the same +Google Storage bucket.</p> +<p>Note that the backend login and password for accessing your Google +Storage bucket are not your Google account name and password, but the +<em>Google Storage developer access key</em> and <em>Google Storage developer +secret</em> that you can manage with the <a class="reference external" href="https://code.google.com/apis/console/#:storage:legacy">Google Storage key management +tool</a>.</p> +<p>If you would like S3QL to connect using HTTPS instead of standard +HTTP, start the storage url with <tt class="docutils literal"><span class="pre">gss://</span></tt> instead of <tt class="docutils literal"><span class="pre">gs://</span></tt>. Note +that at this point S3QL does not perform any server certificate +validation (see <a class="reference external" href="http://code.google.com/p/s3ql/issues/detail?id=267">issue 267</a>).</p> </div> +<div class="section" id="amazon-s3"> +<h2>Amazon S3<a class="headerlink" href="#amazon-s3" title="Permalink to this headline">¶</a></h2> +<p><a class="reference external" href="http://aws.amazon.com/s3">Amazon S3</a> is the online storage service +offered by <a class="reference external" href="http://aws.amazon.com/">Amazon Web Services (AWS)</a>. To +use the S3 backend, you first need to sign up for an AWS account. The +account is free, you will pay only for the amount of storage and +traffic that you actually use. After that, you need to create a bucket +that will hold the S3QL file system, e.g. using the <a class="reference external" href="https://console.aws.amazon.com/s3/home">AWS Management +Console</a>. For best +performance, it is recommend to create the bucket in the +geographically closest storage region, but not the US Standard +region (see below).</p> +<p>The storage URL for accessing S3 buckets in S3QL has the form</p> +<div class="highlight-commandline"><div class="highlight"><pre><span class="l">s3://</span><span class="nv"><bucketname></span><span class="l">/</span><span class="nv"><prefix></span><span class="l"></span> +</pre></div> </div> -<div class="section" id="consistency-guarantees"> -<h2>Consistency Guarantees<a class="headerlink" href="#consistency-guarantees" title="Permalink to this headline">¶</a></h2> -<p>The different backends provide different types of <em>consistency -guarantees</em>. Informally, a consistency guarantee tells you how fast -the backend will apply changes to the stored data.</p> -<p>S3QL defines the following three levels:</p> -<ul> -<li><p class="first"><strong>Read-after-Write Consistency.</strong> This is the strongest consistency -guarantee. If a backend offers read-after-write consistency, it -guarantees that as soon as you have committed any changes to the -backend, subsequent requests will take into account these changes.</p> -</li> -<li><p class="first"><strong>Read-after-Create Consistency.</strong> If a backend provides only -read-after-create consistency, only the creation of a new object is -guaranteed to be taken into account for subsequent requests. This -means that, for example, if you overwrite data in an existing -object, subsequent requests may still return the old data for a -certain period of time.</p> -</li> -<li><p class="first"><strong>Eventual consistency.</strong> This is the lowest consistency level. -Basically, any changes that you make to the backend may not be -visible for a certain amount of time after the change has been made. -However, you are guaranteed that no change will be lost. All changes -will <em>eventually</em> become visible.</p> -<p>.</p> -</li> +<p>Here <em>bucketname</em> is the name of the bucket, and <em>prefix</em> can be +an arbitrary prefix that will be prepended to all object names used by +S3QL. This allows you to store several S3QL file systems in the same +S3 bucket.</p> +<p>Note that the backend login and password for accessing S3 are not the +user id and password that you use to log into the Amazon Webpage, but +the <em>AWS access key id</em> and <em>AWS secret access key</em> shown under <a class="reference external" href="https://aws-portal.amazon.com/gp/aws/developer/account/index.html?ie=UTF8&action=access-key">My +Account/Access Identifiers</a>.</p> +<p>If you would like S3QL to connect using HTTPS instead of standard +HTTP, start the storage url with <tt class="docutils literal"><span class="pre">s3s://</span></tt> instead of <tt class="docutils literal"><span class="pre">s3://</span></tt>. Note +that, as of May 2011, Amazon S3 is faster when accessed using a +standard HTTP connection, and that S3QL does not perform any server +certificate validation (see <a class="reference external" href="http://code.google.com/p/s3ql/issues/detail?id=267">issue 267</a>).</p> +<div class="section" id="reduced-redundancy-storage-rrs"> +<h3>Reduced Redundancy Storage (RRS)<a class="headerlink" href="#reduced-redundancy-storage-rrs" title="Permalink to this headline">¶</a></h3> +<p>S3QL does not allow the use of <a class="reference external" href="http://aws.amazon.com/s3/#protecting">reduced redundancy storage</a>. The reason for that is a +combination of three factors:</p> +<ul class="simple"> +<li>RRS has a relatively low reliability, on average you loose one +out of every ten-thousand objects a year. So you can expect to +occasionally loose some data.</li> +<li>When <tt class="docutils literal"><span class="pre">fsck.s3ql</span></tt> asks S3 for a list of the stored objects, this list +includes even those objects that have been lost. Therefore +<tt class="docutils literal"><span class="pre">fsck.s3ql</span></tt> <em>can not detect lost objects</em> and lost data will only +become apparent when you try to actually read from a file whose data +has been lost. This is a (very unfortunate) peculiarity of Amazon +S3.</li> +<li>Due to the data de-duplication feature of S3QL, unnoticed lost +objects may cause subsequent data loss later in time (see +<a class="reference internal" href="general.html#backend-reliability"><em>On Backend Reliability</em></a> for details).</li> </ul> -<p>As long as your backend provides read-after-write or read-after-create -consistency, you do not have to worry about consistency guarantees at -all. However, if you plan to use a backend with only eventual -consistency, you have to be a bit careful in some situations.</p> -<div class="section" id="dealing-with-eventual-consistency"> -<span id="eventual-consistency"></span><h3>Dealing with Eventual Consistency<a class="headerlink" href="#dealing-with-eventual-consistency" title="Permalink to this headline">¶</a></h3> -<div class="admonition note"> -<p class="first admonition-title">Note</p> -<p class="last">The following applies only to storage backends that do not provide -read-after-create or read-after-write consistency. Currently, -this is only the Amazon S3 backend <em>if used with the US-Standard -storage region</em>. If you use a different storage backend, or the S3 -backend with a different storage region, this section does not apply -to you.</p> </div> -<p>While the file system is mounted, S3QL is able to automatically handle -all issues related to the weak eventual consistency guarantee. -However, some issues may arise during the mount process and when the -file system is checked.</p> +<div class="section" id="potential-issues-when-using-the-us-standard-storage-region"> +<h3>Potential issues when using the US Standard storage region<a class="headerlink" href="#potential-issues-when-using-the-us-standard-storage-region" title="Permalink to this headline">¶</a></h3> +<p>In the US Standard storage region, Amazon S3 does not guarantee read +after create consistency. This means that after a new object has been +stored, requests to read this object may still fail for a little +while. While the file system is mounted, S3QL is able to automatically +handle all issues related to this so-called eventual consistency. +However, problems may arise during the mount process and when the file +system is checked:</p> <p>Suppose that you mount the file system, store some new data, delete -some old data and unmount it again. Now remember that eventual -consistency means that there is no guarantee that these changes will -be visible immediately. At least in theory it is therefore possible -that if you mount the file system again, S3QL does not see any of the -changes that you have done and presents you an “old version” of the -file system without them. Even worse, if you notice the problem and -unmount the file system, S3QL will upload the old status (which S3QL -necessarily has to consider as current) and thereby permanently -override the newer version (even though this change may not become -immediately visible either).</p> -<p>The same problem applies when checking the file system. If the backend +some old data and unmount it again. Now there is no guarantee that +these changes will be visible immediately. At least in theory it is +therefore possible that if you mount the file system again, S3QL +does not see any of the changes that you have done and presents you +an “old version” of the file system without them. Even worse, if you +notice the problem and unmount the file system, S3QL will upload the +old status (which S3QL necessarily has to consider as current) and +thereby permanently override the newer version (even though this +change may not become immediately visible either).</p> +<p>The same problem applies when checking the file system. If S3 provides S3QL with only partially updated data, S3QL has no way to find out if this a real consistency problem that needs to be fixed or if it is only a temporary problem that will resolve itself automatically (because there are still changes that have not become visible yet).</p> -<p>While this may seem to be a rather big problem, the likelihood of it -to occur is rather low. In practice, most storage providers rarely -need more than a few seconds to apply incoming changes, so to trigger -this problem one would have to unmount and remount the file system in -a very short time window. Many people therefore make sure that they -wait a few minutes between successive mounts (or file system checks) -and decide that the remaining risk is negligible.</p> -<p>Nevertheless, the eventual consistency guarantee does not impose an -upper limit on the time that it may take for change to become visible. -Therefore there is no “totally safe” waiting time that would totally -eliminate this problem; a theoretical possibility always remains.</p> +<p>The likelihood of this to happen is rather low. In practice, most +objects are ready for retrieval just a few seconds after they have +been stored, so to trigger this problem one would have to unmount and +remount the file system in a very short time window. However, since S3 +does not place any upper limit on the length of this window, it is +recommended to not place S3QL buckets in the US Standard storage +region. As of May 2011, all other storage regions provide stronger +consistency guarantees that completely eliminate any of the described +problems.</p> </div> </div> -<div class="section" id="the-amazon-s3-backend"> -<h2>The Amazon S3 Backend<a class="headerlink" href="#the-amazon-s3-backend" title="Permalink to this headline">¶</a></h2> -<p>To store your file system in an Amazon S3 bucket, use a storage URL of -the form <tt class=" docutils literal"><span class="pre">s3://<bucketname></span></tt>. Bucket names must conform to the <a class="reference external" href="http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/BucketRestrictions.html">S3 -Bucket Name Restrictions</a>.</p> -<p>The S3 backend offers exceptionally strong reliability guarantees. As -of August 2010, Amazon guarantees a durability of 99.999999999% per -year. In other words, if you store a thousand million objects then on -average you would loose less than one object in a hundred years.</p> -<p>The Amazon S3 backend provides read-after-create consistency for the -EU, Asia-Pacific and US-West storage regions. <em>For the US-Standard -storage region, Amazon S3 provides only eventual consistency</em> (please -refer to <a class="reference internal" href="#eventual-consistency"><em>Dealing with Eventual Consistency</em></a> for information about -what this entails).</p> -<p>When connecting to Amazon S3, S3QL uses an unencrypted HTTP -connection, so if you want your data to stay confidential, you have -to create the S3QL file system with encryption (this is also the default).</p> -<p>When reading the authentication information for the S3 backend from -the <tt class=" docutils literal"><span class="pre">authinfo</span></tt> file, the <tt class=" docutils literal"><span class="pre">host</span></tt> field is ignored, i.e. the first entry -with <tt class=" docutils literal"><span class="pre">s3</span></tt> as a backend will be used. For example</p> -<div class="highlight-commandline"><div class="highlight"><pre><span class="l">backend s3 machine any login myAWSaccessKeyId password myAwsSecretAccessKey</span> +<div class="section" id="s3-compatible"> +<h2>S3 compatible<a class="headerlink" href="#s3-compatible" title="Permalink to this headline">¶</a></h2> +<p>S3QL is also able to access other, S3 compatible storage services for +which no specific backend exists. Note that when accessing such +services, only the lowest common denominator of available features can +be used, so it is generally recommended to use a service specific +backend instead.</p> +<p>The storage URL for accessing an arbitrary S3 compatible storage +service is</p> +<div class="highlight-commandline"><div class="highlight"><pre><span class="l">s3c://</span><span class="nv"><hostname></span><span class="l">:</span><span class="nv"><port></span><span class="l">/</span><span class="nv"><bucketname></span><span class="l">/</span><span class="nv"><prefix></span><span class="l"></span> </pre></div> </div> -<p>Note that the bucket names come from a global pool, so chances are -that your favorite name has already been taken by another S3 user. -Usually a longer bucket name containing some random numbers, like -<tt class=" docutils literal"><span class="pre">19283712_yourname_s3ql</span></tt>, will work better.</p> -<p>If you do not already have one, you need to obtain an Amazon S3 -account from <a class="reference external" href="http://aws.amazon.com/">Amazon AWS</a>. The account is -free, you will pay only for the amount of storage that you actually -use.</p> -<p>Note that the login and password for accessing S3 are not the user id -and password that you use to log into the Amazon Webpage, but the “AWS -access key id” and “AWS secret access key” shown under <a class="reference external" href="https://aws-portal.amazon.com/gp/aws/developer/account/index.html?ie=UTF8&action=access-key">My -Account/Access Identifiers</a>.</p> -<div class="admonition note"> -<p class="first admonition-title">Note</p> -<p>S3QL also allows you to use <a class="reference external" href="http://aws.amazon.com/s3/#protecting">reduced redundancy storage</a> by using <tt class="docutils literal"><span class="pre">s3rr://</span></tt> -instead of <tt class="docutils literal"><span class="pre">s3://</span></tt> in the storage url. However, this not -recommended. The reason is a combination of three factors:</p> -<ul class="simple"> -<li>RRS has a relatively low reliability, on average you loose one -out of every ten-thousand objects a year. So you can expect to -occasionally loose some data.</li> -<li>When <tt class=" docutils literal"><span class="pre">fsck.s3ql</span></tt> asks Amazon S3 for a list of the stored objects, -this list includes even those objects that have been lost. -Therefore <tt class=" docutils literal"><span class="pre">fsck.s3ql</span></tt> <em>can not detect lost objects</em> and lost data -will only become apparent when you try to actually read from a -file whose data has been lost. This is a (very unfortunate) -peculiarity of Amazon S3.</li> -<li>Due to the data de-duplication feature of S3QL, unnoticed lost -objects may cause subsequent data loss later in time (see <a class="reference internal" href="#on-backend-reliability">On -Backend Reliability</a> for details).</li> -</ul> -<p class="last">In other words, you should really only store an S3QL file system -using RRS if you know exactly what you are getting into.</p> -</div> +<p>or</p> +<div class="highlight-commandline"><div class="highlight"><pre><span class="l">s3cs://</span><span class="nv"><hostname></span><span class="l">:</span><span class="nv"><port></span><span class="l">/</span><span class="nv"><bucketname></span><span class="l">/</span><span class="nv"><prefix></span><span class="l"></span> +</pre></div> </div> -<div class="section" id="the-local-backend"> -<h2>The Local Backend<a class="headerlink" href="#the-local-backend" title="Permalink to this headline">¶</a></h2> -<p>The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -<tt class=" docutils literal"><span class="pre">local://<path></span></tt>. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. <tt class=" docutils literal"><span class="pre">local:///var/archive</span></tt>.</p> -<p>The local backend provides read-after-write consistency.</p> +<p>to use HTTPS connections. Note, however, that at this point S3QL does +not verify the server certificate (cf. <a class="reference external" href="http://code.google.com/p/s3ql/issues/detail?id=267">issue 267</a>).</p> </div> -<div class="section" id="the-sftp-backend"> -<h2>The SFTP Backend<a class="headerlink" href="#the-sftp-backend" title="Permalink to this headline">¶</a></h2> -<p>The SFTP backend uses the SFTP protocol, which is a file transfer -protocol similar to ftp, but uses an encrypted SSH connection. -It provides read-after-write consistency.</p> -<p>Note that the SFTP backend is rather slow and has not been tested -as extensively as the S3 and Local backends.</p> -<p>The storage URL for SFTP connections has the form</p> -<div class="highlight-commandline"><div class="highlight"><pre><span class="l">sftp://</span><span class="nv"><host></span><span class="ge">[:port]</span><span class="l">/</span><span class="nv"><path></span><span class="l"></span> +<div class="section" id="local"> +<h2>Local<a class="headerlink" href="#local" title="Permalink to this headline">¶</a></h2> +<p>S3QL is also able to store its data on the local file system. This can +be used to backup data on external media, or to access external +services that S3QL can not talk to directly (e.g., it is possible to +store data over SSH by first mounting the remote system using +<a class="reference external" href="http://fuse.sourceforge.net/sshfs.html">sshfs</a>, then using the local backend to store the data in the sshfs +mountpoint).</p> +<p>The storage URL for local storage is</p> +<div class="highlight-commandline"><div class="highlight"><pre><span class="l">local://</span><span class="nv"><path></span><span class="l"></span> </pre></div> </div> -<p>The SFTP backend will always ask you for a password if you haven’t -defined one in <tt class=" docutils literal"><span class="pre">~/.s3ql/authinfo</span></tt>. However, public key authentication -is tried first and the password will only be used if the public key -authentication fails.</p> -<p>The public and private keys will be read from the standard files in -<tt class=" docutils literal"><span class="pre">~/.ssh/</span></tt>. Note that S3QL will refuse to connect to a computer with -unknown host key; to add the key to your local keyring you have to -establish a connection to that computer with the standard SSH command -line programs first.</p> +<p>Note that you have to write three consecutive slashes to specify an +absolute path, e.g. <tt class="docutils literal"><span class="pre">local:///var/archive</span></tt>. Also, relative paths will +automatically be converted to absolute paths before the authentication +file is read, i.e. if you are in the <tt class="docutils literal"><span class="pre">/home/john</span></tt> directory and try to +mount <tt class="docutils literal"><span class="pre">local://bucket</span></tt>, the corresponding section in the +authentication file must match the storage url +<tt class="docutils literal"><span class="pre">local:///home/john/bucket</span></tt>.</p> +</div> +<div class="section" id="ssh-sftp"> +<h2>SSH/SFTP<a class="headerlink" href="#ssh-sftp" title="Permalink to this headline">¶</a></h2> +<p>Previous versions of S3QL included an SSH/SFTP backend. With newer +S3QL versions, it is recommended to instead combine the local backend +with <a class="reference external" href="http://fuse.sourceforge.net/sshfs.html">sshfs</a> (cf. <a class="reference internal" href="tips.html#ssh-tipp"><em>SSH Backend</em></a>).</p> </div> </div> @@ -347,9 +271,9 @@ line programs first.</p> <a href="mkfs.html" title="File System Creation" >next</a></li> <li class="right" > - <a href="installation.html" title="Installation" + <a href="general.html" title="General Information" >previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/html/contrib.html b/doc/html/contrib.html index cd77857..3238221 100644 --- a/doc/html/contrib.html +++ b/doc/html/contrib.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>Contributed Programs — S3QL 1.0.1 documentation</title> + <title>Contributed Programs — S3QL 1.2 documentation</title> <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="_static/underscore.js"></script> <script type="text/javascript" src="_static/doctools.js"></script> <link rel="author" title="About these documents" href="about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="index.html" /> <link rel="next" title="Tips & Tricks" href="tips.html" /> <link rel="prev" title="Checking for Errors" href="fsck.html" /> </head> @@ -40,7 +40,7 @@ <li class="right" > <a href="fsck.html" title="Checking for Errors" accesskey="P">previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -49,6 +49,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="adm.html">Managing Buckets</a></li> @@ -98,12 +99,14 @@ <p>S3QL comes with a few contributed programs that are not part of the core distribution (and are therefore not installed automatically by default), but which may nevertheless be useful. These programs are in -the <tt class=" docutils literal"><span class="pre">contrib</span></tt> directory of the source distribution or in -<tt class=" docutils literal"><span class="pre">/usr/share/doc/s3ql/contrib</span></tt> if you installed S3QL from a package.</p> +the <tt class="docutils literal"><span class="pre">contrib</span></tt> directory of the source distribution or in +<tt class="docutils literal"><span class="pre">/usr/share/doc/s3ql/contrib</span></tt> if you installed S3QL from a package.</p> <div class="section" id="benchmark-py"> <h2>benchmark.py<a class="headerlink" href="#benchmark-py" title="Permalink to this headline">¶</a></h2> -<p>This program measures your uplink bandwidth and compression speed and -recommends a compression algorithm for optimal throughput.</p> +<p>This program measures S3QL write performance, uplink bandwidth and +compression speed to determine the limiting factor. It also gives +recommendation for compression algorithm and number of upload threads +to achieve maximum performance.</p> </div> <div class="section" id="s3-copy-py"> <h2>s3_copy.py<a class="headerlink" href="#s3-copy-py" title="Permalink to this headline">¶</a></h2> @@ -116,7 +119,7 @@ migrate buckets to a different storage region or storage class <p><tt class="docutils literal"><span class="pre">pcp.py</span></tt> is a wrapper program that starts several rsync processes to copy directory trees in parallel. This is important because transferring files in parallel significantly enhances performance when -copying data from an S3QL file system (see <a class="reference internal" href="tips.html#copy-performance"><em>Permanently mounted backup file system</em></a> for +copying data from an S3QL file system (see <a class="reference internal" href="tips.html#copy-performance"><em>Improving copy performance</em></a> for details).</p> <p>To recursively copy the directory <tt class="docutils literal"><span class="pre">/mnt/home-backup</span></tt> into <tt class="docutils literal"><span class="pre">/home/joe</span></tt> using 8 parallel processes and preserving permissions, @@ -129,7 +132,7 @@ you would execute</p> <h2>s3_backup.sh<a class="headerlink" href="#s3-backup-sh" title="Permalink to this headline">¶</a></h2> <p>This is an example script that demonstrates how to set up a simple but powerful backup solution using S3QL and <a class="reference external" href="http://samba.org/rsync">rsync</a>.</p> -<p>The <tt class=" docutils literal"><span class="pre">s3_backup.sh</span></tt> script automates the following steps:</p> +<p>The <tt class="docutils literal"><span class="pre">s3_backup.sh</span></tt> script automates the following steps:</p> <ol class="arabic simple"> <li>Mount the file system</li> <li>Replicate the previous backup with <a class="reference internal" href="special.html#s3qlcp"><em>s3qlcp</em></a></li> @@ -139,7 +142,7 @@ powerful backup solution using S3QL and <a class="reference external" href="http <li>Unmount the file system</li> </ol> <p>The backups are stored in directories of the form -<tt class=" docutils literal"><span class="pre">YYYY-MM-DD_HH:mm:SS</span></tt> and the <a class="reference internal" href="#expire-backups-py">expire_backups.py</a> command is used to +<tt class="docutils literal"><span class="pre">YYYY-MM-DD_HH:mm:SS</span></tt> and the <a class="reference internal" href="#expire-backups-py">expire_backups.py</a> command is used to delete old backups.</p> </div> <div class="section" id="expire-backups-py"> @@ -226,7 +229,7 @@ properly unmounts it when the system is shut down.</p> <li class="right" > <a href="fsck.html" title="Checking for Errors" >previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/html/fsck.html b/doc/html/fsck.html index df56342..131ecaf 100644 --- a/doc/html/fsck.html +++ b/doc/html/fsck.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>Checking for Errors — S3QL 1.0.1 documentation</title> + <title>Checking for Errors — S3QL 1.2 documentation</title> <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="_static/underscore.js"></script> <script type="text/javascript" src="_static/doctools.js"></script> <link rel="author" title="About these documents" href="about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="index.html" /> <link rel="next" title="Contributed Programs" href="contrib.html" /> <link rel="prev" title="Unmounting" href="umount.html" /> </head> @@ -40,7 +40,7 @@ <li class="right" > <a href="umount.html" title="Unmounting" accesskey="P">previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -49,6 +49,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="adm.html">Managing Buckets</a></li> @@ -89,7 +90,7 @@ <h1>Checking for Errors<a class="headerlink" href="#checking-for-errors" title="Permalink to this headline">¶</a></h1> <p>If, for some reason, the filesystem has not been correctly unmounted, or if you suspect that there might be errors, you should run the -<tt class=" docutils literal"><span class="pre">fsck.s3ql</span></tt> utility. It has the following syntax:</p> +<tt class="docutils literal"><span class="pre">fsck.s3ql</span></tt> utility. It has the following syntax:</p> <div class="highlight-commandline"><div class="highlight"><pre><span class="l">fsck.s3ql </span><span class="ge">[options]</span><span class="l"> </span><span class="nv"><storage url></span><span class="l"></span> </pre></div> </div> @@ -99,15 +100,25 @@ or if you suspect that there might be errors, you should run the <col class="option" /> <col class="description" /> <tbody valign="top"> +<tr><td class="option-group"> +<kbd><span class="option">--log <var><target></var></span></kbd></td> +<td>Write logging info into this file. File will be rotated +when it reaches 1 MB, and at most 5 old log files will be +kept. Specify <tt class="docutils literal"><span class="pre">none</span></tt> to disable logging. Default: +<tt class="docutils literal"><span class="pre">~/.s3ql/fsck.log</span></tt></td></tr> +<tr><td class="option-group" colspan="2"> +<kbd><span class="option">--cachedir <var><path></var></span></kbd></td> +</tr> +<tr><td> </td><td>Store cached data in this directory (default: <tt class="docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> <tr><td class="option-group" colspan="2"> -<kbd><span class="option">--homedir <var><path></var></span></kbd></td> +<kbd><span class="option">--authfile <var><path></var></span></kbd></td> </tr> -<tr><td> </td><td>Directory for log files, cache and authentication info. -(default: <tt class=" docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> +<tr><td> </td><td>Read authentication credentials from this file (default: +<tt class="docutils literal"><span class="pre">~/.s3ql/authinfo2)</span></tt></td></tr> <tr><td class="option-group" colspan="2"> <kbd><span class="option">--debug <var><module></var></span></kbd></td> </tr> -<tr><td> </td><td>activate debugging output from <module>. Use <tt class=" docutils literal"><span class="pre">all</span></tt> to get +<tr><td> </td><td>activate debugging output from <module>. Use <tt class="docutils literal"><span class="pre">all</span></tt> to get debug messages from all modules. This option can be specified multiple times.</td></tr> <tr><td class="option-group"> @@ -117,13 +128,6 @@ specified multiple times.</td></tr> <kbd><span class="option">--version</span></kbd></td> <td>just print program version and exit</td></tr> <tr><td class="option-group"> -<kbd><span class="option">--ssl</span></kbd></td> -<td>Use SSL when connecting to remote servers. This option is -not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext even -for unencrypted file systems.</td></tr> -<tr><td class="option-group"> <kbd><span class="option">--batch</span></kbd></td> <td>If user input is required, exit without prompting.</td></tr> <tr><td class="option-group"> @@ -149,7 +153,7 @@ for unencrypted file systems.</td></tr> <li class="right" > <a href="umount.html" title="Unmounting" >previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/html/general.html b/doc/html/general.html new file mode 100644 index 0000000..a7300d4 --- /dev/null +++ b/doc/html/general.html @@ -0,0 +1,251 @@ + + +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> + + +<html xmlns="http://www.w3.org/1999/xhtml"> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> + + <title>General Information — S3QL 1.2 documentation</title> + + <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> + <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> + + <script type="text/javascript"> + var DOCUMENTATION_OPTIONS = { + URL_ROOT: '', + VERSION: '1.2', + COLLAPSE_INDEX: false, + FILE_SUFFIX: '.html', + HAS_SOURCE: true + }; + </script> + <script type="text/javascript" src="_static/jquery.js"></script> + <script type="text/javascript" src="_static/underscore.js"></script> + <script type="text/javascript" src="_static/doctools.js"></script> + <link rel="author" title="About these documents" href="about.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="index.html" /> + <link rel="next" title="Storage Backends" href="backends.html" /> + <link rel="prev" title="Installation" href="installation.html" /> + </head> + <body> + <div class="related"> + <h3>Navigation</h3> + <ul> + <li class="right" style="margin-right: 10px"> + <a href="backends.html" title="Storage Backends" + accesskey="N">next</a></li> + <li class="right" > + <a href="installation.html" title="Installation" + accesskey="P">previous</a> |</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> + </ul> + </div> + <div class="sphinxsidebar"> + <div class="sphinxsidebarwrapper"> + <h3><a href="index.html">Table Of Contents</a></h3> + <ul class="current"> +<li class="toctree-l1"><a class="reference internal" href="about.html">About S3QL</a></li> +<li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l1 current"><a class="current reference internal" href="">General Information</a><ul> +<li class="toctree-l2"><a class="reference internal" href="#terminology">Terminology</a></li> +<li class="toctree-l2"><a class="reference internal" href="#storing-authentication-information">Storing Authentication Information</a></li> +<li class="toctree-l2"><a class="reference internal" href="#on-backend-reliability">On Backend Reliability</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a></li> +<li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> +<li class="toctree-l1"><a class="reference internal" href="adm.html">Managing Buckets</a></li> +<li class="toctree-l1"><a class="reference internal" href="mount.html">Mounting</a></li> +<li class="toctree-l1"><a class="reference internal" href="special.html">Advanced S3QL Features</a></li> +<li class="toctree-l1"><a class="reference internal" href="umount.html">Unmounting</a></li> +<li class="toctree-l1"><a class="reference internal" href="fsck.html">Checking for Errors</a></li> +<li class="toctree-l1"><a class="reference internal" href="contrib.html">Contributed Programs</a></li> +<li class="toctree-l1"><a class="reference internal" href="tips.html">Tips & Tricks</a></li> +<li class="toctree-l1"><a class="reference internal" href="issues.html">Known Issues</a></li> +<li class="toctree-l1"><a class="reference internal" href="man/index.html">Manpages</a></li> +<li class="toctree-l1"><a class="reference internal" href="resources.html">Further Resources / Getting Help</a></li> +</ul> + + + <div id="searchbox" style="display: none"> + <h3>Quick search</h3> + <form class="search" action="search.html" method="get"> + <input type="text" name="q" size="18" /> + <input type="submit" value="Go" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> + <p class="searchtip" style="font-size: 90%"> + Enter search terms. + </p> + </div> + <script type="text/javascript">$('#searchbox').show(0);</script> + </div> + </div> + + <div class="document"> + <div class="documentwrapper"> + <div class="bodywrapper"> + <div class="body"> + + <div class="section" id="general-information"> +<h1>General Information<a class="headerlink" href="#general-information" title="Permalink to this headline">¶</a></h1> +<div class="section" id="terminology"> +<h2>Terminology<a class="headerlink" href="#terminology" title="Permalink to this headline">¶</a></h2> +<p>S3QL can store data at different service providers and using different +protocols. The term <em>backend</em> refers to both the part of S3QL that +implements communication with a specific storage service and the +storage service itself. Most backends can hold more than one S3QL file +system and thus require some additional information that specifies the +file system location within the backend. This location is called a +<em>bucket</em> (for historical reasons).</p> +<p>Many S3QL commands expect a <em>storage url</em> as a parameter. A storage +url specifies both the backend and the bucket and thus uniquely +identifies an S3QL file system. The form of the storage url depends on +the backend and is described together with the +<a class="reference internal" href="backends.html#storage-backends"><em>Storage Backends</em></a>.</p> +</div> +<div class="section" id="storing-authentication-information"> +<span id="bucket-pw"></span><h2>Storing Authentication Information<a class="headerlink" href="#storing-authentication-information" title="Permalink to this headline">¶</a></h2> +<p>Normally, S3QL reads username and password for the backend as well as +an encryption passphrase for the bucket from the terminal. Most +commands also accept an <tt class="cmdopt docutils literal"><span class="pre">--authfile</span></tt> parameter that can be +used to read this information from a file instead.</p> +<p>The authentication file consists of sections, led by a <tt class="docutils literal"><span class="pre">[section]</span></tt> +header and followed by <tt class="docutils literal"><span class="pre">name:</span> <span class="pre">value</span></tt> entries. The section headers +themselves are not used by S3QL but have to be unique within the file.</p> +<p>In each section, the following entries can be defined:</p> +<table class="docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">storage-url:</th><td class="field-body">Specifies the storage url to which this section applies. If a +storage url starts with the value of this entry, the section is +considered applicable.</td> +</tr> +<tr class="field-even field"><th class="field-name">backend-login:</th><td class="field-body">Specifies the username to use for authentication with the backend.</td> +</tr> +<tr class="field-odd field"><th class="field-name" colspan="2">backend-password:</th></tr> +<tr class="field-odd field"><td> </td><td class="field-body">Specifies the password to use for authentication with the backend.</td> +</tr> +<tr class="field-even field"><th class="field-name" colspan="2">bucket-passphrase:</th></tr> +<tr class="field-even field"><td> </td><td class="field-body">Specifies the passphrase to use to decrypt the bucket (if it is +encrypted).</td> +</tr> +</tbody> +</table> +<p>When reading the authentication file, S3QL considers every applicable +section in order and uses the last value that it found for each entry. +For example, consider the following authentication file:</p> +<div class="highlight-commandline"><div class="highlight"><pre><span class="ge">[s3]</span><span class="l"></span> +<span class="l">storage-url: s3://</span> +<span class="l">backend-login: joe</span> +<span class="l">backend-password: notquitesecret</span> + +<span class="ge">[bucket1]</span><span class="l"></span> +<span class="l">storage-url: s3://joes-first-bucket</span> +<span class="l">bucket-passphrase: neitheristhis</span> + +<span class="ge">[bucket2]</span><span class="l"></span> +<span class="l">storage-url: s3://joes-second-bucket</span> +<span class="l">bucket-passphrase: swordfish</span> + +<span class="ge">[bucket3]</span><span class="l"></span> +<span class="l">storage-url: s3://joes-second-bucket/with-prefix</span> +<span class="l">backend-login: bill</span> +<span class="l">backend-password: bi23ll</span> +<span class="l">bucket-passphrase: ll23bi</span> +</pre></div> +</div> +<p>With this authentication file, S3QL would try to log in as “joe” +whenever the s3 backend is used, except when accessing a storage url +that begins with “s3://joes-second-bucket/with-prefix”. In that case, +the last section becomes active and S3QL would use the “bill” +credentials. Furthermore, bucket encryption passphrases will be used +for storage urls that start with “s3://joes-first-bucket” or +“s3://joes-second-bucket”.</p> +<p>The authentication file is parsed by the <a class="reference external" href="http://docs.python.org/library/configparser.html">Python ConfigParser +module</a>.</p> +</div> +<div class="section" id="on-backend-reliability"> +<span id="backend-reliability"></span><h2>On Backend Reliability<a class="headerlink" href="#on-backend-reliability" title="Permalink to this headline">¶</a></h2> +<p>S3QL has been designed for use with a storage backend where data loss +is so infrequent that it can be completely neglected (e.g. the Amazon +S3 backend). If you decide to use a less reliable backend, you should +keep the following warning in mind and read this section carefully.</p> +<div class="admonition warning"> +<p class="first admonition-title">Warning</p> +<p class="last">S3QL is not able to compensate for any failures of the backend. In +particular, it is not able reconstruct any data that has been lost +or corrupted by the backend. The persistence and durability of data +stored in an S3QL file system is limited and determined by the +backend alone.</p> +</div> +<p>On the plus side, if a backend looses or corrupts some of the stored +data, S3QL <em>will</em> detect the problem. Missing data will be detected +when running <tt class="docutils literal"><span class="pre">fsck.s3ql</span></tt> or when attempting to access the data in the +mounted file system. In the later case you will get an IO Error, and +on unmounting S3QL will warn you that the file system is damaged and +you need to run <tt class="docutils literal"><span class="pre">fsck.s3ql</span></tt>.</p> +<p><tt class="docutils literal"><span class="pre">fsck.s3ql</span></tt> will report all the affected files and move them into the +<tt class="docutils literal"><span class="pre">/lost+found</span></tt> directory of the file system.</p> +<p>You should be aware that, because of S3QL’s data de-duplication +feature, the consequences of a data loss in the backend can be +significantly more severe than you may expect. More concretely, a data +loss in the backend at time <em>x</em> may cause data that is written <em>after</em> +time <em>x</em> to be lost as well. What may happen is this:</p> +<ol class="arabic simple"> +<li>You store an important file in the S3QL file system.</li> +<li>The backend looses the data blocks of this file. As long as you +do not access the file or run <tt class="docutils literal"><span class="pre">fsck.s3ql</span></tt>, S3QL +is not aware that the data has been lost by the backend.</li> +<li>You save an additional copy of the important file in a different +location on the same S3QL file system.</li> +<li>S3QL detects that the contents of the new file are identical to the +data blocks that have been stored earlier. Since at this point S3QL +is not aware that these blocks have been lost by the backend, it +does not save another copy of the file contents in the backend but +relies on the (presumably) existing blocks instead.</li> +<li>Therefore, even though you saved another copy, you still do not +have a backup of the important file (since both copies refer to the +same data blocks that have been lost by the backend).</li> +</ol> +<p>As one can see, this effect becomes the less important the more often +one runs <tt class="docutils literal"><span class="pre">fsck.s3ql</span></tt>, since <tt class="docutils literal"><span class="pre">fsck.s3ql</span></tt> will make S3QL aware of any +blocks that the backend may have lost. Figuratively, this establishes +a “checkpoint”: data loss in the backend that occurred before running +<tt class="docutils literal"><span class="pre">fsck.s3ql</span></tt> can not affect any file system operations performed after +running <tt class="docutils literal"><span class="pre">fsck.s3ql</span></tt>.</p> +<p>Nevertheless, the recommended way to use S3QL is in combination with a +sufficiently reliable storage backend. In that case none of the above +will ever be a concern.</p> +</div> +</div> + + + </div> + </div> + </div> + <div class="clearer"></div> + </div> + <div class="related"> + <h3>Navigation</h3> + <ul> + <li class="right" style="margin-right: 10px"> + <a href="backends.html" title="Storage Backends" + >next</a></li> + <li class="right" > + <a href="installation.html" title="Installation" + >previous</a> |</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> + </ul> + </div> + <div class="footer"> + © Copyright 2008-2011, Nikolaus Rath. + Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 1.1pre. + </div> + </body> +</html>
\ No newline at end of file diff --git a/doc/html/index.html b/doc/html/index.html index f2e42d1..fa0fc02 100644 --- a/doc/html/index.html +++ b/doc/html/index.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>S3QL User’s Guide — S3QL 1.0.1 documentation</title> + <title>S3QL User’s Guide — S3QL 1.2 documentation</title> <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="_static/underscore.js"></script> <script type="text/javascript" src="_static/doctools.js"></script> <link rel="author" title="About these documents" href="about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="#" /> + <link rel="top" title="S3QL 1.2 documentation" href="#" /> <link rel="next" title="About S3QL" href="about.html" /> </head> <body> @@ -36,7 +36,7 @@ <li class="right" style="margin-right: 10px"> <a href="about.html" title="About S3QL" accesskey="N">next</a></li> - <li><a href="#">S3QL 1.0.1 documentation</a> »</li> + <li><a href="#">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -45,6 +45,7 @@ <ul> <li class="toctree-l1"><a class="reference internal" href="about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="adm.html">Managing Buckets</a></li> @@ -95,13 +96,18 @@ <li class="toctree-l2"><a class="reference internal" href="installation.html#installing-s3ql">Installing S3QL</a></li> </ul> </li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a><ul> +<li class="toctree-l2"><a class="reference internal" href="general.html#terminology">Terminology</a></li> +<li class="toctree-l2"><a class="reference internal" href="general.html#storing-authentication-information">Storing Authentication Information</a></li> +<li class="toctree-l2"><a class="reference internal" href="general.html#on-backend-reliability">On Backend Reliability</a></li> +</ul> +</li> <li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a><ul> -<li class="toctree-l2"><a class="reference internal" href="backends.html#on-backend-reliability">On Backend Reliability</a></li> -<li class="toctree-l2"><a class="reference internal" href="backends.html#the-authinfo-file">The <tt class=" docutils literal"><span class="pre">authinfo</span></tt> file</a></li> -<li class="toctree-l2"><a class="reference internal" href="backends.html#consistency-guarantees">Consistency Guarantees</a></li> -<li class="toctree-l2"><a class="reference internal" href="backends.html#the-amazon-s3-backend">The Amazon S3 Backend</a></li> -<li class="toctree-l2"><a class="reference internal" href="backends.html#the-local-backend">The Local Backend</a></li> -<li class="toctree-l2"><a class="reference internal" href="backends.html#the-sftp-backend">The SFTP Backend</a></li> +<li class="toctree-l2"><a class="reference internal" href="backends.html#google-storage">Google Storage</a></li> +<li class="toctree-l2"><a class="reference internal" href="backends.html#amazon-s3">Amazon S3</a></li> +<li class="toctree-l2"><a class="reference internal" href="backends.html#s3-compatible">S3 compatible</a></li> +<li class="toctree-l2"><a class="reference internal" href="backends.html#local">Local</a></li> +<li class="toctree-l2"><a class="reference internal" href="backends.html#ssh-sftp">SSH/SFTP</a></li> </ul> </li> <li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> @@ -113,7 +119,6 @@ </ul> </li> <li class="toctree-l1"><a class="reference internal" href="mount.html">Mounting</a><ul> -<li class="toctree-l2"><a class="reference internal" href="mount.html#storing-encryption-passwords">Storing Encryption Passwords</a></li> <li class="toctree-l2"><a class="reference internal" href="mount.html#compression-algorithms">Compression Algorithms</a></li> <li class="toctree-l2"><a class="reference internal" href="mount.html#parallel-compression">Parallel Compression</a></li> <li class="toctree-l2"><a class="reference internal" href="mount.html#notes-about-caching">Notes about Caching</a></li> @@ -140,6 +145,7 @@ </ul> </li> <li class="toctree-l1"><a class="reference internal" href="tips.html">Tips & Tricks</a><ul> +<li class="toctree-l2"><a class="reference internal" href="tips.html#ssh-backend">SSH Backend</a></li> <li class="toctree-l2"><a class="reference internal" href="tips.html#permanently-mounted-backup-file-system">Permanently mounted backup file system</a></li> <li class="toctree-l2"><a class="reference internal" href="tips.html#improving-copy-performance">Improving copy performance</a></li> </ul> @@ -177,7 +183,7 @@ <li class="right" style="margin-right: 10px"> <a href="about.html" title="About S3QL" >next</a></li> - <li><a href="#">S3QL 1.0.1 documentation</a> »</li> + <li><a href="#">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/html/installation.html b/doc/html/installation.html index cb7912b..eaf3225 100644 --- a/doc/html/installation.html +++ b/doc/html/installation.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>Installation — S3QL 1.0.1 documentation</title> + <title>Installation — S3QL 1.2 documentation</title> <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,8 +26,8 @@ <script type="text/javascript" src="_static/underscore.js"></script> <script type="text/javascript" src="_static/doctools.js"></script> <link rel="author" title="About these documents" href="about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="index.html" /> - <link rel="next" title="Storage Backends" href="backends.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="index.html" /> + <link rel="next" title="General Information" href="general.html" /> <link rel="prev" title="About S3QL" href="about.html" /> </head> <body> @@ -35,12 +35,12 @@ <h3>Navigation</h3> <ul> <li class="right" style="margin-right: 10px"> - <a href="backends.html" title="Storage Backends" + <a href="general.html" title="General Information" accesskey="N">next</a></li> <li class="right" > <a href="about.html" title="About S3QL" accesskey="P">previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -53,6 +53,7 @@ <li class="toctree-l2"><a class="reference internal" href="#installing-s3ql">Installing S3QL</a></li> </ul> </li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="adm.html">Managing Buckets</a></li> @@ -108,19 +109,16 @@ running S3QL. Generally, you should first check if your distribution already provides a suitable packages and only install from source if that is not the case.</p> <ul> -<li><p class="first">Kernel version 2.6.9 or newer. Starting with kernel 2.6.26 -you will get significantly better write performance, so you should -actually use <em>2.6.26 or newer whenever possible</em>.</p> -</li> -<li><p class="first">The <a class="reference external" href="http://fuse.sourceforge.net/">FUSE Library</a> should already be -installed on your system. However, you have to make sure that you -have at least version 2.8.0.</p> +<li><p class="first">Kernel: Linux 2.6.9 or newer or FreeBSD with <a class="reference external" href="http://www.freshports.org/sysutils/fusefs-kmod/">FUSE4BSD</a>. Starting with +kernel 2.6.26 you will get significantly better write performance, +so under Linux you should actually use <em>2.6.26 or newer whenever +possible</em>.</p> </li> <li><p class="first">The <a class="reference external" href="http://pypi.python.org/pypi/pycryptopp">PyCrypto++ Python Module</a>. To check if this module -is installed, try to execute <tt class=" docutils literal"><span class="pre">python</span> <span class="pre">-c</span> <span class="pre">'import</span> <span class="pre">pycryptopp'</span></tt>.</p> +is installed, try to execute <tt class="docutils literal"><span class="pre">python</span> <span class="pre">-c</span> <span class="pre">'import</span> <span class="pre">pycryptopp'</span></tt>.</p> </li> <li><p class="first">The <a class="reference external" href="http://pypi.python.org/pypi/argparse">argparse Python Module</a>. To check if this module is -installed, try to execute <tt class=" docutils literal"><span class="pre">python</span> <span class="pre">-c</span> <span class="pre">'import</span> <span class="pre">argparse;</span> <span class="pre">print</span> +installed, try to execute <tt class="docutils literal"><span class="pre">python</span> <span class="pre">-c</span> <span class="pre">'import</span> <span class="pre">argparse;</span> <span class="pre">print</span> <span class="pre">argparse.__version__'</span></tt>. If argparse is installed, this will print the version number. You need version 1.1 or later.</p> </li> @@ -133,23 +131,18 @@ which (if any) version of APWS is installed, run the command</p> both have to be at least 3.7.0.</p> </li> <li><p class="first">The <a class="reference external" href="http://pypi.python.org/pypi/pyliblzma">PyLibLZMA Python module</a>. To check if this module -is installed, execute <tt class=" docutils literal"><span class="pre">python</span> <span class="pre">-c</span> <span class="pre">'import</span> <span class="pre">lzma;</span> <span class="pre">print</span> +is installed, execute <tt class="docutils literal"><span class="pre">python</span> <span class="pre">-c</span> <span class="pre">'import</span> <span class="pre">lzma;</span> <span class="pre">print</span> <span class="pre">lzma.__version__'</span></tt>. This should print a version number. You need at least version 0.5.3.</p> </li> <li><p class="first">The <a class="reference external" href="http://code.google.com/p/python-llfuse/">Python LLFUSE module</a>. To check if this module -is installed, execute <tt class=" docutils literal"><span class="pre">python</span> <span class="pre">-c</span> <span class="pre">'import</span> <span class="pre">llfuse;</span> <span class="pre">print</span> +is installed, execute <tt class="docutils literal"><span class="pre">python</span> <span class="pre">-c</span> <span class="pre">'import</span> <span class="pre">llfuse;</span> <span class="pre">print</span> <span class="pre">llfuse.__version__'</span></tt>. This should print a version number. You need at least version 0.29.</p> <p>Note that earlier S3QL versions shipped with a builtin version of this module. If you are upgrading from such a version, make sure to completely remove the old S3QL version first.</p> </li> -<li><p class="first">If you want to use the SFTP backend, then you also need the -<a class="reference external" href="http://www.lag.net/paramiko/">Paramiko Python Module</a>. To check -if this module is installed, try to execute <tt class=" docutils literal"><span class="pre">python</span> <span class="pre">-c</span> <span class="pre">'import</span> -<span class="pre">paramiko'</span></tt>.</p> -</li> </ul> </div> <div class="section" id="installing-s3ql"> @@ -158,18 +151,18 @@ if this module is installed, try to execute <tt class=" docutils literal"><span <ol class="arabic simple"> <li>Download S3QL from <a class="reference external" href="http://code.google.com/p/s3ql/downloads/list">http://code.google.com/p/s3ql/downloads/list</a></li> <li>Unpack it into a folder of your choice</li> -<li>Run <tt class=" docutils literal"><span class="pre">python</span> <span class="pre">setup.py</span> <span class="pre">test</span></tt> to run a self-test. If this fails, ask +<li>Run <tt class="docutils literal"><span class="pre">python</span> <span class="pre">setup.py</span> <span class="pre">test</span></tt> to run a self-test. If this fails, ask for help on the <a class="reference external" href="http://groups.google.com/group/s3ql">mailing list</a> or report a bug in the <a class="reference external" href="http://code.google.com/p/s3ql/issues/list">issue tracker</a>.</li> </ol> <p>Now you have three options:</p> <ul class="simple"> -<li>You can run the S3QL commands from the <tt class=" docutils literal"><span class="pre">bin/</span></tt> directory.</li> +<li>You can run the S3QL commands from the <tt class="docutils literal"><span class="pre">bin/</span></tt> directory.</li> <li>You can install S3QL system-wide for all users. To do that, you -have to run <tt class=" docutils literal"><span class="pre">sudo</span> <span class="pre">python</span> <span class="pre">setup.py</span> <span class="pre">install</span></tt>.</li> -<li>You can install S3QL into <tt class=" docutils literal"><span class="pre">~/.local</span></tt> by executing <tt class=" docutils literal"><span class="pre">python</span> +have to run <tt class="docutils literal"><span class="pre">sudo</span> <span class="pre">python</span> <span class="pre">setup.py</span> <span class="pre">install</span></tt>.</li> +<li>You can install S3QL into <tt class="docutils literal"><span class="pre">~/.local</span></tt> by executing <tt class="docutils literal"><span class="pre">python</span> <span class="pre">setup.py</span> <span class="pre">install</span> <span class="pre">--user</span></tt>. In this case you should make sure that -<tt class=" docutils literal"><span class="pre">~/.local/bin</span></tt> is in your <tt class=" docutils literal"><span class="pre">$PATH</span></tt> environment variable.</li> +<tt class="docutils literal"><span class="pre">~/.local/bin</span></tt> is in your <tt class="docutils literal"><span class="pre">$PATH</span></tt> environment variable.</li> </ul> </div> </div> @@ -184,12 +177,12 @@ have to run <tt class=" docutils literal"><span class="pre">sudo</span> <span cl <h3>Navigation</h3> <ul> <li class="right" style="margin-right: 10px"> - <a href="backends.html" title="Storage Backends" + <a href="general.html" title="General Information" >next</a></li> <li class="right" > <a href="about.html" title="About S3QL" >previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/html/issues.html b/doc/html/issues.html index 3acafe7..0e91509 100644 --- a/doc/html/issues.html +++ b/doc/html/issues.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>Known Issues — S3QL 1.0.1 documentation</title> + <title>Known Issues — S3QL 1.2 documentation</title> <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="_static/underscore.js"></script> <script type="text/javascript" src="_static/doctools.js"></script> <link rel="author" title="About these documents" href="about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="index.html" /> <link rel="next" title="Manpages" href="man/index.html" /> <link rel="prev" title="Tips & Tricks" href="tips.html" /> </head> @@ -40,7 +40,7 @@ <li class="right" > <a href="tips.html" title="Tips & Tricks" accesskey="P">previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -49,6 +49,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="adm.html">Managing Buckets</a></li> @@ -88,6 +89,10 @@ <div class="section" id="known-issues"> <h1>Known Issues<a class="headerlink" href="#known-issues" title="Permalink to this headline">¶</a></h1> <ul> +<li><p class="first">S3QL does not verify TLS/SSL server certificates, so a +man-in-the-middle attack is principally possible. See <a class="reference external" href="http://code.google.com/p/s3ql/issues/detail?id=267">issue 267</a> for more +details.</p> +</li> <li><p class="first">S3QL is rather slow when an application tries to write data in unreasonably small chunks. If a 1 MB file is copied in chunks of 1 KB, this will take more than 10 times as long as when it’s copied @@ -95,7 +100,7 @@ with the (recommended) chunk size of 128 KB.</p> <p>This is a limitation of the FUSE library (which does not yet support write caching) which will hopefully be addressed in some future FUSE version.</p> -<p>Most applications, including e.g. GNU <tt class=" docutils literal"><span class="pre">cp</span></tt> and <tt class=" docutils literal"><span class="pre">rsync</span></tt>, use +<p>Most applications, including e.g. GNU <tt class="docutils literal"><span class="pre">cp</span></tt> and <tt class="docutils literal"><span class="pre">rsync</span></tt>, use reasonably large buffers and are therefore not affected by this problem and perform very efficient on S3QL file systems.</p> <p>However, if you encounter unexpectedly slow performance with a @@ -108,15 +113,15 @@ mount option has been specified: the access time (“atime”) is only u if it is currently earlier than either the status change time (“ctime”) or modification time (“mtime”).</p> </li> -<li><p class="first">S3QL directories always have an <tt class=" docutils literal"><span class="pre">st_nlink</span></tt> value of 1. This may confuse -programs that rely on directories having <tt class=" docutils literal"><span class="pre">st_nlink</span></tt> values of <em>(2 + +<li><p class="first">S3QL directories always have an <tt class="docutils literal"><span class="pre">st_nlink</span></tt> value of 1. This may confuse +programs that rely on directories having <tt class="docutils literal"><span class="pre">st_nlink</span></tt> values of <em>(2 + number of sub directories)</em>.</p> <p>Note that this is not a bug in S3QL. Including sub directories in -the <tt class=" docutils literal"><span class="pre">st_nlink</span></tt> value is a Unix convention, but by no means a +the <tt class="docutils literal"><span class="pre">st_nlink</span></tt> value is a Unix convention, but by no means a requirement. If an application blindly relies on this convention being followed, then this is a bug in the application.</p> <p>A prominent example are early versions of GNU find, which required -the <tt class=" docutils literal"><span class="pre">--noleaf</span></tt> option to work correctly on S3QL file systems. This +the <tt class="docutils literal"><span class="pre">--noleaf</span></tt> option to work correctly on S3QL file systems. This bug has already been fixed in recent find versions.</p> </li> <li><p class="first">In theory, S3QL is not fully compatible with NFS. Since S3QL does @@ -134,23 +139,23 @@ client 1 knowing about this)</li> <p>In this situation it is possible that NFS client 1 actually writes or reads the newly created file B instead. The chances of this are 1 to (2^32 - <em>n</em>) where <em>n</em> is the total number of directory entries -in the S3QL file system (as displayed by <tt class=" docutils literal"><span class="pre">s3qlstat</span></tt>).</p> +in the S3QL file system (as displayed by <tt class="docutils literal"><span class="pre">s3qlstat</span></tt>).</p> <p>Luckily enough, as long as you have less than about 2 thousand million directory entries (2^31), the chances for this are totally irrelevant and you don’t have to worry about it.</p> </li> -<li><p class="first">The <tt class=" docutils literal"><span class="pre">umount</span></tt> and <tt class=" docutils literal"><span class="pre">fusermount</span> <span class="pre">-u</span></tt> commands will <em>not</em> block until all +<li><p class="first">The <tt class="docutils literal"><span class="pre">umount</span></tt> and <tt class="docutils literal"><span class="pre">fusermount</span> <span class="pre">-u</span></tt> commands will <em>not</em> block until all data has been uploaded to the backend. (this is a FUSE limitation that will hopefully be removed in the future, see <a class="reference external" href="http://code.google.com/p/s3ql/issues/detail?id=159">issue 159</a>). If you use either command to unmount an S3QL file system, you have to take care -to explicitly wait for the <tt class=" docutils literal"><span class="pre">mount.s3ql</span></tt> process to terminate before +to explicitly wait for the <tt class="docutils literal"><span class="pre">mount.s3ql</span></tt> process to terminate before you shut down or restart the system. Therefore it is generally not a -good idea to mount an S3QL file system in <tt class=" docutils literal"><span class="pre">/etc/fstab</span></tt> (you should +good idea to mount an S3QL file system in <tt class="docutils literal"><span class="pre">/etc/fstab</span></tt> (you should use a dedicated init script instead).</p> </li> <li><p class="first">S3QL relies on the backends not to run out of space. This is a given for big storage providers like Amazon S3, but you may stumble upon -this if you store buckets e.g. on a small sftp server.</p> +this if you store buckets e.g. on smaller servers or servies.</p> <p>If there is no space left in the backend, attempts to write more data into the S3QL file system will fail and the file system will be in an inconsistent state and require a file system check (and you @@ -180,7 +185,7 @@ the backend.</p> <li class="right" > <a href="tips.html" title="Tips & Tricks" >previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/html/man/adm.html b/doc/html/man/adm.html index 7345230..a28cfbc 100644 --- a/doc/html/man/adm.html +++ b/doc/html/man/adm.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>The s3qladm command — S3QL 1.0.1 documentation</title> + <title>The s3qladm command — S3QL 1.2 documentation</title> <link rel="stylesheet" href="../_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '../', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="../_static/underscore.js"></script> <script type="text/javascript" src="../_static/doctools.js"></script> <link rel="author" title="About these documents" href="../about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="../index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="../index.html" /> <link rel="up" title="Manpages" href="index.html" /> <link rel="next" title="The mount.s3ql command" href="mount.html" /> <link rel="prev" title="The mkfs.s3ql command" href="mkfs.html" /> @@ -41,7 +41,7 @@ <li class="right" > <a href="mkfs.html" title="The mkfs.s3ql command" accesskey="P">previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" accesskey="U">Manpages</a> »</li> </ul> </div> @@ -51,6 +51,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="../about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="../backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="../mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="../adm.html">Managing Buckets</a></li> @@ -116,28 +117,8 @@ <p>The <strong class="program">s3qladm</strong> command performs various operations on S3QL buckets. The file system contained in the bucket <em>must not be mounted</em> when using <strong class="program">s3qladm</strong> or things will go wrong badly.</p> -<p>The form of the storage url depends on the backend that is used. The -following backends are supported:</p> -<div class="section" id="amazon-s3"> -<h3>Amazon S3<a class="headerlink" href="#amazon-s3" title="Permalink to this headline">¶</a></h3> -<p>To store your file system in an Amazon S3 bucket, use a storage URL of -the form <tt class=" docutils literal"><span class="pre">s3://<bucketname></span></tt>. Bucket names must conform to the S3 Bucket -Name Restrictions.</p> -</div> -<div class="section" id="local"> -<h3>Local<a class="headerlink" href="#local" title="Permalink to this headline">¶</a></h3> -<p>The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -<tt class=" docutils literal"><span class="pre">local://<path></span></tt>. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. <tt class=" docutils literal"><span class="pre">local:///var/archive</span></tt>.</p> -</div> -<div class="section" id="sftp"> -<h3>SFTP<a class="headerlink" href="#sftp" title="Permalink to this headline">¶</a></h3> -<p>The storage URL for SFTP connections has the form</p> -<div class="highlight-commandline"><div class="highlight"><pre><span class="l">sftp://</span><span class="nv"><host></span><span class="ge">[:port]</span><span class="l">/</span><span class="nv"><path></span><span class="l"></span> -</pre></div> -</div> -</div> +<p>The storage url depends on the backend that is used. The S3QL User’s +Guide should be consulted for a description of the available backends.</p> </div> <div class="section" id="options"> <h2>Options<a class="headerlink" href="#options" title="Permalink to this headline">¶</a></h2> @@ -150,31 +131,34 @@ slashes to specify an absolute path, e.g. <tt class=" docutils literal"><span cl <tr><td class="option-group" colspan="2"> <kbd><span class="option">--debug <var><module></var></span></kbd></td> </tr> -<tr><td> </td><td>activate debugging output from <module>. Use <tt class=" docutils literal"><span class="pre">all</span></tt> to get +<tr><td> </td><td>activate debugging output from <module>. Use <tt class="docutils literal"><span class="pre">all</span></tt> to get debug messages from all modules. This option can be specified multiple times.</td></tr> <tr><td class="option-group"> <kbd><span class="option">--quiet</span></kbd></td> <td>be really quiet</td></tr> +<tr><td class="option-group"> +<kbd><span class="option">--log <var><target></var></span></kbd></td> +<td>Write logging info into this file. File will be rotated +when it reaches 1 MB, and at most 5 old log files will be +kept. Specify <tt class="docutils literal"><span class="pre">none</span></tt> to disable logging. Default: +<tt class="docutils literal"><span class="pre">none</span></tt></td></tr> +<tr><td class="option-group" colspan="2"> +<kbd><span class="option">--authfile <var><path></var></span></kbd></td> +</tr> +<tr><td> </td><td>Read authentication credentials from this file (default: +<tt class="docutils literal"><span class="pre">~/.s3ql/authinfo2)</span></tt></td></tr> <tr><td class="option-group" colspan="2"> -<kbd><span class="option">--homedir <var><path></var></span></kbd></td> +<kbd><span class="option">--cachedir <var><path></var></span></kbd></td> </tr> -<tr><td> </td><td>Directory for log files, cache and authentication info. -(default: <tt class=" docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> +<tr><td> </td><td>Store cached data in this directory (default: <tt class="docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> <tr><td class="option-group"> <kbd><span class="option">--version</span></kbd></td> <td>just print program version and exit</td></tr> -<tr><td class="option-group"> -<kbd><span class="option">--ssl</span></kbd></td> -<td>Use SSL when connecting to remote servers. This option is -not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext -even for unencrypted file systems.</td></tr> </tbody> </table> </div></blockquote> -<p>Hint: run <tt class=" docutils literal"><span class="pre">s3qladm</span> <span class="pre"><action></span> <span class="pre">--help</span></tt> to get help on the additional +<p>Hint: run <tt class="docutils literal"><span class="pre">s3qladm</span> <span class="pre"><action></span> <span class="pre">--help</span></tt> to get help on the additional arguments that the different actions take.</p> </div> <div class="section" id="actions"> @@ -191,13 +175,6 @@ arguments that the different actions take.</p> <dd>Interactively download backups of the file system metadata.</dd> </dl> </div> -<div class="section" id="files"> -<h2>Files<a class="headerlink" href="#files" title="Permalink to this headline">¶</a></h2> -<p>Authentication data for backends and bucket encryption passphrases are -read from <tt class="file docutils literal"><span class="pre">authinfo</span></tt> in <tt class="file docutils literal"><span class="pre">~/.s3ql</span></tt> or the directory -specified with <tt class="cmdopt docutils literal"><span class="pre">--homedir</span></tt>. Log files are placed in the same -directory.</p> -</div> <div class="section" id="exit-status"> <h2>Exit Status<a class="headerlink" href="#exit-status" title="Permalink to this headline">¶</a></h2> <p><strong class="program">s3qladm</strong> returns exit code 0 if the operation succeeded and 1 if some @@ -227,7 +204,7 @@ system, conventional locations are <tt class="file docutils literal"><span class <li class="right" > <a href="mkfs.html" title="The mkfs.s3ql command" >previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" >Manpages</a> »</li> </ul> </div> diff --git a/doc/html/man/cp.html b/doc/html/man/cp.html index bfc9248..c2e8e0c 100644 --- a/doc/html/man/cp.html +++ b/doc/html/man/cp.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>The s3qlcp command — S3QL 1.0.1 documentation</title> + <title>The s3qlcp command — S3QL 1.2 documentation</title> <link rel="stylesheet" href="../_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '../', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="../_static/underscore.js"></script> <script type="text/javascript" src="../_static/doctools.js"></script> <link rel="author" title="About these documents" href="../about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="../index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="../index.html" /> <link rel="up" title="Manpages" href="index.html" /> <link rel="next" title="The s3qlrm command" href="rm.html" /> <link rel="prev" title="The s3qlctrl command" href="ctrl.html" /> @@ -41,7 +41,7 @@ <li class="right" > <a href="ctrl.html" title="The s3qlctrl command" accesskey="P">previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" accesskey="U">Manpages</a> »</li> </ul> </div> @@ -51,6 +51,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="../about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="../backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="../mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="../adm.html">Managing Buckets</a></li> @@ -117,30 +118,26 @@ Both source and destination must lie inside the same S3QL file system.</p> <p>The replication will not take any additional space. Only if one of directories is modified later on, the modified data will take additional storage space.</p> -<p><tt class=" docutils literal"><span class="pre">s3qlcp</span></tt> can only be called by the user that mounted the file system -and (if the file system was mounted with <tt class=" docutils literal"><span class="pre">--allow-other</span></tt> or <tt class=" docutils literal"><span class="pre">--allow-root</span></tt>) +<p><tt class="docutils literal"><span class="pre">s3qlcp</span></tt> can only be called by the user that mounted the file system +and (if the file system was mounted with <tt class="docutils literal"><span class="pre">--allow-other</span></tt> or <tt class="docutils literal"><span class="pre">--allow-root</span></tt>) the root user. This limitation might be removed in the future (see <a class="reference external" href="http://code.google.com/p/s3ql/issues/detail?id=155">issue 155</a>).</p> <p>Note that:</p> <ul class="simple"> <li>After the replication, both source and target directory will still -be completely ordinary directories. You can regard <tt class=" docutils literal"><span class="pre"><src></span></tt> as a -snapshot of <tt class=" docutils literal"><span class="pre"><target></span></tt> or vice versa. However, the most common -usage of <tt class=" docutils literal"><span class="pre">s3qlcp</span></tt> is to regularly duplicate the same source -directory, say <tt class=" docutils literal"><span class="pre">documents</span></tt>, to different target directories. For a +be completely ordinary directories. You can regard <tt class="docutils literal"><span class="pre"><src></span></tt> as a +snapshot of <tt class="docutils literal"><span class="pre"><target></span></tt> or vice versa. However, the most common +usage of <tt class="docutils literal"><span class="pre">s3qlcp</span></tt> is to regularly duplicate the same source +directory, say <tt class="docutils literal"><span class="pre">documents</span></tt>, to different target directories. For a e.g. monthly replication, the target directories would typically be -named something like <tt class=" docutils literal"><span class="pre">documents_Januray</span></tt> for the replication in -January, <tt class=" docutils literal"><span class="pre">documents_February</span></tt> for the replication in February etc. +named something like <tt class="docutils literal"><span class="pre">documents_January</span></tt> for the replication in +January, <tt class="docutils literal"><span class="pre">documents_February</span></tt> for the replication in February etc. In this case it is clear that the target directories should be regarded as snapshots of the source directory.</li> <li>Exactly the same effect could be achieved by an ordinary copy -program like <tt class=" docutils literal"><span class="pre">cp</span> <span class="pre">-a</span></tt>. However, this procedure would be orders of -magnitude slower, because <tt class=" docutils literal"><span class="pre">cp</span></tt> would have to read every file +program like <tt class="docutils literal"><span class="pre">cp</span> <span class="pre">-a</span></tt>. However, this procedure would be orders of +magnitude slower, because <tt class="docutils literal"><span class="pre">cp</span></tt> would have to read every file completely (so that S3QL had to fetch all the data over the network from the backend) before writing them into the destination folder.</li> -<li>Before starting with the replication, S3QL has to flush the local -cache. So if you just copied lots of new data into the file system -that has not yet been uploaded, replication will take longer than -usual.</li> </ul> <div class="section" id="snapshotting-vs-hardlinking"> <h3>Snapshotting vs Hardlinking<a class="headerlink" href="#snapshotting-vs-hardlinking" title="Permalink to this headline">¶</a></h3> @@ -213,7 +210,7 @@ system, conventional locations are <tt class="file docutils literal"><span class <li class="right" > <a href="ctrl.html" title="The s3qlctrl command" >previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" >Manpages</a> »</li> </ul> </div> diff --git a/doc/html/man/ctrl.html b/doc/html/man/ctrl.html index b6acd2e..adbcb74 100644 --- a/doc/html/man/ctrl.html +++ b/doc/html/man/ctrl.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>The s3qlctrl command — S3QL 1.0.1 documentation</title> + <title>The s3qlctrl command — S3QL 1.2 documentation</title> <link rel="stylesheet" href="../_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '../', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="../_static/underscore.js"></script> <script type="text/javascript" src="../_static/doctools.js"></script> <link rel="author" title="About these documents" href="../about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="../index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="../index.html" /> <link rel="up" title="Manpages" href="index.html" /> <link rel="next" title="The s3qlcp command" href="cp.html" /> <link rel="prev" title="The s3qlstat command" href="stat.html" /> @@ -41,7 +41,7 @@ <li class="right" > <a href="stat.html" title="The s3qlstat command" accesskey="P">previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" accesskey="U">Manpages</a> »</li> </ul> </div> @@ -51,6 +51,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="../about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="../backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="../mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="../adm.html">Managing Buckets</a></li> @@ -116,6 +117,10 @@ <h2>Description<a class="headerlink" href="#description" title="Permalink to this headline">¶</a></h2> <p>The <strong class="command">s3qlctrl</strong> command performs various actions on the S3QL file system mounted in <tt class="var docutils literal"><span class="pre">mountpoint</span></tt>.</p> +<p><strong class="command">s3qlctrl</strong> can only be called by the user that mounted the file system +and (if the file system was mounted with <tt class="cmdopt docutils literal"><span class="pre">--allow-other</span></tt> or +<tt class="cmdopt docutils literal"><span class="pre">--allow-root</span></tt>) the root user. This limitation might be +removed in the future (see <a class="reference external" href="http://code.google.com/p/s3ql/issues/detail?id=155">issue 155</a>).</p> <p>The following actions may be specified:</p> <dl class="docutils"> <dt>flushcache</dt> @@ -164,7 +169,7 @@ what specific action is being invoked:</p> </tbody> </table> </div></blockquote> -<p>Hint: run <tt class=" docutils literal"><span class="pre">s3qlctrl</span> <span class="pre"><action></span> <span class="pre">--help</span></tt> to get help on the additional +<p>Hint: run <tt class="docutils literal"><span class="pre">s3qlctrl</span> <span class="pre"><action></span> <span class="pre">--help</span></tt> to get help on the additional arguments that the different actions take.</p> </div> <div class="section" id="exit-status"> @@ -196,7 +201,7 @@ system, conventional locations are <tt class="file docutils literal"><span class <li class="right" > <a href="stat.html" title="The s3qlstat command" >previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" >Manpages</a> »</li> </ul> </div> diff --git a/doc/html/man/expire_backups.html b/doc/html/man/expire_backups.html index e50daa6..4ebdbc8 100644 --- a/doc/html/man/expire_backups.html +++ b/doc/html/man/expire_backups.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>The expire_backups command — S3QL 1.0.1 documentation</title> + <title>The expire_backups command — S3QL 1.2 documentation</title> <link rel="stylesheet" href="../_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '../', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="../_static/underscore.js"></script> <script type="text/javascript" src="../_static/doctools.js"></script> <link rel="author" title="About these documents" href="../about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="../index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="../index.html" /> <link rel="up" title="Manpages" href="index.html" /> <link rel="next" title="Further Resources / Getting Help" href="../resources.html" /> <link rel="prev" title="The pcp command" href="pcp.html" /> @@ -41,7 +41,7 @@ <li class="right" > <a href="pcp.html" title="The pcp command" accesskey="P">previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" accesskey="U">Manpages</a> »</li> </ul> </div> @@ -51,6 +51,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="../about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="../backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="../mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="../adm.html">Managing Buckets</a></li> @@ -199,7 +200,7 @@ tamper with the state file.</p> dates.</td></tr> <tr><td class="option-group"> <kbd><span class="option">--use-s3qlrm</span></kbd></td> -<td>Use <tt class=" docutils literal"><span class="pre">s3qlrm</span></tt> command to delete backups.</td></tr> +<td>Use <tt class="docutils literal"><span class="pre">s3qlrm</span></tt> command to delete backups.</td></tr> </tbody> </table> </div></blockquote> @@ -230,7 +231,7 @@ error occured.</p> <li class="right" > <a href="pcp.html" title="The pcp command" >previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" >Manpages</a> »</li> </ul> </div> diff --git a/doc/html/man/fsck.html b/doc/html/man/fsck.html index 363ea51..875d0a5 100644 --- a/doc/html/man/fsck.html +++ b/doc/html/man/fsck.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>The fsck.s3ql command — S3QL 1.0.1 documentation</title> + <title>The fsck.s3ql command — S3QL 1.2 documentation</title> <link rel="stylesheet" href="../_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '../', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="../_static/underscore.js"></script> <script type="text/javascript" src="../_static/doctools.js"></script> <link rel="author" title="About these documents" href="../about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="../index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="../index.html" /> <link rel="up" title="Manpages" href="index.html" /> <link rel="next" title="The pcp command" href="pcp.html" /> <link rel="prev" title="The umount.s3ql command" href="umount.html" /> @@ -41,7 +41,7 @@ <li class="right" > <a href="umount.html" title="The umount.s3ql command" accesskey="P">previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" accesskey="U">Manpages</a> »</li> </ul> </div> @@ -51,6 +51,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="../about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="../backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="../mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="../adm.html">Managing Buckets</a></li> @@ -113,29 +114,9 @@ <h2>Description<a class="headerlink" href="#description" title="Permalink to this headline">¶</a></h2> <p>The <strong class="command">mkfs.s3ql</strong> command checks the new file system in the location specified by <em>storage url</em> for errors and attempts to repair any -problems.</p> -<p>The form of the storage url depends on the backend that is used. The -following backends are supported:</p> -<div class="section" id="amazon-s3"> -<h3>Amazon S3<a class="headerlink" href="#amazon-s3" title="Permalink to this headline">¶</a></h3> -<p>To store your file system in an Amazon S3 bucket, use a storage URL of -the form <tt class=" docutils literal"><span class="pre">s3://<bucketname></span></tt>. Bucket names must conform to the S3 Bucket -Name Restrictions.</p> -</div> -<div class="section" id="local"> -<h3>Local<a class="headerlink" href="#local" title="Permalink to this headline">¶</a></h3> -<p>The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -<tt class=" docutils literal"><span class="pre">local://<path></span></tt>. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. <tt class=" docutils literal"><span class="pre">local:///var/archive</span></tt>.</p> -</div> -<div class="section" id="sftp"> -<h3>SFTP<a class="headerlink" href="#sftp" title="Permalink to this headline">¶</a></h3> -<p>The storage URL for SFTP connections has the form</p> -<div class="highlight-commandline"><div class="highlight"><pre><span class="l">sftp://</span><span class="nv"><host></span><span class="ge">[:port]</span><span class="l">/</span><span class="nv"><path></span><span class="l"></span> -</pre></div> -</div> -</div> +problems. The storage url depends on the backend that is used. The +S3QL User’s Guide should be consulted for a description of the +available backends.</p> </div> <div class="section" id="options"> <h2>Options<a class="headerlink" href="#options" title="Permalink to this headline">¶</a></h2> @@ -145,15 +126,25 @@ slashes to specify an absolute path, e.g. <tt class=" docutils literal"><span cl <col class="option" /> <col class="description" /> <tbody valign="top"> +<tr><td class="option-group"> +<kbd><span class="option">--log <var><target></var></span></kbd></td> +<td>Write logging info into this file. File will be rotated +when it reaches 1 MB, and at most 5 old log files will be +kept. Specify <tt class="docutils literal"><span class="pre">none</span></tt> to disable logging. Default: +<tt class="docutils literal"><span class="pre">~/.s3ql/fsck.log</span></tt></td></tr> +<tr><td class="option-group" colspan="2"> +<kbd><span class="option">--cachedir <var><path></var></span></kbd></td> +</tr> +<tr><td> </td><td>Store cached data in this directory (default: <tt class="docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> <tr><td class="option-group" colspan="2"> -<kbd><span class="option">--homedir <var><path></var></span></kbd></td> +<kbd><span class="option">--authfile <var><path></var></span></kbd></td> </tr> -<tr><td> </td><td>Directory for log files, cache and authentication info. -(default: <tt class=" docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> +<tr><td> </td><td>Read authentication credentials from this file (default: +<tt class="docutils literal"><span class="pre">~/.s3ql/authinfo2)</span></tt></td></tr> <tr><td class="option-group" colspan="2"> <kbd><span class="option">--debug <var><module></var></span></kbd></td> </tr> -<tr><td> </td><td>activate debugging output from <module>. Use <tt class=" docutils literal"><span class="pre">all</span></tt> to get +<tr><td> </td><td>activate debugging output from <module>. Use <tt class="docutils literal"><span class="pre">all</span></tt> to get debug messages from all modules. This option can be specified multiple times.</td></tr> <tr><td class="option-group"> @@ -163,13 +154,6 @@ specified multiple times.</td></tr> <kbd><span class="option">--version</span></kbd></td> <td>just print program version and exit</td></tr> <tr><td class="option-group"> -<kbd><span class="option">--ssl</span></kbd></td> -<td>Use SSL when connecting to remote servers. This option is -not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext even -for unencrypted file systems.</td></tr> -<tr><td class="option-group"> <kbd><span class="option">--batch</span></kbd></td> <td>If user input is required, exit without prompting.</td></tr> <tr><td class="option-group"> @@ -179,13 +163,6 @@ for unencrypted file systems.</td></tr> </table> </div></blockquote> </div> -<div class="section" id="files"> -<h2>Files<a class="headerlink" href="#files" title="Permalink to this headline">¶</a></h2> -<p>Authentication data for backends and bucket encryption passphrases are -read from <tt class="file docutils literal"><span class="pre">authinfo</span></tt> in <tt class="file docutils literal"><span class="pre">~/.s3ql</span></tt> or the directory -specified with <tt class="cmdopt docutils literal"><span class="pre">--homedir</span></tt>. Log files are placed in the same -directory.</p> -</div> <div class="section" id="exit-status"> <h2>Exit Status<a class="headerlink" href="#exit-status" title="Permalink to this headline">¶</a></h2> <p><strong class="command">mkfs.s3ql</strong> returns exit code 0 if the operation succeeded and 1 if some @@ -215,7 +192,7 @@ system, conventional locations are <tt class="file docutils literal"><span class <li class="right" > <a href="umount.html" title="The umount.s3ql command" >previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" >Manpages</a> »</li> </ul> </div> diff --git a/doc/html/man/index.html b/doc/html/man/index.html index a3141b9..8b5c000 100644 --- a/doc/html/man/index.html +++ b/doc/html/man/index.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>Manpages — S3QL 1.0.1 documentation</title> + <title>Manpages — S3QL 1.2 documentation</title> <link rel="stylesheet" href="../_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '../', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="../_static/underscore.js"></script> <script type="text/javascript" src="../_static/doctools.js"></script> <link rel="author" title="About these documents" href="../about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="../index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="../index.html" /> <link rel="next" title="The mkfs.s3ql command" href="mkfs.html" /> <link rel="prev" title="Known Issues" href="../issues.html" /> </head> @@ -40,7 +40,7 @@ <li class="right" > <a href="../issues.html" title="Known Issues" accesskey="P">previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -49,6 +49,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="../about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="../backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="../mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="../adm.html">Managing Buckets</a></li> @@ -137,7 +138,7 @@ here in the User’s Guide.</p> <li class="right" > <a href="../issues.html" title="Known Issues" >previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/html/man/lock.html b/doc/html/man/lock.html index acb65da..36a08d0 100644 --- a/doc/html/man/lock.html +++ b/doc/html/man/lock.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>The s3qllock command — S3QL 1.0.1 documentation</title> + <title>The s3qllock command — S3QL 1.2 documentation</title> <link rel="stylesheet" href="../_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '../', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="../_static/underscore.js"></script> <script type="text/javascript" src="../_static/doctools.js"></script> <link rel="author" title="About these documents" href="../about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="../index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="../index.html" /> <link rel="up" title="Manpages" href="index.html" /> <link rel="next" title="The umount.s3ql command" href="umount.html" /> <link rel="prev" title="The s3qlrm command" href="rm.html" /> @@ -41,7 +41,7 @@ <li class="right" > <a href="rm.html" title="The s3qlrm command" accesskey="P">previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" accesskey="U">Manpages</a> »</li> </ul> </div> @@ -51,6 +51,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="../about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="../backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="../mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="../adm.html">Managing Buckets</a></li> @@ -116,6 +117,10 @@ system immutable. Immutable trees can no longer be changed in any way whatsoever. You can not add new files or directories and you can not change or delete existing files and directories. The only way to get rid of an immutable tree is to use the <strong class="program">s3qlrm</strong> command.</p> +<p><strong class="command">s3qllock</strong> can only be called by the user that mounted the file system +and (if the file system was mounted with <tt class="cmdopt docutils literal"><span class="pre">--allow-other</span></tt> or +<tt class="cmdopt docutils literal"><span class="pre">--allow-root</span></tt>) the root user. This limitation might be +removed in the future (see <a class="reference external" href="http://code.google.com/p/s3ql/issues/detail?id=155">issue 155</a>).</p> </div> <div class="section" id="rationale"> <h2>Rationale<a class="headerlink" href="#rationale" title="Permalink to this headline">¶</a></h2> @@ -196,7 +201,7 @@ system, conventional locations are <tt class="file docutils literal"><span class <li class="right" > <a href="rm.html" title="The s3qlrm command" >previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" >Manpages</a> »</li> </ul> </div> diff --git a/doc/html/man/mkfs.html b/doc/html/man/mkfs.html index 8117601..3047249 100644 --- a/doc/html/man/mkfs.html +++ b/doc/html/man/mkfs.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>The mkfs.s3ql command — S3QL 1.0.1 documentation</title> + <title>The mkfs.s3ql command — S3QL 1.2 documentation</title> <link rel="stylesheet" href="../_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '../', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="../_static/underscore.js"></script> <script type="text/javascript" src="../_static/doctools.js"></script> <link rel="author" title="About these documents" href="../about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="../index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="../index.html" /> <link rel="up" title="Manpages" href="index.html" /> <link rel="next" title="The s3qladm command" href="adm.html" /> <link rel="prev" title="Manpages" href="index.html" /> @@ -41,7 +41,7 @@ <li class="right" > <a href="index.html" title="Manpages" accesskey="P">previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" accesskey="U">Manpages</a> »</li> </ul> </div> @@ -51,6 +51,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="../about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="../backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="../mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="../adm.html">Managing Buckets</a></li> @@ -112,29 +113,13 @@ <div class="section" id="description"> <h2>Description<a class="headerlink" href="#description" title="Permalink to this headline">¶</a></h2> <p>The <strong class="command">mkfs.s3ql</strong> command creates a new file system in the location -specified by <em>storage url</em>.</p> -<p>The form of the storage url depends on the backend that is used. The -following backends are supported:</p> -<div class="section" id="amazon-s3"> -<h3>Amazon S3<a class="headerlink" href="#amazon-s3" title="Permalink to this headline">¶</a></h3> -<p>To store your file system in an Amazon S3 bucket, use a storage URL of -the form <tt class=" docutils literal"><span class="pre">s3://<bucketname></span></tt>. Bucket names must conform to the S3 Bucket -Name Restrictions.</p> -</div> -<div class="section" id="local"> -<h3>Local<a class="headerlink" href="#local" title="Permalink to this headline">¶</a></h3> -<p>The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -<tt class=" docutils literal"><span class="pre">local://<path></span></tt>. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. <tt class=" docutils literal"><span class="pre">local:///var/archive</span></tt>.</p> -</div> -<div class="section" id="sftp"> -<h3>SFTP<a class="headerlink" href="#sftp" title="Permalink to this headline">¶</a></h3> -<p>The storage URL for SFTP connections has the form</p> -<div class="highlight-commandline"><div class="highlight"><pre><span class="l">sftp://</span><span class="nv"><host></span><span class="ge">[:port]</span><span class="l">/</span><span class="nv"><path></span><span class="l"></span> -</pre></div> -</div> -</div> +specified by <em>storage url</em>. The storage url depends on the backend +that is used. The S3QL User’s Guide should be consulted for a +description of the available backends.</p> +<p>Unless you have specified the <tt class="docutils literal"><span class="pre">--plain</span></tt> option, <tt class="docutils literal"><span class="pre">mkfs.s3ql</span></tt> will ask +you to enter an encryption password. This password will <em>not</em> be read +from an authentication file specified with the <tt class="cmdopt docutils literal"><span class="pre">--authfile</span></tt> +option to prevent accidental creation of an encrypted bucket.</p> </div> <div class="section" id="options"> <h2>Options<a class="headerlink" href="#options" title="Permalink to this headline">¶</a></h2> @@ -145,16 +130,20 @@ slashes to specify an absolute path, e.g. <tt class=" docutils literal"><span cl <col class="description" /> <tbody valign="top"> <tr><td class="option-group" colspan="2"> -<kbd><span class="option">--homedir <var><path></var></span></kbd></td> +<kbd><span class="option">--cachedir <var><path></var></span></kbd></td> </tr> -<tr><td> </td><td>Directory for log files, cache and authentication -info. (default: <tt class=" docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> +<tr><td> </td><td>Store cached data in this directory (default: <tt class="docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> +<tr><td class="option-group" colspan="2"> +<kbd><span class="option">--authfile <var><path></var></span></kbd></td> +</tr> +<tr><td> </td><td>Read authentication credentials from this file (default: +<tt class="docutils literal"><span class="pre">~/.s3ql/authinfo2)</span></tt></td></tr> <tr><td class="option-group" colspan="2"> <kbd><span class="option">--debug <var><module></var></span></kbd></td> </tr> -<tr><td> </td><td>activate debugging output from <module>. Use <tt class=" docutils literal"><span class="pre">all</span></tt> to -get debug messages from all modules. This option can -be specified multiple times.</td></tr> +<tr><td> </td><td>activate debugging output from <module>. Use <tt class="docutils literal"><span class="pre">all</span></tt> to +get debug messages from all modules. This option can be +specified multiple times.</td></tr> <tr><td class="option-group"> <kbd><span class="option">--quiet</span></kbd></td> <td>be really quiet</td></tr> @@ -162,19 +151,6 @@ be specified multiple times.</td></tr> <kbd><span class="option">--version</span></kbd></td> <td>just print program version and exit</td></tr> <tr><td class="option-group"> -<kbd><span class="option">--ssl</span></kbd></td> -<td>Use SSL when connecting to remote servers. This option -is not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext -even for unencrypted file systems.</td></tr> -<tr><td class="option-group" colspan="2"> -<kbd><span class="option">--s3-location <var><name></var></span></kbd></td> -</tr> -<tr><td> </td><td>Storage location for new S3 buckets. Allowed values: -<tt class=" docutils literal"><span class="pre">EU</span></tt>, <tt class=" docutils literal"><span class="pre">us-west-1</span></tt>, <tt class=" docutils literal"><span class="pre">ap-southeast-1</span></tt>, or <tt class=" docutils literal"><span class="pre">us-standard</span></tt>. -(default: EU)</td></tr> -<tr><td class="option-group"> <kbd><span class="option">-L <var><name></var></span></kbd></td> <td>Filesystem label</td></tr> <tr><td class="option-group" colspan="2"> @@ -191,13 +167,6 @@ even for unencrypted file systems.</td></tr> </table> </div></blockquote> </div> -<div class="section" id="files"> -<h2>Files<a class="headerlink" href="#files" title="Permalink to this headline">¶</a></h2> -<p>Authentication data for backends and bucket encryption passphrases are -read from <tt class="file docutils literal"><span class="pre">authinfo</span></tt> in <tt class="file docutils literal"><span class="pre">~/.s3ql</span></tt> or the directory -specified with <tt class="cmdopt docutils literal"><span class="pre">--homedir</span></tt>. Log files are placed in the same -directory.</p> -</div> <div class="section" id="exit-status"> <h2>Exit Status<a class="headerlink" href="#exit-status" title="Permalink to this headline">¶</a></h2> <p><strong class="command">mkfs.s3ql</strong> returns exit code 0 if the operation succeeded and 1 if some @@ -227,7 +196,7 @@ system, conventional locations are <tt class="file docutils literal"><span class <li class="right" > <a href="index.html" title="Manpages" >previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" >Manpages</a> »</li> </ul> </div> diff --git a/doc/html/man/mount.html b/doc/html/man/mount.html index 5f91bc6..2873b2d 100644 --- a/doc/html/man/mount.html +++ b/doc/html/man/mount.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>The mount.s3ql command — S3QL 1.0.1 documentation</title> + <title>The mount.s3ql command — S3QL 1.2 documentation</title> <link rel="stylesheet" href="../_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '../', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="../_static/underscore.js"></script> <script type="text/javascript" src="../_static/doctools.js"></script> <link rel="author" title="About these documents" href="../about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="../index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="../index.html" /> <link rel="up" title="Manpages" href="index.html" /> <link rel="next" title="The s3qlstat command" href="stat.html" /> <link rel="prev" title="The s3qladm command" href="adm.html" /> @@ -41,7 +41,7 @@ <li class="right" > <a href="adm.html" title="The s3qladm command" accesskey="P">previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" accesskey="U">Manpages</a> »</li> </ul> </div> @@ -51,6 +51,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="../about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="../backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="../mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="../adm.html">Managing Buckets</a></li> @@ -112,29 +113,9 @@ <div class="section" id="description"> <h2>Description<a class="headerlink" href="#description" title="Permalink to this headline">¶</a></h2> <p>The <strong class="command">mount.s3ql</strong> command mounts the S3QL file system stored in <em>storage -url</em> in the directory <em>mount point</em>.</p> -<p>The form of the storage url depends on the backend that is used. The -following backends are supported:</p> -<div class="section" id="amazon-s3"> -<h3>Amazon S3<a class="headerlink" href="#amazon-s3" title="Permalink to this headline">¶</a></h3> -<p>To store your file system in an Amazon S3 bucket, use a storage URL of -the form <tt class=" docutils literal"><span class="pre">s3://<bucketname></span></tt>. Bucket names must conform to the S3 Bucket -Name Restrictions.</p> -</div> -<div class="section" id="local"> -<h3>Local<a class="headerlink" href="#local" title="Permalink to this headline">¶</a></h3> -<p>The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -<tt class=" docutils literal"><span class="pre">local://<path></span></tt>. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. <tt class=" docutils literal"><span class="pre">local:///var/archive</span></tt>.</p> -</div> -<div class="section" id="sftp"> -<h3>SFTP<a class="headerlink" href="#sftp" title="Permalink to this headline">¶</a></h3> -<p>The storage URL for SFTP connections has the form</p> -<div class="highlight-commandline"><div class="highlight"><pre><span class="l">sftp://</span><span class="nv"><host></span><span class="ge">[:port]</span><span class="l">/</span><span class="nv"><path></span><span class="l"></span> -</pre></div> -</div> -</div> +url</em> in the directory <em>mount point</em>. The storage url depends on the +backend that is used. The S3QL User’s Guide should be consulted for a +description of the available backends.</p> </div> <div class="section" id="options"> <h2>Options<a class="headerlink" href="#options" title="Permalink to this headline">¶</a></h2> @@ -144,15 +125,26 @@ slashes to specify an absolute path, e.g. <tt class=" docutils literal"><span cl <col class="option" /> <col class="description" /> <tbody valign="top"> +<tr><td class="option-group"> +<kbd><span class="option">--log <var><target></var></span></kbd></td> +<td>Write logging info into this file. File will be +rotated when it reaches 1 MB, and at most 5 old log +files will be kept. Specify <tt class="docutils literal"><span class="pre">none</span></tt> to disable +logging. Default: <tt class="docutils literal"><span class="pre">~/.s3ql/mount.log</span></tt></td></tr> +<tr><td class="option-group" colspan="2"> +<kbd><span class="option">--cachedir <var><path></var></span></kbd></td> +</tr> +<tr><td> </td><td>Store cached data in this directory (default: +<tt class="docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> <tr><td class="option-group" colspan="2"> -<kbd><span class="option">--homedir <var><path></var></span></kbd></td> +<kbd><span class="option">--authfile <var><path></var></span></kbd></td> </tr> -<tr><td> </td><td>Directory for log files, cache and authentication -info. (default: <tt class=" docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> +<tr><td> </td><td>Read authentication credentials from this file +(default: <tt class="docutils literal"><span class="pre">~/.s3ql/authinfo2)</span></tt></td></tr> <tr><td class="option-group" colspan="2"> <kbd><span class="option">--debug <var><module></var></span></kbd></td> </tr> -<tr><td> </td><td>activate debugging output from <module>. Use <tt class=" docutils literal"><span class="pre">all</span></tt> to +<tr><td> </td><td>activate debugging output from <module>. Use <tt class="docutils literal"><span class="pre">all</span></tt> to get debug messages from all modules. This option can be specified multiple times.</td></tr> <tr><td class="option-group"> @@ -161,13 +153,6 @@ be specified multiple times.</td></tr> <tr><td class="option-group"> <kbd><span class="option">--version</span></kbd></td> <td>just print program version and exit</td></tr> -<tr><td class="option-group"> -<kbd><span class="option">--ssl</span></kbd></td> -<td>Use SSL when connecting to remote servers. This option -is not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext -even for unencrypted file systems.</td></tr> <tr><td class="option-group" colspan="2"> <kbd><span class="option">--cachesize <var><size></var></span></kbd></td> </tr> @@ -183,20 +168,20 @@ operation.</td></tr> Each cache entry requires one file descriptor, so if you increase this number you have to make sure that your process file descriptor limit (as set with -<tt class=" docutils literal"><span class="pre">ulimit</span> <span class="pre">-n</span></tt>) is high enough (at least the number of +<tt class="docutils literal"><span class="pre">ulimit</span> <span class="pre">-n</span></tt>) is high enough (at least the number of cache entries + 100).</td></tr> <tr><td class="option-group"> <kbd><span class="option">--allow-other</span></kbd></td> -<td>Normally, only the user who called <tt class=" docutils literal"><span class="pre">mount.s3ql</span></tt> can +<td>Normally, only the user who called <tt class="docutils literal"><span class="pre">mount.s3ql</span></tt> can access the mount point. This user then also has full access to it, independent of individual file -permissions. If the <tt class=" docutils literal"><span class="pre">--allow-other</span></tt> option is +permissions. If the <tt class="docutils literal"><span class="pre">--allow-other</span></tt> option is specified, other users can access the mount point as well and individual file permissions are taken into account for all users.</td></tr> <tr><td class="option-group"> <kbd><span class="option">--allow-root</span></kbd></td> -<td>Like <tt class=" docutils literal"><span class="pre">--allow-other</span></tt>, but restrict access to the +<td>Like <tt class="docutils literal"><span class="pre">--allow-other</span></tt>, but restrict access to the mounting user and the root user.</td></tr> <tr><td class="option-group"> <kbd><span class="option">--fg</span></kbd></td> @@ -217,29 +202,25 @@ this, then you don’t need it.</td></tr> <kbd><span class="option">--compress <var><name></var></span></kbd></td> </tr> <tr><td> </td><td>Compression algorithm to use when storing new data. -Allowed values: <tt class=" docutils literal"><span class="pre">lzma</span></tt>, <tt class=" docutils literal"><span class="pre">bzip2</span></tt>, <tt class=" docutils literal"><span class="pre">zlib</span></tt>, none. -(default: <tt class=" docutils literal"><span class="pre">lzma</span></tt>)</td></tr> +Allowed values: <tt class="docutils literal"><span class="pre">lzma</span></tt>, <tt class="docutils literal"><span class="pre">bzip2</span></tt>, <tt class="docutils literal"><span class="pre">zlib</span></tt>, none. +(default: <tt class="docutils literal"><span class="pre">lzma</span></tt>)</td></tr> <tr><td class="option-group" colspan="2"> <kbd><span class="option">--metadata-upload-interval <var><seconds></var></span></kbd></td> </tr> <tr><td> </td><td>Interval in seconds between complete metadata uploads. Set to 0 to disable. Default: 24h.</td></tr> -<tr><td class="option-group" colspan="2"> -<kbd><span class="option">--compression-threads <var><no></var></span></kbd></td> -</tr> -<tr><td> </td><td>Number of parallel compression and encryption threads -to use (default: 1).</td></tr> +<tr><td class="option-group"> +<kbd><span class="option">--threads <var><no></var></span></kbd></td> +<td>Number of parallel upload threads to use (default: +auto).</td></tr> +<tr><td class="option-group"> +<kbd><span class="option">--nfs</span></kbd></td> +<td>Support export of S3QL file systems over NFS (default: +False)</td></tr> </tbody> </table> </div></blockquote> </div> -<div class="section" id="files"> -<h2>Files<a class="headerlink" href="#files" title="Permalink to this headline">¶</a></h2> -<p>Authentication data for backends and bucket encryption passphrases are -read from <tt class="file docutils literal"><span class="pre">authinfo</span></tt> in <tt class="file docutils literal"><span class="pre">~/.s3ql</span></tt> or the directory -specified with <tt class="cmdopt docutils literal"><span class="pre">--homedir</span></tt>. Log files are placed in the same -directory.</p> -</div> <div class="section" id="exit-status"> <h2>Exit Status<a class="headerlink" href="#exit-status" title="Permalink to this headline">¶</a></h2> <p><strong class="command">mount.s3ql</strong> returns exit code 0 if the operation succeeded and 1 if some @@ -269,7 +250,7 @@ system, conventional locations are <tt class="file docutils literal"><span class <li class="right" > <a href="adm.html" title="The s3qladm command" >previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" >Manpages</a> »</li> </ul> </div> diff --git a/doc/html/man/pcp.html b/doc/html/man/pcp.html index 4dff9aa..1f03004 100644 --- a/doc/html/man/pcp.html +++ b/doc/html/man/pcp.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>The pcp command — S3QL 1.0.1 documentation</title> + <title>The pcp command — S3QL 1.2 documentation</title> <link rel="stylesheet" href="../_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '../', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="../_static/underscore.js"></script> <script type="text/javascript" src="../_static/doctools.js"></script> <link rel="author" title="About these documents" href="../about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="../index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="../index.html" /> <link rel="up" title="Manpages" href="index.html" /> <link rel="next" title="The expire_backups command" href="expire_backups.html" /> <link rel="prev" title="The fsck.s3ql command" href="fsck.html" /> @@ -41,7 +41,7 @@ <li class="right" > <a href="fsck.html" title="The fsck.s3ql command" accesskey="P">previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" accesskey="U">Manpages</a> »</li> </ul> </div> @@ -51,6 +51,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="../about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="../backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="../mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="../adm.html">Managing Buckets</a></li> @@ -115,6 +116,9 @@ <strong class="program">sync</strong> processes to copy directory trees in parallel. This is allows much better copying performance on file system that have relatively high latency when retrieving individual files like S3QL.</p> +<p><strong>Note</strong>: Using this program only improves performance when copying +<em>from</em> an S3QL file system. When copying <em>to</em> an S3QL file system, +using <strong class="command">pcp</strong> is more likely to <em>decrease</em> performance.</p> </div> <div class="section" id="options"> <h2>Options<a class="headerlink" href="#options" title="Permalink to this headline">¶</a></h2> @@ -170,7 +174,7 @@ error occured.</p> <li class="right" > <a href="fsck.html" title="The fsck.s3ql command" >previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" >Manpages</a> »</li> </ul> </div> diff --git a/doc/html/man/rm.html b/doc/html/man/rm.html index 2ab13b1..a308d9a 100644 --- a/doc/html/man/rm.html +++ b/doc/html/man/rm.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>The s3qlrm command — S3QL 1.0.1 documentation</title> + <title>The s3qlrm command — S3QL 1.2 documentation</title> <link rel="stylesheet" href="../_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '../', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="../_static/underscore.js"></script> <script type="text/javascript" src="../_static/doctools.js"></script> <link rel="author" title="About these documents" href="../about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="../index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="../index.html" /> <link rel="up" title="Manpages" href="index.html" /> <link rel="next" title="The s3qllock command" href="lock.html" /> <link rel="prev" title="The s3qlcp command" href="cp.html" /> @@ -41,7 +41,7 @@ <li class="right" > <a href="cp.html" title="The s3qlcp command" accesskey="P">previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" accesskey="U">Manpages</a> »</li> </ul> </div> @@ -51,6 +51,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="../about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="../backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="../mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="../adm.html">Managing Buckets</a></li> @@ -118,6 +119,10 @@ you to delete immutable trees (which can be created with <strong class="program">s3qllock</strong>) as well.</p> <p>Be warned that there is no additional confirmation. The directory will be removed entirely and immediately.</p> +<p><strong class="command">s3qlrm</strong> can only be called by the user that mounted the file system +and (if the file system was mounted with <tt class="cmdopt docutils literal"><span class="pre">--allow-other</span></tt> or +<tt class="cmdopt docutils literal"><span class="pre">--allow-root</span></tt>) the root user. This limitation might be +removed in the future (see <a class="reference external" href="http://code.google.com/p/s3ql/issues/detail?id=155">issue 155</a>).</p> </div> <div class="section" id="options"> <h2>Options<a class="headerlink" href="#options" title="Permalink to this headline">¶</a></h2> @@ -169,7 +174,7 @@ system, conventional locations are <tt class="file docutils literal"><span class <li class="right" > <a href="cp.html" title="The s3qlcp command" >previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" >Manpages</a> »</li> </ul> </div> diff --git a/doc/html/man/stat.html b/doc/html/man/stat.html index 9182687..d96a19d 100644 --- a/doc/html/man/stat.html +++ b/doc/html/man/stat.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>The s3qlstat command — S3QL 1.0.1 documentation</title> + <title>The s3qlstat command — S3QL 1.2 documentation</title> <link rel="stylesheet" href="../_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '../', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="../_static/underscore.js"></script> <script type="text/javascript" src="../_static/doctools.js"></script> <link rel="author" title="About these documents" href="../about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="../index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="../index.html" /> <link rel="up" title="Manpages" href="index.html" /> <link rel="next" title="The s3qlctrl command" href="ctrl.html" /> <link rel="prev" title="The mount.s3ql command" href="mount.html" /> @@ -41,7 +41,7 @@ <li class="right" > <a href="mount.html" title="The mount.s3ql command" accesskey="P">previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" accesskey="U">Manpages</a> »</li> </ul> </div> @@ -51,6 +51,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="../about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="../backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="../mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="../adm.html">Managing Buckets</a></li> @@ -168,7 +169,7 @@ system, conventional locations are <tt class="file docutils literal"><span class <li class="right" > <a href="mount.html" title="The mount.s3ql command" >previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" >Manpages</a> »</li> </ul> </div> diff --git a/doc/html/man/umount.html b/doc/html/man/umount.html index 30039fc..24eebf3 100644 --- a/doc/html/man/umount.html +++ b/doc/html/man/umount.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>The umount.s3ql command — S3QL 1.0.1 documentation</title> + <title>The umount.s3ql command — S3QL 1.2 documentation</title> <link rel="stylesheet" href="../_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '../', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="../_static/underscore.js"></script> <script type="text/javascript" src="../_static/doctools.js"></script> <link rel="author" title="About these documents" href="../about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="../index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="../index.html" /> <link rel="up" title="Manpages" href="index.html" /> <link rel="next" title="The fsck.s3ql command" href="fsck.html" /> <link rel="prev" title="The s3qllock command" href="lock.html" /> @@ -41,7 +41,7 @@ <li class="right" > <a href="lock.html" title="The s3qllock command" accesskey="P">previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" accesskey="U">Manpages</a> »</li> </ul> </div> @@ -51,6 +51,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="../about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="../backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="../mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="../adm.html">Managing Buckets</a></li> @@ -178,7 +179,7 @@ system, conventional locations are <tt class="file docutils literal"><span class <li class="right" > <a href="lock.html" title="The s3qllock command" >previous</a> |</li> - <li><a href="../index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="../index.html">S3QL 1.2 documentation</a> »</li> <li><a href="index.html" >Manpages</a> »</li> </ul> </div> diff --git a/doc/html/mkfs.html b/doc/html/mkfs.html index 4acbe34..a627a91 100644 --- a/doc/html/mkfs.html +++ b/doc/html/mkfs.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>File System Creation — S3QL 1.0.1 documentation</title> + <title>File System Creation — S3QL 1.2 documentation</title> <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="_static/underscore.js"></script> <script type="text/javascript" src="_static/doctools.js"></script> <link rel="author" title="About these documents" href="about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="index.html" /> <link rel="next" title="Managing Buckets" href="adm.html" /> <link rel="prev" title="Storage Backends" href="backends.html" /> </head> @@ -40,7 +40,7 @@ <li class="right" > <a href="backends.html" title="Storage Backends" accesskey="P">previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -49,6 +49,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a></li> <li class="toctree-l1 current"><a class="current reference internal" href="">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="adm.html">Managing Buckets</a></li> @@ -87,7 +88,7 @@ <div class="section" id="file-system-creation"> <h1>File System Creation<a class="headerlink" href="#file-system-creation" title="Permalink to this headline">¶</a></h1> -<p>A S3QL file system is created with the <tt class=" docutils literal"><span class="pre">mkfs.s3ql</span></tt> command. It has the +<p>A S3QL file system is created with the <tt class="docutils literal"><span class="pre">mkfs.s3ql</span></tt> command. It has the following syntax:</p> <div class="highlight-commandline"><div class="highlight"><pre><span class="l">mkfs.s3ql </span><span class="ge">[options]</span><span class="l"> </span><span class="nv"><storage url></span><span class="l"></span> </pre></div> @@ -99,16 +100,20 @@ following syntax:</p> <col class="description" /> <tbody valign="top"> <tr><td class="option-group" colspan="2"> -<kbd><span class="option">--homedir <var><path></var></span></kbd></td> +<kbd><span class="option">--cachedir <var><path></var></span></kbd></td> </tr> -<tr><td> </td><td>Directory for log files, cache and authentication -info. (default: <tt class=" docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> +<tr><td> </td><td>Store cached data in this directory (default: <tt class="docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> +<tr><td class="option-group" colspan="2"> +<kbd><span class="option">--authfile <var><path></var></span></kbd></td> +</tr> +<tr><td> </td><td>Read authentication credentials from this file (default: +<tt class="docutils literal"><span class="pre">~/.s3ql/authinfo2)</span></tt></td></tr> <tr><td class="option-group" colspan="2"> <kbd><span class="option">--debug <var><module></var></span></kbd></td> </tr> -<tr><td> </td><td>activate debugging output from <module>. Use <tt class=" docutils literal"><span class="pre">all</span></tt> to -get debug messages from all modules. This option can -be specified multiple times.</td></tr> +<tr><td> </td><td>activate debugging output from <module>. Use <tt class="docutils literal"><span class="pre">all</span></tt> to +get debug messages from all modules. This option can be +specified multiple times.</td></tr> <tr><td class="option-group"> <kbd><span class="option">--quiet</span></kbd></td> <td>be really quiet</td></tr> @@ -116,19 +121,6 @@ be specified multiple times.</td></tr> <kbd><span class="option">--version</span></kbd></td> <td>just print program version and exit</td></tr> <tr><td class="option-group"> -<kbd><span class="option">--ssl</span></kbd></td> -<td>Use SSL when connecting to remote servers. This option -is not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext -even for unencrypted file systems.</td></tr> -<tr><td class="option-group" colspan="2"> -<kbd><span class="option">--s3-location <var><name></var></span></kbd></td> -</tr> -<tr><td> </td><td>Storage location for new S3 buckets. Allowed values: -<tt class=" docutils literal"><span class="pre">EU</span></tt>, <tt class=" docutils literal"><span class="pre">us-west-1</span></tt>, <tt class=" docutils literal"><span class="pre">ap-southeast-1</span></tt>, or <tt class=" docutils literal"><span class="pre">us-standard</span></tt>. -(default: EU)</td></tr> -<tr><td class="option-group"> <kbd><span class="option">-L <var><name></var></span></kbd></td> <td>Filesystem label</td></tr> <tr><td class="option-group" colspan="2"> @@ -144,10 +136,10 @@ even for unencrypted file systems.</td></tr> </tbody> </table> </div></blockquote> -<p>Unless you have specified the <tt class=" docutils literal"><span class="pre">--plain</span></tt> option, <tt class=" docutils literal"><span class="pre">mkfs.s3ql</span></tt> will ask you -to enter an encryption password. If you do not want to enter this -password every time that you mount the file system, you can store it -in the <tt class=" docutils literal"><span class="pre">~/.s3ql/authinfo</span></tt> file, see <a class="reference internal" href="mount.html#bucket-pw"><em>Storing Encryption Passwords</em></a>.</p> +<p>Unless you have specified the <tt class="docutils literal"><span class="pre">--plain</span></tt> option, <tt class="docutils literal"><span class="pre">mkfs.s3ql</span></tt> will ask +you to enter an encryption password. This password will <em>not</em> be read +from an authentication file specified with the <tt class="cmdopt docutils literal"><span class="pre">--authfile</span></tt> +option to prevent accidental creation of an encrypted bucket.</p> </div> @@ -165,7 +157,7 @@ in the <tt class=" docutils literal"><span class="pre">~/.s3ql/authinfo</span></ <li class="right" > <a href="backends.html" title="Storage Backends" >previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/html/mount.html b/doc/html/mount.html index f8de8a4..e0de7dd 100644 --- a/doc/html/mount.html +++ b/doc/html/mount.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>Mounting — S3QL 1.0.1 documentation</title> + <title>Mounting — S3QL 1.2 documentation</title> <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="_static/underscore.js"></script> <script type="text/javascript" src="_static/doctools.js"></script> <link rel="author" title="About these documents" href="about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="index.html" /> <link rel="next" title="Advanced S3QL Features" href="special.html" /> <link rel="prev" title="Managing Buckets" href="adm.html" /> </head> @@ -40,7 +40,7 @@ <li class="right" > <a href="adm.html" title="Managing Buckets" accesskey="P">previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -49,11 +49,11 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="adm.html">Managing Buckets</a></li> <li class="toctree-l1 current"><a class="current reference internal" href="">Mounting</a><ul> -<li class="toctree-l2"><a class="reference internal" href="#storing-encryption-passwords">Storing Encryption Passwords</a></li> <li class="toctree-l2"><a class="reference internal" href="#compression-algorithms">Compression Algorithms</a></li> <li class="toctree-l2"><a class="reference internal" href="#parallel-compression">Parallel Compression</a></li> <li class="toctree-l2"><a class="reference internal" href="#notes-about-caching">Notes about Caching</a></li> @@ -94,7 +94,7 @@ <div class="section" id="mounting"> <h1>Mounting<a class="headerlink" href="#mounting" title="Permalink to this headline">¶</a></h1> -<p>A S3QL file system is mounted with the <tt class=" docutils literal"><span class="pre">mount.s3ql</span></tt> command. It has +<p>A S3QL file system is mounted with the <tt class="docutils literal"><span class="pre">mount.s3ql</span></tt> command. It has the following syntax:</p> <div class="highlight-commandline"><div class="highlight"><pre><span class="l">mount.s3ql </span><span class="ge">[options]</span><span class="l"> </span><span class="nv"><storage url></span><span class="l"> </span><span class="nv"><mountpoint></span><span class="l"></span> </pre></div> @@ -111,15 +111,26 @@ mounted on one computer at a time.</p> <col class="option" /> <col class="description" /> <tbody valign="top"> +<tr><td class="option-group"> +<kbd><span class="option">--log <var><target></var></span></kbd></td> +<td>Write logging info into this file. File will be +rotated when it reaches 1 MB, and at most 5 old log +files will be kept. Specify <tt class="docutils literal"><span class="pre">none</span></tt> to disable +logging. Default: <tt class="docutils literal"><span class="pre">~/.s3ql/mount.log</span></tt></td></tr> +<tr><td class="option-group" colspan="2"> +<kbd><span class="option">--cachedir <var><path></var></span></kbd></td> +</tr> +<tr><td> </td><td>Store cached data in this directory (default: +<tt class="docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> <tr><td class="option-group" colspan="2"> -<kbd><span class="option">--homedir <var><path></var></span></kbd></td> +<kbd><span class="option">--authfile <var><path></var></span></kbd></td> </tr> -<tr><td> </td><td>Directory for log files, cache and authentication -info. (default: <tt class=" docutils literal"><span class="pre">~/.s3ql)</span></tt></td></tr> +<tr><td> </td><td>Read authentication credentials from this file +(default: <tt class="docutils literal"><span class="pre">~/.s3ql/authinfo2)</span></tt></td></tr> <tr><td class="option-group" colspan="2"> <kbd><span class="option">--debug <var><module></var></span></kbd></td> </tr> -<tr><td> </td><td>activate debugging output from <module>. Use <tt class=" docutils literal"><span class="pre">all</span></tt> to +<tr><td> </td><td>activate debugging output from <module>. Use <tt class="docutils literal"><span class="pre">all</span></tt> to get debug messages from all modules. This option can be specified multiple times.</td></tr> <tr><td class="option-group"> @@ -128,13 +139,6 @@ be specified multiple times.</td></tr> <tr><td class="option-group"> <kbd><span class="option">--version</span></kbd></td> <td>just print program version and exit</td></tr> -<tr><td class="option-group"> -<kbd><span class="option">--ssl</span></kbd></td> -<td>Use SSL when connecting to remote servers. This option -is not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext -even for unencrypted file systems.</td></tr> <tr><td class="option-group" colspan="2"> <kbd><span class="option">--cachesize <var><size></var></span></kbd></td> </tr> @@ -150,20 +154,20 @@ operation.</td></tr> Each cache entry requires one file descriptor, so if you increase this number you have to make sure that your process file descriptor limit (as set with -<tt class=" docutils literal"><span class="pre">ulimit</span> <span class="pre">-n</span></tt>) is high enough (at least the number of +<tt class="docutils literal"><span class="pre">ulimit</span> <span class="pre">-n</span></tt>) is high enough (at least the number of cache entries + 100).</td></tr> <tr><td class="option-group"> <kbd><span class="option">--allow-other</span></kbd></td> -<td>Normally, only the user who called <tt class=" docutils literal"><span class="pre">mount.s3ql</span></tt> can +<td>Normally, only the user who called <tt class="docutils literal"><span class="pre">mount.s3ql</span></tt> can access the mount point. This user then also has full access to it, independent of individual file -permissions. If the <tt class=" docutils literal"><span class="pre">--allow-other</span></tt> option is +permissions. If the <tt class="docutils literal"><span class="pre">--allow-other</span></tt> option is specified, other users can access the mount point as well and individual file permissions are taken into account for all users.</td></tr> <tr><td class="option-group"> <kbd><span class="option">--allow-root</span></kbd></td> -<td>Like <tt class=" docutils literal"><span class="pre">--allow-other</span></tt>, but restrict access to the +<td>Like <tt class="docutils literal"><span class="pre">--allow-other</span></tt>, but restrict access to the mounting user and the root user.</td></tr> <tr><td class="option-group"> <kbd><span class="option">--fg</span></kbd></td> @@ -184,45 +188,24 @@ this, then you don’t need it.</td></tr> <kbd><span class="option">--compress <var><name></var></span></kbd></td> </tr> <tr><td> </td><td>Compression algorithm to use when storing new data. -Allowed values: <tt class=" docutils literal"><span class="pre">lzma</span></tt>, <tt class=" docutils literal"><span class="pre">bzip2</span></tt>, <tt class=" docutils literal"><span class="pre">zlib</span></tt>, none. -(default: <tt class=" docutils literal"><span class="pre">lzma</span></tt>)</td></tr> +Allowed values: <tt class="docutils literal"><span class="pre">lzma</span></tt>, <tt class="docutils literal"><span class="pre">bzip2</span></tt>, <tt class="docutils literal"><span class="pre">zlib</span></tt>, none. +(default: <tt class="docutils literal"><span class="pre">lzma</span></tt>)</td></tr> <tr><td class="option-group" colspan="2"> <kbd><span class="option">--metadata-upload-interval <var><seconds></var></span></kbd></td> </tr> <tr><td> </td><td>Interval in seconds between complete metadata uploads. Set to 0 to disable. Default: 24h.</td></tr> -<tr><td class="option-group" colspan="2"> -<kbd><span class="option">--compression-threads <var><no></var></span></kbd></td> -</tr> -<tr><td> </td><td>Number of parallel compression and encryption threads -to use (default: 1).</td></tr> +<tr><td class="option-group"> +<kbd><span class="option">--threads <var><no></var></span></kbd></td> +<td>Number of parallel upload threads to use (default: +auto).</td></tr> +<tr><td class="option-group"> +<kbd><span class="option">--nfs</span></kbd></td> +<td>Support export of S3QL file systems over NFS (default: +False)</td></tr> </tbody> </table> </div></blockquote> -<div class="section" id="storing-encryption-passwords"> -<span id="bucket-pw"></span><h2>Storing Encryption Passwords<a class="headerlink" href="#storing-encryption-passwords" title="Permalink to this headline">¶</a></h2> -<p>If you are trying to mount an encrypted bucket, <tt class=" docutils literal"><span class="pre">mount.s3ql</span></tt> will first -try to read the password from the <tt class=" docutils literal"><span class="pre">.s3ql/authinfo</span></tt> file (the same file -that is used to read the backend authentication data) and prompt the -user to enter the password only if this fails.</p> -<p>The <tt class=" docutils literal"><span class="pre">authinfo</span></tt> entries to specify bucket passwords are of the form</p> -<div class="highlight-commandline"><div class="highlight"><pre><span class="l">storage-url </span><span class="nv"><storage-url></span><span class="l"> password </span><span class="nv"><password></span><span class="l"></span> -</pre></div> -</div> -<p>So to always use the password <tt class=" docutils literal"><span class="pre">topsecret</span></tt> when mounting <tt class=" docutils literal"><span class="pre">s3://joes_bucket</span></tt>, -the entry would be</p> -<div class="highlight-commandline"><div class="highlight"><pre><span class="l">storage-url s3://joes_bucket password topsecret</span> -</pre></div> -</div> -<div class="admonition note"> -<p class="first admonition-title">Note</p> -<p class="last">If you are using the local backend, the storage url will -always be converted to an absolute path. So if you are in the -<tt class=" docutils literal"><span class="pre">/home/john</span></tt> directory and try to mount <tt class=" docutils literal"><span class="pre">local://bucket</span></tt>, the matching -<tt class=" docutils literal"><span class="pre">authinfo</span></tt> entry has to have a storage url of -<tt class=" docutils literal"><span class="pre">local:///home/john/bucket</span></tt>.</p> -</div> -</div> <div class="section" id="compression-algorithms"> <h2>Compression Algorithms<a class="headerlink" href="#compression-algorithms" title="Permalink to this headline">¶</a></h2> <p>S3QL supports three compression algorithms, LZMA, Bzip2 and zlib (with @@ -236,22 +219,21 @@ compression ratios than zlib.</p> depends on your network connection speed: the compression algorithm should be fast enough to saturate your network connection.</p> <p>To find the optimal algorithm for your system, S3QL ships with a -program called <tt class=" docutils literal"><span class="pre">benchmark.py</span></tt> in the <tt class=" docutils literal"><span class="pre">contrib</span></tt> directory. You should +program called <tt class="docutils literal"><span class="pre">benchmark.py</span></tt> in the <tt class="docutils literal"><span class="pre">contrib</span></tt> directory. You should run this program on a file that has a size that is roughly equal to the block size of your file system and has similar contents. It will then determine the compression speeds for the different algorithms and the upload speeds for the specified backend and recommend the best algorithm that is fast enough to saturate your network connection.</p> <p>Obviously you should make sure that there is little other system load -when you run <tt class=" docutils literal"><span class="pre">benchmark.py</span></tt> (i.e., don’t compile software or encode +when you run <tt class="docutils literal"><span class="pre">benchmark.py</span></tt> (i.e., don’t compile software or encode videos at the same time).</p> </div> <div class="section" id="parallel-compression"> <h2>Parallel Compression<a class="headerlink" href="#parallel-compression" title="Permalink to this headline">¶</a></h2> <p>If you are running S3QL on a system with multiple cores, you might -want to set <tt class="docutils literal"><span class="pre">--compression-threads</span></tt> to a value bigger than one. This -will instruct S3QL to compress and encrypt several blocks at the same -time.</p> +want to set the <tt class="docutils literal"><span class="pre">--threads</span></tt> value larger than one. This will +instruct S3QL to compress and encrypt several blocks at the same time.</p> <p>If you want to do this in combination with using the LZMA compression algorithm, you should keep an eye on memory usage though. Every LZMA compression threads requires about 200 MB of RAM.</p> @@ -259,7 +241,7 @@ LZMA compression threads requires about 200 MB of RAM.</p> <p class="first admonition-title">Note</p> <p class="last">To determine the optimal compression algorithm for your network connection when using multiple threads, you can pass the -<tt class="docutils literal"><span class="pre">--compression-threads</span></tt> option to <tt class=" docutils literal"><span class="pre">contrib/benchmark.py</span></tt>.</p> +<tt class="docutils literal"><span class="pre">--compression-threads</span></tt> option to <tt class="docutils literal"><span class="pre">contrib/benchmark.py</span></tt>.</p> </div> </div> <div class="section" id="notes-about-caching"> @@ -269,17 +251,17 @@ access. The cache is block based, so it is possible that only parts of a file are in the cache.</p> <div class="section" id="maximum-number-of-cache-entries"> <h3>Maximum Number of Cache Entries<a class="headerlink" href="#maximum-number-of-cache-entries" title="Permalink to this headline">¶</a></h3> -<p>The maximum size of the cache can be configured with the <tt class=" docutils literal"><span class="pre">--cachesize</span></tt> +<p>The maximum size of the cache can be configured with the <tt class="docutils literal"><span class="pre">--cachesize</span></tt> option. In addition to that, the maximum number of objects in the -cache is limited by the <tt class=" docutils literal"><span class="pre">--max-cache-entries</span></tt> option, so it is +cache is limited by the <tt class="docutils literal"><span class="pre">--max-cache-entries</span></tt> option, so it is possible that the cache does not grow up to the maximum cache size because the maximum number of cache elements has been reached. The reason for this limit is that each cache entry requires one open file descriptor, and Linux distributions usually limit the total number of file descriptors per process to about a thousand.</p> -<p>If you specify a value for <tt class=" docutils literal"><span class="pre">--max-cache-entries</span></tt>, you should therefore +<p>If you specify a value for <tt class="docutils literal"><span class="pre">--max-cache-entries</span></tt>, you should therefore make sure to also configure your system to increase the maximum number -of open file handles. This can be done temporarily with the <tt class=" docutils literal"><span class="pre">umask</span> <span class="pre">-n</span></tt> +of open file handles. This can be done temporarily with the <tt class="docutils literal"><span class="pre">umask</span> <span class="pre">-n</span></tt> command. The method to permanently change this limit system-wide depends on your distribution.</p> </div> @@ -299,7 +281,7 @@ recently used blocks first.</p> system startup and shutdown, you should do so with one dedicated S3QL init script for each S3QL file system.</p> <p>If your system is using upstart, an appropriate job can be defined -as follows (and should be placed in <tt class=" docutils literal"><span class="pre">/etc/init/</span></tt>):</p> +as follows (and should be placed in <tt class="docutils literal"><span class="pre">/etc/init/</span></tt>):</p> <div class="highlight-commandline"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre> 1 2 3 @@ -362,16 +344,16 @@ as follows (and should be placed in <tt class=" docutils literal"><span class="p <div class="admonition note"> <p class="first admonition-title">Note</p> <p>In principle, it is also possible to automatically mount an S3QL -file system with an appropriate entry in <tt class=" docutils literal"><span class="pre">/etc/fstab</span></tt>. However, +file system with an appropriate entry in <tt class="docutils literal"><span class="pre">/etc/fstab</span></tt>. However, this is not recommended for several reasons:</p> <ul class="last simple"> -<li>file systems mounted in <tt class=" docutils literal"><span class="pre">/etc/fstab</span></tt> will be unmounted with the -<tt class=" docutils literal"><span class="pre">umount</span></tt> command, so your system will not wait until all data has +<li>file systems mounted in <tt class="docutils literal"><span class="pre">/etc/fstab</span></tt> will be unmounted with the +<tt class="docutils literal"><span class="pre">umount</span></tt> command, so your system will not wait until all data has been uploaded but shutdown (or restart) immediately (this is a FUSE limitation, see <a class="reference external" href="http://code.google.com/p/s3ql/issues/detail?id=159">issue 159</a>).</li> <li>There is no way to tell the system that mounting S3QL requires a Python interpreter to be available, so it may attempt to run -<tt class=" docutils literal"><span class="pre">mount.s3ql</span></tt> before it has mounted the volume containing the +<tt class="docutils literal"><span class="pre">mount.s3ql</span></tt> before it has mounted the volume containing the Python interpreter.</li> <li>There is no standard way to tell the system that internet connection has to be up before the S3QL file system can be @@ -396,7 +378,7 @@ mounted.</li> <li class="right" > <a href="adm.html" title="Managing Buckets" >previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/html/objects.inv b/doc/html/objects.inv Binary files differindex be510e2..fbb9b14 100644 --- a/doc/html/objects.inv +++ b/doc/html/objects.inv diff --git a/doc/html/resources.html b/doc/html/resources.html index 37e568d..a90f6db 100644 --- a/doc/html/resources.html +++ b/doc/html/resources.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>Further Resources / Getting Help — S3QL 1.0.1 documentation</title> + <title>Further Resources / Getting Help — S3QL 1.2 documentation</title> <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="_static/underscore.js"></script> <script type="text/javascript" src="_static/doctools.js"></script> <link rel="author" title="About these documents" href="about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="index.html" /> <link rel="prev" title="The expire_backups command" href="man/expire_backups.html" /> </head> <body> @@ -36,7 +36,7 @@ <li class="right" style="margin-right: 10px"> <a href="man/expire_backups.html" title="The expire_backups command" accesskey="P">previous</a></li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -45,6 +45,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="adm.html">Managing Buckets</a></li> @@ -107,7 +108,7 @@ can subscribe by sending a mail to <li class="right" style="margin-right: 10px"> <a href="man/expire_backups.html" title="The expire_backups command" >previous</a></li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/html/search.html b/doc/html/search.html index 30b0686..b1f5b87 100644 --- a/doc/html/search.html +++ b/doc/html/search.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>Search — S3QL 1.0.1 documentation</title> + <title>Search — S3QL 1.2 documentation</title> <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -27,7 +27,7 @@ <script type="text/javascript" src="_static/doctools.js"></script> <script type="text/javascript" src="_static/searchtools.js"></script> <link rel="author" title="About these documents" href="about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="index.html" /> <script type="text/javascript"> jQuery(function() { Search.loadIndex("searchindex.js"); }); </script> @@ -38,7 +38,7 @@ <div class="related"> <h3>Navigation</h3> <ul> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -47,6 +47,7 @@ <ul> <li class="toctree-l1"><a class="reference internal" href="about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="adm.html">Managing Buckets</a></li> @@ -102,7 +103,7 @@ <div class="related"> <h3>Navigation</h3> <ul> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/html/searchindex.js b/doc/html/searchindex.js index d4a910d..e9b5e1a 100644 --- a/doc/html/searchindex.js +++ b/doc/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({objects:{},terms:{all:[7,16,0,19,21,10,22,20,23,18,24,1,11,26,6,2,4,13,14,25,12],code:[7,16,0,21,10,22,9,24,20,2,3,25,5,19],partial:11,global:11,mnt:[4,13,26],month:[4,24],stumbl:12,per:[26,11],follow:[0,1,2,3,4,5,6,7,9,10,11,12,13,14,15,16,18,19,21,22,23,24,25,26],disk:[13,20],intrins:13,locat:[7,16,0,10,22,9,1,11,2,25,5,19,6],whose:11,decid:[13,11],depend:[7,8,21,22,24,2,4,13,19,26],system:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,16,17,18,19,20,21,22,23,24,25,26],ulimit:[26,19],readabl:20,specif:[0,21,12,25,14,6],send:15,rel:[11,3],init:[26,12],program:[0,1,2,3,4,5,6,7,8,9,10,11,12,14,16,18,19,21,22,23,24,25,26],present:11,under:11,aris:11,sha256:20,neglig:11,worth:12,introduc:[16,6],sourc:[16,20,21,3,4,6],everi:[16,0,26,24,1,11,4,13,6],risk:11,far:13,faq:15,account:[19,11,26],util:[13,18],volum:26,failur:11,veri:[20,11,12],appar:11,tri:[14,11,12],administr:[0,6],level:[25,11,6],did:20,list:[20,21,24,25,11,6,4,14,15],upload:[16,20,10,26,23,11,12,25,13,19,6],"try":[4,24,26,11,21],stderr:26,small:[13,12],refer:[11,17],servic:20,dir:[16,26],pleas:[20,11,15],upper:11,smaller:[13,20],slower:[16,26,6],ifac:26,ten:11,compens:11,sync:3,histor:11,consequ:11,hors:[0,6],design:[0,11,20,6],pass:[26,3],download:[14,21,2],further:[8,14,15],port:[7,19,11,22,2],rath:26,even:[7,0,19,10,22,23,18,1,11,2,14,26,6],what:[4,14,11,25,24],bzip2:[20,19,26],sub:12,descriptor:[26,19],preserv:[4,20],section:11,abl:[0,10,23,11,6,15],invok:[25,14],asia:11,anywai:[7,14,22,18,1,2,19,26],access:[20,19,26,11,14,12],delet:[8,0,9,24,11,6,2,4,14,12],version:[7,16,0,19,21,10,22,9,23,18,24,1,11,20,2,3,25,5,14,26,12],suspect:18,consecut:[7,19,11,22,2],"new":[7,16,0,19,22,26,24,1,11,6,4,14,25,12],net:26,significantli:[4,11,21],ever:11,"public":11,contrast:[0,6],metadata:[8,19,26,2,25,13,14,6],elimin:11,full:[7,16,0,10,22,9,25,20,2,4,5,19,26,6],pacif:11,gener:[0,21,24,12,4,14,6],never:[7,14,22,18,1,2,19,26],privat:11,here:[25,17],satisfi:21,slow:[11,12],modif:12,address:12,path:[7,19,21,22,18,1,11,2,14,26],becom:[4,24,13,11],modifi:[16,20,6],sinc:[4,24,26,11,12],valu:[1,22,19,26,12],wait:[10,26,23,11,13,12],dry:24,convert:26,joes_bucket:26,checksum:20,current:[4,14,11,24,12],host:[7,19,11,22,2],loos:11,amount:[25,13,11,20],throughput:4,shrink:20,action:[25,14,6,2],chang:[16,8,0,26,20,11,6,2,25,14,12],magnitud:[16,6],chanc:[11,12],control:6,configur:[8,26,6],retriev:[13,19,26,3],appli:11,modul:[7,14,21,22,18,1,2,25,19,26],apw:21,filenam:13,unix:[13,20,12],leav:12,visibl:11,instal:[7,8,0,19,17,21,16,10,22,9,25,2,4,5,14],total:[26,11,6,12],establish:11,kei:[20,11],from:[7,16,0,19,21,22,20,18,24,1,11,2,4,13,14,26,6],describ:[],would:[16,26,24,11,4,6],apswvers:21,upgrad:[8,14,21,2],few:[4,11,21],concret:11,call:[16,0,26,24,11,4,5,13,19,6],usr:[7,16,0,10,22,9,25,2,4,5,19],recommend:[4,26,11,12],taken:[16,20,26,11,19,6],suppos:11,type:11,until:[10,26,6,12,23],more:[20,21,11,12,25,6],sort:[4,24],desir:25,st_nlink:12,comparison:[],peopl:11,hundr:11,relat:11,benchmark:[4,8,26],deflat:20,notic:[13,11],enhanc:4,warn:[25,14,9,11,6],sqlitelibvers:21,depth:[],sens:13,known:[8,20,12],actual:[14,13,11,21,12],destin:[16,6,3],unpack:21,cach:[7,8,20,19,16,22,18,1,12,2,25,14,26,6],must:[7,16,19,22,11,2,14,6],worst:[0,6],none:[19,11,26],word:11,restor:[16,8,13,14,6],setup:21,work:[20,21,23,24,11,4,13,12],conceptu:20,remain:[26,11],wors:11,obvious:26,can:[0,1,2,4,5,6,7,9,11,12,13,14,15,16,17,18,19,20,21,22,23,24,26],root:[16,10,26,23,5,19,6],fetch:[16,6],overrid:11,prompt:[7,26,18],tar:13,give:[4,24,14],process:[10,26,23,11,3,4,13,19,12],lock:[0,6],sudo:21,share:[7,16,0,10,22,9,25,2,4,5,19],accept:[7,16,0,19,10,22,9,23,18,24,1,2,3,25,5,14,26],high:[20,19,26,3],critic:20,minimum:[13,20],want:[21,10,26,23,24,1,11,4,13,15],onlin:[0,20,6],acl:20,occur:[7,16,0,10,22,9,24,11,2,3,25,5,19],alwai:[16,26,24,11,12,4,6],algorithm:[4,8,20,19,26],end:[26,14],manipul:20,thing:[14,2],rather:[24,11,12,4,13,6],anoth:[11,12],ordinari:[16,10,23,6],write:[7,8,20,21,16,22,11,12,2,13,19,26,6],how:[4,24,13,11],env:26,instead:[16,10,23,11,12,6],simpl:[4,24,6],confid:[],product:[],resourc:[8,14,15],haven:11,max:[26,19],earlier:[11,21,12],pyliblzma:21,badli:[14,2],wrong:[14,12,2],compat:12,endang:20,mai:[16,0,19,21,26,15,20,23,25,11,6,2,4,13,14,12],multipl:[7,14,22,18,1,2,19,26],redund:[4,24,20,11],autom:[4,20],data:[7,16,20,19,10,22,26,23,18,1,11,6,2,4,13,14,25,12],grow:[20,26],physic:[4,16,0,14,6],man:17,handl:[16,20,26,24,11,12,4,13,6],"short":[0,11,6],attempt:[7,26,11,12],practic:[13,11],third:13,read:[7,16,20,22,11,12,2,13,19,26,6],minim:20,favorit:11,apsw:21,element:26,issu:[16,8,20,21,12,11,6,5,26,15],inform:[26,24,11,25,19,6],maintain:26,combin:[26,11],allow:[16,0,19,22,9,24,1,11,26,3,4,5,13,14,25,6],enter:[1,26],exclus:13,pycryptopp:21,order:[16,6],origin:[],help:[8,21,15,25,12,2,4,14,6],over:[16,20,13,6,12],move:[14,20,11],soon:11,topsecret:26,report:[20,11,21,15],affect:[16,26,11,6,12],effici:12,still:[16,20,10,26,23,11,6],dynam:20,fix:[0,24,11,12,4,6],inadvert:[16,6],better:[26,11,21,3],window:11,requir:[7,20,21,26,18,24,25,4,19,12],persist:11,mail:[20,14,21,15],main:[9,6],might:[16,12,18,6,5,26,15],split:20,them:[16,10,23,11,14,6],good:[0,24,12,4,13,6],synopsi:[7,16,0,10,22,9,24,2,3,25,5,19],thei:[16,0,17,20,24,11,4,13,6],python:[26,21],promin:12,safe:[16,13,11,6],dai:[4,24],initi:11,dat:[4,24],therebi:11,interrupt:14,now:[0,21,24,11,4,6],bigger:26,choic:21,term:[4,24],document:[7,16,0,10,22,9,2,25,5,19,6],somewher:[7,16,0,10,22,9,2,25,5,19,6],name:[7,16,22,24,1,11,2,4,13,19,26,6],hardlink:[16,20,6],joesdomain:11,authent:[7,19,22,18,1,11,2,14,26],achiev:[16,20,26,6],mode:[26,19],each:[4,24,26,19],debug:[7,16,0,19,10,22,9,23,18,24,1,2,3,25,5,14,26],found:[13,11],beneath:6,updat:[4,20,11,12],side:11,mean:[20,11,12],compil:26,s3ql:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26],mkfifo:26,chunk:12,hard:[13,20],idea:[4,24,0,6,12],procedur:[16,6],realli:[7,16,0,19,10,22,9,23,18,24,1,11,2,3,25,5,13,14,26,12],ensur:[4,24,13,20],meta:[25,13,6],expect:[26,11],year:[4,24,20,11],s3_copi:[4,8],happen:[0,11,6],special:[16,6],out:[20,11,12],lzma:[20,19,21,26],ftp:11,shown:11,network:[16,13,26,20,6],space:[16,20,6,12],miss:[24,11],newli:12,content:[16,20,26,0,11,2,6],suitabl:[20,21],rational:0,internet:[20,26],print:[7,16,0,19,21,10,22,9,23,18,24,1,2,3,25,5,14,26],eth0:26,bla:26,correct:[4,24],statist:[5,6,8],foreground:[26,19],shut:[4,10,23,12],after:[16,0,20,11,13,14,6],insid:16,advanc:[8,6],migrat:4,documents_janurai:[16,6],million:[11,12],situat:[13,11,12],differ:[16,20,21,26,25,11,2,4,13,14,6],free:[20,11],standard:[20,22,24,1,11,4,26],inod:12,reason:[9,26,23,18,24,11,12,4,13,6],base:26,theori:[11,12],ask:[1,11,21,14,12],org:26,"byte":[],likelihood:11,afterward:13,bash:13,care:[16,20,11,6,12],wai:[20,26,0,11,12,13,6],thread:[26,19],befor:[16,20,10,26,23,11,12,6],guarante:[4,8,11,24],could:[16,6],refus:[4,24,11],keep:[0,26,24,11,4,6],recov:[0,6],turn:[4,24,26,12],fuse:[26,21,12],place:[7,26,11,2,19,22],perman:[8,0,26,11,4,13,6],principl:26,confus:12,neglect:11,first:[20,21,26,11,13,14,12],oper:[7,16,0,19,10,22,9,24,11,20,2,3,25,5,14,26],softwar:26,rang:[4,24],malici:[0,6],carri:20,onc:[0,10,26,20,23,12,13,19,6],independ:[19,11,26,20],number:[20,21,26,24,11,12,3,4,13,19,6],capac:20,restrict:[7,22,11,2,25,19,26],date:[4,24],instruct:[26,21],alreadi:[7,16,19,21,22,18,1,11,12,2,14,26,6],done:[26,11],least:[21,26,24,11,4,19],llfuse:21,indistinguish:20,open:[10,26,23,12],size:[20,22,1,12,25,13,19,26,6],given:12,"long":[4,24,13,11,12],convent:[7,16,0,10,22,9,2,25,5,19,12],script:[4,13,26,12],unknown:[20,11],interact:2,s3qllock:[8,0,17,9,4,13,6],sometim:14,wrapper:[4,3],parallel:[8,26,3,4,13,19],checkpoint:11,attack:[0,6],necessarili:11,demonstr:4,s3qlrm:[8,0,17,9,24,4,6],termin:[10,11,23,12],john:26,"final":6,store:[7,8,20,22,24,1,11,2,4,13,19,26,12],includ:[21,20,11,17,12],luckili:12,option:[0,1,2,3,4,5,6,7,9,10,12,13,14,16,18,19,21,22,23,24,25,26],especi:[13,20],shelf:[0,6],flushcach:[25,6],copi:[16,8,20,11,12,3,4,13,14,6],specifi:[7,19,22,26,18,24,1,11,2,4,14,25,12],part:[4,24,20,26,3],mostli:11,consult:[],exactli:[4,16,11,24,6],than:[16,9,26,24,11,12,4,13,6],wide:[26,21],target:[16,6],whenev:[13,21,26],prevent:[0,6],remov:[16,8,0,21,26,9,24,6,4,5,12],older:14,tree:[16,8,0,9,3,4,20,6],second:[26,24,11,4,13,19],structur:[20,14],matter:[25,13,14],optimum:13,num:[26,19],friend:13,video:26,minut:[4,24,11],pre:26,unencrypt:[7,19,22,18,1,11,2,14,26],sai:[16,6],comput:[7,20,22,24,11,2,4,19,26],entail:11,plaintext:[7,14,22,18,1,2,19,26],explicit:13,ram:26,mind:11,argument:[25,14,2],"13th":20,packag:[4,21],expir:26,increment:[],close:[10,23],need:[16,19,21,26,24,11,4,14,6],seem:[13,11],exclud:[0,6],paramiko:21,caus:11,equival:[4,24],irrepar:14,destroi:[0,6],moreov:6,blindli:12,atim:12,accuraci:[4,24],note:[7,8,20,19,21,16,10,22,23,24,11,6,2,4,14,26,12],also:[7,16,0,17,21,10,22,9,23,24,11,20,2,3,25,5,13,19,26,6],builtin:21,without:[7,16,0,20,18,11,12,13,14,6],take:[16,20,11,12,2,25,13,14,6],which:[16,9,21,20,24,11,12,4,6],discrep:[],transmit:[7,14,22,18,1,2,19,26],environ:21,uplink:4,singl:[13,19,26],mktemp:26,begin:11,sure:[19,21,26,11,13,14,12],unless:[1,0,6],distribut:[4,26,21],plenti:[],normal:[26,19],buffer:12,previou:[4,14],compress:[8,20,26,4,19,6],most:[16,24,11,12,4,14,6],beta:20,detect:[20,11],rigor:[4,24],plan:11,homepag:[7,16,0,10,22,9,2,25,5,19],"class":[7,16,0,10,22,9,24,25,2,3,4,5,19],relatim:12,simplic:20,renam:20,inconveni:20,url:[7,19,22,18,1,11,2,14,26],doc:[7,16,0,10,22,9,25,2,4,5,19],clear:[16,6],later:[16,0,21,11,12,6],cover:13,drive:[0,6],destruct:[0,6],doe:[20,11,26,23,12],declar:20,snapshot:[16,8,20,25,6],runtim:[8,6],determin:[26,11],occasion:11,left:12,hmac:20,gracefulli:[4,24],myawssecretaccesskei:11,show:24,carefulli:11,random:11,syntax:[26,18,1,25,14,6],permiss:[4,20,19,26],bucketnam:[7,19,11,22,2],newest:[13,2],find:[0,11,26,6,12],redirect:26,absolut:[7,22,24,11,2,4,19,26],onli:[16,0,19,21,10,26,20,23,24,25,11,6,4,5,13,14,12],explicitli:12,ratio:26,just:[7,16,0,19,10,22,9,23,18,24,1,2,3,25,5,13,14,26,6],transact:20,fstab:[26,12],activ:[7,16,0,19,10,22,9,23,18,24,1,2,3,25,5,14,26,6],enough:[13,19,26,12],start:[16,21,26,3,4,13,6],peculiar:11,authinfo:[7,8,22,1,11,2,19,26],latenc:[13,20,3],tape:[0,6],factor:11,folder:[16,21,6],local:[7,8,0,21,16,10,22,9,11,20,2,25,5,13,19,26,6],defin:[4,24,26,11],contribut:[4,8],variou:[25,14,2],get:[7,8,0,19,21,22,26,18,24,1,11,15,2,4,14,25,6],googlegroup:15,nasti:[0,6],stop:26,secondari:[],regularli:[16,6],ssl:[7,14,22,18,1,2,19,26],s3rr:11,cannot:[4,24],ssh:11,increas:[13,19,26],reconstruct:[4,24,11],restart:[26,12],myawsaccesskeyid:11,reveal:20,enabl:[7,14,22,18,1,2,19,26],dramat:13,"19283712_yourname_s3ql":11,method:[26,21],provid:[20,11,21,26,12],bad:13,common:[16,0,6],contain:[21,26,24,11,2,4,14],userspac:[],nowher:[],where:[25,14,11,12,2],caches:[25,26,19,6],wiki:[21,15],kernel:21,set:[4,13,19,26],noleaf:12,proce:21,startup:26,displai:12,see:[7,16,0,19,10,22,9,24,1,11,26,6,2,3,4,5,13,14,25,12],temporarili:26,s3qlcp:[4,8,16,17,6],corrupt:11,disadvantag:[16,6],"__version__":21,becaus:[7,16,20,19,22,18,1,11,2,4,14,26,6],whatsoev:[0,20,6],best:[13,21,26],concern:11,infinit:20,awar:11,statu:[7,8,0,16,10,22,9,24,11,20,2,3,25,5,19,12],said:11,extend:20,correctli:[12,18],hopefulli:12,databas:20,boundari:[4,24],label:[1,22],favor:20,state:[4,24,20,12],between:[4,24,19,11,26],"import":[4,11,21],neither:[0,6],param:14,across:20,attribut:20,amazon:[7,8,20,22,11,2,4,19,12],august:11,manpag:[8,17],weak:11,southeast:[1,22],job:[4,26],entir:[20,0,9,6],joe:[4,11],expire_backup:[4,8,17,24],solv:13,come:[4,13,11,20],local0:26,addit:[16,9,26,20,11,2,25,14,6],both:[16,20,11,21,6],protect:[0,20,6],accident:[0,6,12],irregular:[4,24],extens:[20,11],someth:[16,6],howev:[16,0,21,26,20,24,11,12,4,13,6],alon:11,lazi:[10,23],against:[0,20,6],etc:[16,26,6,12],inconsist:12,exec:26,freeli:26,login:11,com:[7,16,0,21,10,22,9,24,11,2,3,25,5,19,15],pcp:[4,8,13,17,3],load:26,simpli:[0,6],figur:11,inspir:[16,6],period:[0,11,6],insist:13,written:[19,11,26,20],littl:[26,21],shutdown:26,linux:[26,21],averag:11,typic:[16,6],guid:[8,17],assum:26,duplic:[4,16,20,11,6],quit:21,worthless:[0,6],strong:[7,16,0,10,22,9,24,11,2,3,25,5,19],west:[1,22,11],devic:26,three:[7,21,22,11,2,19,26],been:[16,0,10,26,20,23,18,24,11,12,4,13,14,6],mark:[7,18],compon:20,secret:11,much:3,interpret:26,interest:6,subscrib:15,monthli:[16,6],immedi:[9,10,26,23,11,6],strategi:[4,24],infect:[0,6],upstart:[4,26,19],great:[0,6],ani:[7,16,0,21,22,20,24,1,11,15,4,6],zero:13,understand:[26,19],els:14,s3qlstat:[5,17,6,12,8],those:11,"case":[16,0,21,20,24,11,4,13,14,6],replic:[4,16,20,6],trojan:[0,6],ident:[16,20,11,6],look:13,gnu:12,solid:20,plain:[1,22],mount:[16,8,0,19,17,10,26,23,1,11,6,2,4,5,13,14,25,12],zlib:[26,19],batch:[7,26,18],trick:[8,13],documents_februari:[16,6],weren:15,abov:[4,24,11],error:[7,8,0,16,10,22,9,18,24,11,20,2,3,25,5,19,12],invoc:[4,24],ahax:3,region:[4,11],jibbadup:11,bin:[13,21],argpars:21,have:[7,16,0,19,21,10,22,15,20,23,24,1,11,6,2,3,4,13,14,26,12],advantag:[4,24,0,6],stdout:26,almost:13,therefor:[4,26,11,12],remount:11,worri:[11,12],quiet:[7,16,0,19,10,22,9,23,18,24,1,2,3,25,5,14,26],exit:[7,16,0,19,10,22,9,23,18,24,1,2,3,25,5,14,26],conf:[4,8],incom:11,revis:[14,2],dedic:[26,12],sever:[20,21,26,11,3,4,13,19],tamper:[4,24],unmount:[8,10,26,23,18,11,4,13,12],develop:[8,20],author:[26,12],perform:[8,20,21,26,25,11,2,3,4,13,14,12],make:[16,0,19,21,26,20,24,11,12,4,13,14,6],same:[7,16,19,22,24,11,2,4,13,14,26,6],"while":[0,21,26,20,23,11,25,6],dest:16,instanc:13,unexpectedli:12,nikolau:26,pai:11,eventu:11,infer:[4,24],complet:[16,21,26,24,25,11,4,19,6],week:[4,24],archiv:[7,20,22,11,2,19],lie:16,optim:[4,26],keyr:11,confidenti:11,upon:12,effect:[16,20,11,6],solut:4,remot:[7,20,19,22,18,1,11,2,14,26],rais:[26,19],temporari:11,user:[7,8,0,17,21,16,10,26,20,23,18,11,5,19,6],mani:[4,24,20,11],extern:[0,6],encrypt:[7,8,20,19,22,18,1,11,2,14,26],tradition:[0,6],recent:[4,24,26,14,12],appropri:26,eleg:20,nevertheless:[4,11],entri:[19,11,26,12],irrelev:12,well:[0,19,26,9,24,11,4,20,14,6],object:[19,11,26],exampl:[20,24,11,12,4,13,6],command:[0,1,2,3,4,5,6,7,8,9,10,11,12,14,16,17,18,19,21,22,23,24,25,26],thi:[0,1,2,3,4,5,6,7,11,12,13,14,15,16,18,19,20,21,22,23,24,25,26],filesystem:[1,22,19,26,18],gzip:20,fail:[26,11,21,12],spend:13,usual:[16,26,11,6],compromis:13,identifi:11,execut:[4,14,21,6],less:[11,12],conform:[7,19,11,22,2],when:[7,0,19,22,20,18,24,1,11,6,2,3,4,13,14,26,12],obtain:11,rest:13,detail:[4,13,11],bandwidth:[4,20],touch:[16,6],passphras:[7,8,14,22,2,19],roughli:26,speed:[4,26],yet:[16,11,6,12],damag:[4,14,11,24],viru:[0,6],detach:[10,23],homedir:[7,14,22,18,1,2,19,26],easi:13,hint:[25,14,2],point:[10,19,11,26,20],had:[4,16,24,6],s3qladm:[8,14,17,2],theoret:[13,11],add:[0,11,6],other:[16,20,21,26,11,6,5,19,15],nor:[0,6],versa:[16,6],runlevel:26,logger:26,subsequ:[4,24,11],match:26,futur:[16,5,6,12],earli:12,applic:12,transpar:20,webpag:11,big:[13,11,12],ctime:12,know:[16,24,11,12,4,6],background:[10,23],amp:13,bit:[20,11],password:[1,26,11,8],recurs:[4,8,9,6],you:[0,1,2,4,6,7,9,10,11,12,13,14,15,16,18,19,20,21,22,23,24,26],like:[16,0,26,20,11,12,3,19,6],loss:11,daemon:[26,19],lost:11,success:11,arbitrari:20,should:[7,16,0,19,21,10,22,9,23,18,11,6,2,25,5,13,14,26,12],anyth:[],manual:[10,21,23,15],resolv:[11,15],noth:[0,6],necessari:14,either:[11,12,2,25,14,6],furthermor:20,output:[7,16,0,19,10,22,9,23,18,24,1,2,3,25,5,14,26],page:17,two:[16,20,14,21,6],yyyi:[4,24],imagin:[4,24,0,6],right:13,old:[0,21,24,11,4,6],often:[13,11],deal:11,interv:[4,24,13,19,26],creation:[1,11,8],some:[7,16,0,21,10,22,9,23,18,24,25,11,20,2,3,4,5,19,12],umount:[8,17,10,26,23,12],self:21,strongest:11,"export":12,flush:[16,26,6],home:[4,13,26],server:[7,20,19,22,18,1,2,14,26,12],librari:[21,12],"24h":[26,19],basic:11,rsync:[16,0,12,3,4,13,6],confirm:[9,6],"function":14,avoid:[4,24],though:[26,11],definit:[4,24],februari:[16,6],protocol:11,backward:[],usernam:11,equal:26,larg:[16,13,6,12],slash:[7,19,11,22,2],cif:26,backend:[7,8,21,16,10,22,23,11,6,2,25,13,19,26,12],blocksiz:[1,22,19,26],machin:11,core:[4,26],plu:11,who:[10,26,19,23],run:[0,19,21,26,18,24,25,11,6,2,4,14,12],power:4,reach:26,view:17,usag:[4,16,26,24,6],symlink:20,speak:26,step:4,unreason:12,although:[20,9,6,12],januari:[16,6],immut:[8,0,9,4,20,13,6],impos:11,stage:[],continu:23,src:[16,6],about:[8,20,26,11,12,5,13,6],rare:11,memori:26,http:[7,16,0,21,10,22,9,24,11,2,3,25,5,19],storebackup:[16,6],unfortun:[13,11,12],her:[0,6],messag:[7,14,22,18,1,2,19,26],commit:[11,23,12],backup:[16,8,0,26,20,24,11,2,4,13,14,6],disabl:[26,19],block:[20,10,22,23,1,11,12,25,26,6],repair:7,client:12,real:11,encount:[15,12],within:6,encod:26,automat:[4,8,26,11,14],due:[11,6,12],down:[4,10,23,12],ahv:13,contrib:[4,13,26],insuffici:12,storag:[7,8,0,19,16,10,22,20,23,18,1,11,6,2,4,13,14,26,12],your:[7,16,0,17,21,10,22,9,24,25,11,2,4,5,13,19,26,6],durabl:11,manag:[8,14],mkf:[7,1,22,17,8],fast:[8,20,11,26,6],fusermount:[10,23,12],prepar:25,suffici:11,transfer:[4,13,11,20],support:[7,16,20,22,11,12,2,19,26,6],question:15,s3_backup:[4,8],overwrit:[1,22,11],avail:[0,26,24,12,4,14,6],intellig:[4,24,20],reli:[11,12],trigger:[11,6],low:[20,11],lot:[16,13,6],"var":[7,19,11,22,2],succeed:[7,16,0,10,22,9,24,2,3,25,5,19],individu:[26,19,3],lowest:11,properli:4,tracker:[20,21,15],form:[7,22,24,11,2,4,19,26],offer:[16,11,6],forc:[7,1,22,18],regard:[16,6],back:[0,6],sigstop:[26,19],satur:26,measur:4,newer:[11,21],don:[26,19,12],line:[25,20,11],bug:[20,21,15,12],longer:[16,0,20,23,24,11,12,4,14,6],info:[7,14,22,18,1,2,25,19,26],made:[16,20,0,11,13,6],input:[7,18],consist:[8,11],possibl:[20,21,26,0,11,12,14,6],"default":[7,19,22,18,24,1,11,2,3,4,14,26],bucket:[7,8,19,22,1,11,2,4,14,26,12],maximum:[1,22,19,26],tell:[4,24,26,11],asynchron:20,below:6,limit:[16,26,11,12,5,19,6],unnot:11,problem:[7,16,0,15,20,24,11,12,4,13,6],similar:[26,11],reliabl:[8,11],connect:[7,20,19,22,18,1,11,2,13,14,26],featur:[16,8,0,20,11,6],creat:[16,0,19,22,9,24,1,11,6,4,20,14,26,12],certain:11,dure:[14,19,11,26],day_hour:[4,24],stabl:20,s3qlctrl:[25,8,13,17,6],strongli:[4,24],workaround:13,exist:[16,0,22,9,24,1,11,4,6],file:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,16,18,19,20,22,23,24,25,26],mtime:12,request:11,ship:[24,26,21,3],improv:[8,13],check:[7,8,21,26,18,11,14,12],probabl:[20,24,4,13,14,6],otherwis:[26,19],again:[0,11,6,12,23],readi:[],rsync_arg:13,umask:26,googl:[7,16,0,21,10,22,9,24,2,3,25,5,19],tip:[8,13],exception:11,field:11,valid:6,creep:20,rememb:11,test:[20,11,21],presum:11,thousand:[26,11,12],rid:[0,6],particular:11,variabl:21,intend:[],clean:[7,18],fulli:12,mountpoint:[26,23,25,5,19,6],"return":[7,16,0,10,22,9,23,24,11,2,3,25,5,19,12],fsck:[7,8,17,26,18,11,14],briefli:[],releas:20,track:[4,24],log:[7,19,22,18,1,11,2,25,14,26,6],consid:[11,15],sql:23,log_fifo:26,dd_hh:[4,24],pool:11,stai:[19,11,26],reduc:[4,11],infrequ:11,faster:[9,6],pycrypto:21,vice:[16,6],directori:[7,16,0,19,21,10,22,9,18,24,1,11,6,2,3,4,20,13,14,26,12],cycl:[4,24],descript:[7,16,0,10,22,9,24,2,3,25,5,13,19,26],save:[24,13,11,20],rule:13,sftp:[7,8,20,21,22,11,2,19,12],itself:[11,21,12],ignor:11,time:[7,0,19,22,20,18,24,1,11,6,2,4,13,14,26,12],profil:[26,19],daili:[]},objtypes:{},titles:["The <strong class=\"program\">s3qllock</strong> command","File System Creation","The <strong class=\"program\">s3qladm</strong> command","The <strong class=\"program\">pcp</strong> command","Contributed Programs","The <strong class=\"program\">s3qlstat</strong> command","Advanced S3QL Features","The <strong class=\"program\">fsck.s3ql</strong> command","S3QL User’s Guide","The <strong class=\"program\">s3qlrm</strong> command","The <strong class=\"program\">umount.s3ql</strong> command","Storage Backends","Known Issues","Tips & Tricks","Managing Buckets","Further Resources / Getting Help","The <strong class=\"program\">s3qlcp</strong> command","Manpages","Checking for Errors","The <strong class=\"program\">mount.s3ql</strong> command","About S3QL","Installation","The <strong class=\"program\">mkfs.s3ql</strong> command","Unmounting","The <strong class=\"program\">expire_backups</strong> command","The <strong class=\"program\">s3qlctrl</strong> command","Mounting"],objnames:{},filenames:["man/lock","mkfs","man/adm","man/pcp","contrib","man/stat","special","man/fsck","index","man/rm","man/umount","backends","issues","tips","adm","resources","man/cp","man/index","fsck","man/mount","about","installation","man/mkfs","umount","man/expire_backups","man/ctrl","mount"]})
\ No newline at end of file +Search.setIndex({objects:{},terms:{suffici:1,all:[0,1,2,3,5,7,8,11,12,13,14,15,17,19,20,21,22,23,24,25,26,27],code:[8,17,0,22,11,23,10,25,21,3,4,26,6,20],partial:12,global:[],mnt:[5,14,27],month:[5,25],prefix:[12,1],stumbl:13,notquitesecret:1,follow:[0,1,2,3,4,5,6,7,8,10,11,12,13,14,15,16,17,19,20,22,23,24,25,26,27],disk:[14,21],whose:12,decid:[14,1],middl:13,depend:[8,9,22,23,1,25,3,5,14,20,27],million:13,ulimit:[27,20],readabl:21,send:16,rel:[12,4],init:[27,13],program:[0,2,3,4,5,6,7,8,9,10,11,13,15,17,19,20,22,23,24,25,26,27],those:12,under:[12,22],aris:12,sha256:21,neglig:[],worth:13,introduc:[17,7],sourc:[17,21,22,4,5,7],everi:[17,0,27,1,25,12,5,14,7],risk:[],mkf:[8,2,23,18,9],far:14,faq:16,account:[20,12,27],util:[14,19],pycryptopp:22,failur:1,veri:[21,12,13],affect:[17,27,7,13,1],tri:[15,13],administr:[0,7],level:[26,7],did:21,who:[11,27,20,24],list:[21,22,25,26,12,7,5,15,16],upload:[21,11,27,24,26,12,13,5,14,20,7],"try":[5,25,12,22,1],larg:[17,14,7,13],stderr:27,small:[14,13],blocksiz:[2,23,20,27],mount:[17,9,0,20,18,11,27,10,24,1,26,12,7,3,5,6,14,15,13],dir:[17,27],pleas:[21,16],upper:12,smaller:[14,21,13],slower:[17,27,7],ifac:27,ten:12,whitespac:[],compens:1,sync:4,sign:12,consequ:1,hors:[0,7],design:[0,21,7,1],pass:[27,4],download:[15,22,3],further:[9,15,16],correspond:12,port:12,rath:27,even:[8,0,11,24,1,12,19,7],what:[5,25,15,26,1],bzip2:[21,20,27],sub:13,entir:[21,0,10,7],descriptor:[27,20],section:[12,1],abl:[0,11,24,1,12,7,16],weren:16,asia:[],find:[0,12,27,7,13],access:[21,20,27,1,12,15,13],delet:[9,0,10,25,12,7,3,5,15,13],version:[8,17,0,20,22,11,23,10,24,19,25,2,12,21,3,4,26,6,15,27,13],suspect:19,consecut:12,"new":[8,0,20,23,1,25,26,12,7,5,15,27,13],net:27,ever:1,"public":[],contrast:[0,7],metadata:[9,20,27,3,26,14,15,7],elimin:12,full:[8,17,0,11,23,10,26,21,3,5,6,20,27,7],themselv:1,absolut:[5,25,12],pacif:[],gener:[9,0,22,1,25,12,7,5,15,13],never:[],privat:[],here:[26,12,18],satisfi:22,explicitli:13,modif:13,address:13,path:[8,20,22,23,19,2,12,3,15,27],becom:[5,25,14,12,1],modifi:[17,21,7],sinc:[27,1,25,12,5,13],valu:[27,20,13,1],wait:[11,14,27,24,13],dry:25,convert:12,joes_bucket:[],checksum:21,larger:27,step:5,amount:[26,14,12,21],throughput:[],action:[26,15,7,3],implement:1,magnitud:[17,7],chanc:13,control:7,fstab:[27,13],appli:[14,12,1],modul:[8,20,1,23,19,2,3,26,22,15,27],apw:22,filenam:14,unix:[14,21,13],visibl:12,instal:[8,9,0,20,18,22,17,11,23,10,26,3,5,6,15],total:[27,7,13],establish:1,from:[8,17,0,20,22,19,23,21,1,25,2,12,3,4,5,14,15,27,7],describ:[12,1],would:[17,1,25,12,5,7],apswvers:22,upgrad:[9,15,22,3],few:[5,12,22],concret:1,call:[17,0,27,10,1,25,26,12,5,6,14,20,7],usr:[8,17,0,11,23,10,26,3,5,6,20],recommend:[5,27,12,13,1],taken:[17,21,20,27,7],tape:[0,7],bill:1,type:[],until:[11,27,7,13,24],more:[21,22,1,13,4,26,7],sort:[5,25],desir:26,st_nlink:13,src:[17,7],peopl:[],hundr:[],relat:12,benchmark:[5,9,27],"19283712_yourname_s3ql":[],notic:[14,12],enhanc:5,warn:[26,10,15,7,1],sqlitelibvers:22,sens:14,known:[9,21,13],rare:[],hold:[12,1],unpack:22,cach:[8,9,21,20,23,19,2,13,3,26,15,27,7],must:[17,15,12,7,3],worst:[0,7],none:[8,20,1,27,19,3,15],word:[],sometim:15,restor:[17,9,14,15,7],dest:17,setup:22,work:[21,22,24,25,5,14,13],uniqu:1,conceptu:21,remain:27,wors:12,obvious:27,can:[0,1,2,3,5,6,7,8,10,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27],mktemp:27,about:[9,21,27,13,6,14,7],root:[17,0,11,27,10,24,26,6,20,7],fetch:[17,7],overrid:12,prompt:[8,19],tar:14,give:[5,25,14,15],process:[11,27,24,12,4,5,14,20,13],lock:[0,7],sudo:22,share:[8,17,0,11,23,10,26,3,5,6,20],accept:[8,17,0,20,19,11,23,10,24,1,25,2,3,4,26,6,15,27],high:[21,20,27,4],critic:21,minimum:[14,21],want:[11,27,24,25,5,14,16],onlin:[0,12,21,7],unfortun:[14,12,13],occur:[8,17,0,11,23,10,1,25,3,4,26,6,20],ratio:27,alwai:[17,27,25,13,5,7],end:[27,15],turn:[5,25,27,13],rather:[25,12,13,5,14,7],anoth:[13,1],ordinari:[17,11,24,7],write:[8,9,21,20,22,17,27,19,12,7,3,5,14,15,13],how:[5,25,14],manpag:[9,18],env:27,webpag:12,verifi:[12,13],simpl:[5,25,7],updat:[5,21,12,13],product:[],resourc:[9,15,16],max:[27,20],earlier:[22,13,1],pyliblzma:22,badli:[15,3],wrong:[15,13,3],endang:21,mai:[17,0,20,22,27,16,21,24,1,26,12,7,3,5,14,15,13],multipl:[8,15,23,19,2,3,20,27],redund:[5,25,21,12],secondari:[],data:[8,17,21,20,19,11,23,27,24,1,2,12,7,3,5,14,15,26,13],grow:[21,27],physic:[5,17,0,15,7],man:[18,13],indistinguish:21,"short":[0,12,7],attempt:[8,27,13,1],practic:[14,12],third:14,read:[8,17,21,20,19,23,1,2,12,7,3,14,15,27,13],neitheristhi:1,author:[27,13],favorit:[],apsw:22,element:27,issu:[17,9,0,22,27,16,10,21,12,7,26,6,13],inform:[9,27,1,25,26,20,7],maintain:27,combin:[14,12,27,1],allow:[17,0,20,27,10,25,26,12,4,5,6,14,15,7],enter:[2,23],exclus:14,volum:27,order:[17,7,1],talk:12,oper:[8,17,0,20,11,23,10,1,25,21,3,4,26,6,15,27],help:[9,22,16,26,13,3,5,15,7],over:[17,21,27,12,13,14,20,7],move:[21,15,1],soon:[],topsecret:[],increas:[14,20,27],appar:12,effici:13,still:[17,21,11,27,24,1,12,7],dynam:21,paramet:1,overwrit:[2,23],fix:[0,25,12,13,5,7],inadvert:[17,7],better:[27,22,4],window:12,html:[],restart:[27,13],persist:1,mail:[21,15,22,16],main:[10,7],might:[17,0,27,16,10,19,13,26,6,7],documents_januari:[17,7],them:[17,11,24,1,12,15,7],good:[0,25,13,5,14,7],synopsi:[8,17,0,11,23,10,25,3,4,26,6,20],thei:[17,0,18,21,25,12,5,14,7],python:[27,22,1],promin:13,safe:[17,14,7],fuse4bsd:22,dai:[5,25],initi:[],dat:[5,25],terminolog:[9,1],therebi:12,instead:[17,11,24,1,12,13,7],interrupt:15,potenti:12,now:[0,22,25,12,5,7],bigger:[],choic:22,term:[5,25,1],"__version__":22,somewher:[8,17,0,11,23,10,3,26,6,20,7],name:[17,23,1,25,2,12,5,14,20,27,7],joesdomain:[],authent:[8,9,20,19,23,1,2,12,3,15,27],achiev:[5,17,21,27,7],mode:[27,20],each:[5,25,27,20,1],debug:[8,17,0,20,11,23,10,24,19,25,2,3,4,26,6,15,27],found:[14,1],beneath:7,confid:[],side:1,mean:[21,12,13],compil:27,s3ql:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27],due:[12,13],mkfifo:27,chunk:13,hard:[14,21],idea:[5,25,0,7,13],procedur:[17,7],realli:[8,17,0,20,11,23,10,24,19,25,2,3,4,26,6,14,15,27,13],contrib:[5,14,27],meta:[26,14,7],significantli:[5,22,1],year:[5,25,21,12],s3_copi:[5,9],happen:[0,12,7,1],todo:[],special:[17,7],out:[21,12,13],lzma:[21,20,22,27],ftp:[],shown:12,network:[17,14,27,21,7],space:[17,21,7,13],open:[11,27,24,13],newli:13,log_fifo:27,content:[17,21,27,0,1,3,7],suitabl:[21,22],rational:0,internet:[21,27],print:[8,17,0,20,22,11,23,10,24,19,25,2,3,4,26,6,15,27],eth0:27,bla:27,correct:[5,25],common:[17,0,12,7],foreground:[27,20],shut:[5,11,24,13],after:[17,0,21,1,12,14,15,7],insid:17,advanc:[9,7],migrat:5,documents_janurai:[],manipul:21,situat:[14,13],given:13,free:[21,12],standard:[5,25,21,12,27],inod:13,reason:[10,27,24,1,25,12,13,5,14,19,7],base:27,theori:[12,13],usual:27,ask:[22,23,2,12,15,13],org:27,"byte":[],likelihood:12,afterward:14,bash:14,care:[17,21,7,13],her:[0,7],thread:[5,27,20],befor:[17,21,11,27,24,1,12,13,7],could:[17,7],success:[],refus:[5,25],keep:[0,27,1,25,5,7],recov:[0,7],thing:[15,3],length:12,rais:[27,20],place:[27,12],perman:[9,0,27,12,14,7],pycrypto:22,principl:27,confus:13,neglect:1,first:[21,22,27,1,12,14,15,13],origin:[],softwar:27,rang:[5,25],becaus:[17,21,27,1,12,5,7],directli:12,malici:[0,7],carri:21,onc:[0,11,27,21,24,12,13,14,20,7],clariti:[],s3q:[],number:[21,22,27,25,13,4,5,14,20,7],capac:21,restrict:[26,27,20],date:[5,25],instruct:[27,22],alreadi:[17,22,7,13],done:[27,12],wrapper:[5,4],llfuse:22,stabl:21,miss:[25,1],s3c:12,size:[21,23,2,13,26,14,20,27,7],differ:[17,21,22,27,1,26,3,5,14,15,7],convent:[8,17,0,11,23,10,3,26,6,20,13],script:[5,14,27,13],profil:[27,20],unknown:21,interact:3,s3qllock:[9,0,18,10,5,14,7],system:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,17,18,19,20,21,22,23,24,25,26,27],least:[22,27,25,12,5,20],parallel:[9,27,4,5,14,20],checkpoint:1,attack:[0,7,13],necessarili:12,demonstr:5,s3qlrm:[9,0,18,10,25,5,7],termin:[11,24,13,1],lost:[12,1],john:12,"final":7,store:[8,9,21,20,19,23,1,25,2,12,3,5,14,15,27,13],low:[21,12],servi:13,luckili:13,consol:12,option:[0,2,3,4,5,6,7,8,10,11,13,14,15,17,19,20,22,23,24,25,26,27],especi:[14,21],shelf:[0,7],tool:12,copi:[17,9,21,1,13,4,5,14,15,7],specifi:[8,20,19,23,27,1,25,2,12,3,5,15,26,13],arbitrari:[14,12],part:[21,27,1,25,4,5],pars:1,mostli:[],consult:[8,23,20,3],exactli:[5,17,25,7],than:[10,27,1,25,13,5,14,7],wide:[27,22],target:[8,17,20,27,19,3,14,15,7],cachedir:[8,15,23,19,2,3,20,27],whenev:[14,22,27,1],prevent:[2,0,23,7],remov:[17,9,0,22,27,10,25,26,7,5,6,13],eleg:21,tree:[17,9,0,10,4,5,21,7],second:[27,1,25,12,5,14,20],structur:[21,15],exampl:[21,1,25,13,5,14,7],matter:[26,14,15],temporarili:27,friend:14,video:27,minut:[5,25],led:1,pre:27,unencrypt:[2,23],sai:[17,7],comput:[5,25,21,27],entail:[],januari:[17,7],plaintext:[],explicit:14,ram:27,mind:1,argument:[26,15,3],peculiar:12,"13th":21,packag:[5,22],expir:27,increment:[],disadvantag:[17,7],need:[17,20,22,27,1,25,12,5,15,7],seem:14,exclud:[0,7],paramiko:[],caus:[12,1],equival:[5,25],irrepar:15,destroi:[0,7],moreov:7,blindli:13,atim:13,accuraci:[5,25],note:[17,9,21,22,11,27,24,25,12,7,4,5,14,15,13],also:[8,17,0,18,11,23,10,24,1,25,26,12,21,3,4,5,6,14,20,27,7],builtin:22,denomin:12,take:[17,21,13,3,26,14,15,7],which:[17,10,22,21,1,25,12,13,5,7],discrep:[],transmit:[],environ:22,uplink:5,singl:[14,20,27],swordfish:1,compat:[9,12,13],begin:1,sure:[20,22,27,12,14,15,13],unless:[2,0,23,7],distribut:[5,27,22],plenti:[],normal:[27,20,1],buffer:13,previou:[5,15,12],compress:[9,21,27,5,20,7],most:[8,17,20,19,27,1,25,12,7,3,5,15,13],beta:21,said:[],rigor:[5,25],plan:[],choos:[],homepag:[8,17,0,11,23,10,3,26,6,20],"class":[8,17,0,11,23,10,25,26,3,4,5,6,20],independ:[21,20,27],simplic:21,renam:21,correctli:[13,19],ship:[25,27,22,4],url:[8,20,19,23,1,2,12,3,15,27],doc:[8,17,0,11,23,10,26,3,5,6,20],clear:[17,15,7],later:[17,0,22,1,12,13,7],cover:14,drive:[0,7],destruct:[0,7],doe:[21,27,24,1,12,13],declar:21,snapshot:[17,9,21,26,7],runtim:[9,7],determin:[5,27,1],sourceforg:[],occasion:12,region:[5,12],hmac:21,gracefulli:[5,25],myawssecretaccesskei:[],theoret:14,show:25,carefulli:1,random:[],syntax:[27,19,2,26,15,7],connect:[14,12,27,21],permiss:[5,21,20,27],bucketnam:12,newest:[14,3],anywai:[],rotat:[8,15,27,19,3,20],redirect:27,current:[5,15,12,25,13],onli:[17,25,0,20,22,11,27,10,24,21,26,12,7,4,5,6,14,15,13],slow:13,locat:[8,17,0,11,23,10,1,3,26,6,20,7],execut:[5,15,22,7],transact:21,configur:[9,27,7],activ:[8,17,0,20,19,11,23,10,24,1,25,2,12,3,4,26,6,15,27,7],state:[5,25,21,13],haven:[],authinfo:[],latenc:[14,21,4],suppos:12,rich:12,factor:[5,12],folder:[17,22,7],local:[8,9,0,22,17,11,23,10,12,21,3,26,6,14,20,27],defin:[5,25,27,1],contribut:[5,9],variou:[26,15,3],get:[8,9,0,20,22,19,23,27,1,25,2,16,3,5,15,26,7],googlegroup:16,nasti:[0,7],stop:27,autom:[5,21],regularli:[17,7],ssl:13,s3rr:[],cannot:[5,25],ssh:[9,14,12],report:[21,22,16,1],reconstruct:[5,25,1],requir:[8,21,1,27,19,25,26,5,22,20,13],myawsaccesskeyid:[],reveal:21,enabl:[],dramat:14,intrins:14,method:[27,22],provid:[21,22,27,1,12,13],bad:14,statist:[6,7,9],though:[27,12,1],contain:[22,27,25,3,5,15],userspac:[],nowher:[],where:[26,15,13,3,1],caches:[26,27,20,7],wiki:[22,16],kernel:22,set:[5,14,20,27],bucket1:1,bucket3:1,bucket2:1,startup:27,maximum:[5,2,23,20,27],see:[8,17,0,20,11,23,10,1,25,26,12,7,3,4,5,6,14,15,27,13],num:[27,20],s3qlcp:[5,9,17,18,7],fail:[12,22,13],close:[11,24],optimum:14,whatsoev:[0,21,7],best:[14,12,22,27],concern:1,infinit:21,awar:1,statu:[8,9,0,17,11,23,10,25,12,21,3,4,26,6,20,13],detect:[21,12,1],extend:21,inconveni:21,hopefulli:13,databas:21,boundari:[5,25],label:[2,23],favor:21,enough:[14,20,27,13],between:[5,25,27,20],"import":[5,22,1],neither:[0,7],across:21,attribut:21,amazon:[9,21,1,12,5,13],august:[],kei:[21,12],weak:[],southeast:[],job:[5,27],hardlink:[17,21,7],joe:[5,1],expire_backup:[5,9,18,25],solv:14,come:[5,14,21],local0:27,addit:[17,10,27,21,1,3,26,15,7],both:[17,21,22,7,1],protect:[0,21,7],accident:[2,0,23,7,13],last:1,irregular:[5,25],extens:21,someth:[17,7],howev:[17,0,27,21,25,12,13,5,14,7],alon:1,lazi:[11,24],against:[0,21,7],configpars:1,etc:[17,27,7,13],instanc:14,freeli:27,corrupt:1,com:[8,17,0,22,11,23,10,25,3,4,26,6,14,20,16],pcp:[5,9,14,18,4],load:27,simpli:[0,7],figur:1,inspir:[17,7],buti:[],period:[0,7],insist:14,batch:[8,27,19],written:[21,20,27,1],littl:[27,12,22],shutdown:27,linux:[27,22],averag:12,guid:[8,9,18,23,3,20],assum:27,damag:[5,25,15,1],quit:22,worthless:[0,7],strong:[8,17,0,11,23,10,25,3,4,26,6,20],nikolau:27,west:[],devic:27,three:[27,12,22],been:[0,19,11,27,21,24,1,25,12,7,5,14,15,13],mark:[8,19],compon:21,secret:12,much:4,interpret:27,interest:7,subscrib:16,monthli:[17,7],immedi:[10,11,27,24,12,7],strategi:[5,25],legaci:12,infect:[0,7],upstart:[5,27,20],great:[0,7],ani:[8,17,0,22,23,21,1,25,2,12,16,5,7],rsync_arg:14,zero:14,understand:[27,20],togeth:1,els:15,tradition:[0,7],s3qlstat:[6,18,7,13,9],present:12,"case":[17,0,22,21,1,25,5,14,15,7],replic:[5,17,21,7],trojan:[0,7],ident:[17,21,7,1],look:14,gnu:13,solid:21,plain:[2,23],servic:[21,12,1],zlib:[27,20],histor:1,trick:[9,14],documents_februari:[17,7],invok:[26,15],abov:[5,25,1],error:[8,9,0,20,17,11,23,10,1,25,21,3,4,26,6,19,13],login:[12,1],invoc:[5,25],ahax:4,loos:[12,1],jibbadup:[],earli:13,runlevel:27,argpars:22,have:[17,0,20,22,11,23,16,21,24,1,25,2,12,7,4,5,14,15,27,13],advantag:[5,25,0,7],stdout:27,almost:14,mtime:13,therefor:[5,27,12,13,1],remount:12,worri:13,destin:[17,14,7,4],exit:[8,17,0,20,11,23,10,24,19,25,2,3,4,26,6,15,27],gsutil:[],conf:[5,9],incom:[],revis:[15,3],dedic:[27,13],sever:[21,22,27,1,12,4,5,14,20],tamper:[5,25],unmount:[9,11,27,24,1,12,5,14,19,13],develop:[9,21,12],minim:21,perform:[9,21,22,27,1,26,12,3,4,5,14,15,13],media:12,make:[17,0,20,22,27,21,1,25,12,13,5,14,15,7],flushcach:[26,7],same:[17,27,1,25,12,5,14,15,7],"while":[0,22,27,21,24,12,26,7],handl:[17,21,27,25,12,13,5,14,7],inconsist:13,unexpectedli:13,split:21,auto:[27,20],pai:12,document:[8,17,0,11,23,10,3,26,6,20,7],infer:[5,25],complet:[17,22,27,1,25,26,12,5,20,7],week:[5,25],geograph:12,archiv:[21,12],hostnam:12,closest:12,lie:17,optim:27,keyr:[],confidenti:[],upon:13,effect:[17,21,7,1],cycl:[5,25],solut:5,remot:[14,12,21],fuse:[27,13],temporari:12,user:[8,9,0,18,22,17,11,23,10,24,19,12,21,3,26,6,14,20,27,7],mani:[5,25,21,1],extern:[0,12,7],encrypt:[21,27,1,2,3,23],typic:[17,7],recent:[5,25,27,15,13],gss:12,appropri:27,kept:[8,15,27,19,3,20],older:15,nevertheless:[5,1],entri:[27,20,13,1],thu:1,irrelev:13,well:[0,20,27,10,1,25,5,15,7],without:[8,17,0,21,19,12,13,14,15,7],command:[0,1,2,3,4,5,6,7,8,9,10,11,13,15,17,18,19,20,22,23,24,25,26,27],thi:[0,1,2,3,4,5,6,7,8,10,12,13,14,15,16,17,19,20,21,22,23,24,25,26,27],filesystem:[27,19,2,14,20,23],gzip:21,credenti:[8,20,1,23,19,2,3,15,27],spend:14,left:13,compromis:14,identifi:[12,1],just:[8,17,0,20,11,23,10,24,19,25,2,12,3,4,26,6,14,15,27,7],less:[13,1],conform:21,tip:[9,14],lowest:12,obtain:[],rest:14,bandwidth:[5,21],touch:[17,7],openstack:21,passphras:[9,15,3,1],roughli:27,speed:[5,27],yet:[12,13],web:12,viru:[0,7],detach:[11,24],homedir:[],easi:14,hint:[26,15,3],trigger:[12,7],point:[21,11,27,1,12,20],had:[5,17,25,7],except:1,param:15,thousand:[27,12,13],add:[0,7],valid:[12,7],nor:[0,7],versa:[17,7],input:[8,19],logger:27,subsequ:[5,25,12],match:12,bin:[14,22],applic:[13,1],transpar:21,preserv:[5,21],big:[14,13],regard:[17,7],exception:[],traffic:12,know:[5,17,25,7,13],background:[11,24],amp:14,bit:21,password:[2,23,12,1],recurs:[5,9,10,7],presum:1,like:[17,0,27,21,12,13,4,20,7],loss:[12,1],daemon:[27,20],ctime:13,specif:[0,22,1,12,13,26,15,7],header:1,should:[8,17,0,20,22,19,11,23,10,24,1,7,3,26,6,14,15,27,13],anyth:[],manual:[11,22,24,16],resolv:[12,16],noth:[0,7],princip:13,necessari:15,either:[12,13,3,26,15,7],output:[8,17,0,20,11,23,10,24,19,25,2,3,4,26,6,15,27],per:27,page:18,yyyi:[5,25],imagin:[5,25,0,7],right:14,old:[8,0,20,22,27,19,25,12,3,5,15,7],often:[14,1],deal:[],ll23bi:1,interv:[5,25,14,20,27],creation:[2,23,9],some:[8,17,0,20,22,11,23,10,24,1,25,26,12,21,3,4,5,6,19,13],umount:[9,18,11,27,24,13],self:22,certain:[],strongest:[],"export":[27,20,13],flush:[27,7],guarante:[5,25,12],server:[14,12,13],librari:[22,13],"24h":[27,20],rsync:[17,0,13,4,5,14,7],backend:[8,9,17,11,23,24,1,12,7,3,26,14,20,27,13],confirm:[10,7],stronger:12,freebsd:22,avoid:[5,25],exec:27,definit:[5,25],februari:[17,7],protocol:1,usernam:1,equal:27,leav:13,slash:12,cif:27,duplic:[17,21,1,12,5,7],creep:21,refer:[18,1],machin:[],core:[5,27],plu:1,object:[20,12,27],run:[0,20,22,19,27,1,25,26,13,3,5,15,7],itself:[12,22,13,1],power:5,certif:[12,13],reach:[8,15,27,19,3,20],intellig:[5,25,21],view:18,usag:[5,17,27,25,7],symlink:21,speak:27,host:[],unreason:13,although:[21,10,7,13],eventu:12,bi23ll:1,immut:[9,0,10,5,21,14,7],impos:[],stage:[],sshf:[14,12],comparison:[],deflat:21,actual:[15,14,12,22,13],proce:22,memori:27,http:[8,17,0,22,11,23,10,25,12,3,4,26,6,20],storebackup:[17,7],acl:21,messag:[8,15,23,19,2,3,20,27],fals:[27,20],commit:[24,13],backup:[17,9,0,27,21,1,25,12,3,5,14,15,7],disabl:[8,15,27,19,3,20],block:[21,11,23,24,1,2,13,26,27,7],repair:8,client:13,real:12,encount:[16,13],xyz:[],within:[7,1],encod:27,automat:[5,9,27,12,15],two:[17,21,15,22,7],down:[5,11,24,13],ahv:14,authinfo2:[8,15,23,19,2,3,20,27],ensur:[5,25,14,21],chang:[17,9,0,27,21,12,7,3,26,15,13],insuffici:13,storag:[8,9,0,20,19,17,11,23,21,24,1,2,12,7,3,5,14,15,27,13],your:[8,17,0,18,22,11,23,10,25,26,12,3,5,6,14,20,27,7],durabl:1,manag:[9,12,15],east:[],fast:[9,21,27,7],fusermount:[11,24,13],prepar:26,wai:[0,27,21,1,12,13,14,7],transfer:[5,14,21],support:[17,21,27,12,13,20,7],question:16,s3_backup:[5,9],"long":[5,25,14,13,1],avail:[8,0,20,23,25,12,7,3,5,15,27,13],start:[22,27,1,12,4,5,14],reli:[13,1],quiet:[8,17,0,20,11,23,10,24,19,25,2,3,4,26,6,15,27],includ:[22,21,12,18,13],lot:14,"var":12,succeed:[8,17,0,11,23,10,25,3,4,26,6,20],individu:[27,20,4],"function":15,properli:5,tracker:[21,22,16],form:[5,25,12,1],offer:[17,12,7],forc:[8,2,23,19],basic:[],continu:24,sigstop:[27,20],satur:27,measur:5,newer:[12,22],don:[27,20,13],line:[26,21],bug:[21,22,16,13],faster:[10,12,7],info:[8,15,27,19,3,26,20],commun:1,made:[17,21,0,14,7],furthermor:[21,1],consist:[12,1],possibl:[0,22,27,21,12,13,14,15,7],"default":[8,20,23,19,25,2,3,4,5,15,27],bucket:[9,23,1,2,12,3,5,15,27,13],displai:13,tell:[5,25,27],asynchron:21,authfil:[8,20,1,23,19,2,3,15,27],below:[12,7],limit:[17,0,27,10,1,26,12,7,5,6,20,13],unnot:12,problem:[8,17,0,16,21,1,25,12,7,5,14,13],similar:27,expect:[27,12,1],featur:[17,9,0,21,1,12,7],creat:[17,0,20,23,10,25,2,12,7,5,21,15,27,13],classic:[],retriev:[14,12,27,20,4],dure:[15,20,12,27],day_hour:[5,25],decrypt:1,s3qlctrl:[26,9,14,18,7],strongli:[5,25],workaround:14,decreas:4,file:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,17,19,20,21,23,24,25,26,27],home:[5,14,12],request:12,exist:[17,0,23,10,1,25,2,12,5,7],improv:[5,9,14,4],mybucket:14,check:[8,9,22,27,19,12,15,13],probabl:[21,25,5,14,15,7],otherwis:[27,20],again:[0,12,7,13,24],readi:12,relatim:13,umask:27,googl:[8,9,0,22,17,11,23,10,25,12,21,3,4,26,6,20],when:[8,0,20,19,27,21,1,25,12,7,3,4,5,14,15,13],detail:[5,14,12,13],prepend:12,field:[],other:[17,0,22,27,10,21,12,16,26,6,20,7],futur:[17,0,10,13,26,6,7],rememb:[],test:[21,22],you:[17,0,20,22,19,11,23,16,10,24,1,25,2,12,7,5,14,15,27,13],shrink:21,rid:[0,7],particular:1,variabl:22,intend:[],clean:[8,19],fulli:13,mountpoint:[27,24,12,26,6,14,20,7],"return":[8,17,0,11,23,10,24,25,3,4,26,6,20,13],fsck:[8,9,18,19,27,1,12,15],briefli:[],releas:21,track:[5,25],log:[8,20,19,27,1,12,3,26,15,7],consid:[12,16,1],sql:24,noleaf:13,dd_hh:[5,25],pool:[],stai:[27,20],reduc:[5,12],infrequ:1,longer:[21,0,24,25,13,5,15,7],algorithm:[5,9,21,20,27],vice:[17,7],directori:[0,1,2,3,4,5,7,8,10,11,12,13,14,15,17,19,20,21,22,23,25,27],reliabl:[9,12,1],descript:[8,17,0,11,23,10,25,3,4,26,6,14,20,27],save:[25,14,21,1],rule:14,sftp:[9,12],depth:[],ignor:[],back:[0,7],time:[8,0,20,19,23,21,1,25,2,12,7,3,5,14,15,27,13],backward:[],s3qladm:[9,15,18,3],daili:[]},objtypes:{},titles:["The <strong class=\"program\">s3qllock</strong> command","General Information","File System Creation","The <strong class=\"program\">s3qladm</strong> command","The <strong class=\"program\">pcp</strong> command","Contributed Programs","The <strong class=\"program\">s3qlstat</strong> command","Advanced S3QL Features","The <strong class=\"program\">fsck.s3ql</strong> command","S3QL User’s Guide","The <strong class=\"program\">s3qlrm</strong> command","The <strong class=\"program\">umount.s3ql</strong> command","Storage Backends","Known Issues","Tips & Tricks","Managing Buckets","Further Resources / Getting Help","The <strong class=\"program\">s3qlcp</strong> command","Manpages","Checking for Errors","The <strong class=\"program\">mount.s3ql</strong> command","About S3QL","Installation","The <strong class=\"program\">mkfs.s3ql</strong> command","Unmounting","The <strong class=\"program\">expire_backups</strong> command","The <strong class=\"program\">s3qlctrl</strong> command","Mounting"],objnames:{},filenames:["man/lock","general","mkfs","man/adm","man/pcp","contrib","man/stat","special","man/fsck","index","man/rm","man/umount","backends","issues","tips","adm","resources","man/cp","man/index","fsck","man/mount","about","installation","man/mkfs","umount","man/expire_backups","man/ctrl","mount"]})
\ No newline at end of file diff --git a/doc/html/special.html b/doc/html/special.html index 768f100..1d795cd 100644 --- a/doc/html/special.html +++ b/doc/html/special.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>Advanced S3QL Features — S3QL 1.0.1 documentation</title> + <title>Advanced S3QL Features — S3QL 1.2 documentation</title> <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="_static/underscore.js"></script> <script type="text/javascript" src="_static/doctools.js"></script> <link rel="author" title="About these documents" href="about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="index.html" /> <link rel="next" title="Unmounting" href="umount.html" /> <link rel="prev" title="Mounting" href="mount.html" /> </head> @@ -40,7 +40,7 @@ <li class="right" > <a href="mount.html" title="Mounting" accesskey="P">previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -49,6 +49,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="adm.html">Managing Buckets</a></li> @@ -96,44 +97,40 @@ <h1>Advanced S3QL Features<a class="headerlink" href="#advanced-s3ql-features" title="Permalink to this headline">¶</a></h1> <div class="section" id="snapshotting-and-copy-on-write"> <span id="s3qlcp"></span><h2>Snapshotting and Copy-on-Write<a class="headerlink" href="#snapshotting-and-copy-on-write" title="Permalink to this headline">¶</a></h2> -<p>The command <tt class=" docutils literal"><span class="pre">s3qlcp</span></tt> can be used to duplicate a directory tree without -physically copying the file contents. This is possible due to the data -de-duplication feature of S3QL.</p> -<p>The syntax of <tt class=" docutils literal"><span class="pre">s3qlcp</span></tt> is:</p> +<p>The command <tt class="docutils literal"><span class="pre">s3qlcp</span></tt> can be used to duplicate a directory tree without +physically copying the file contents. This is made possible by the +data de-duplication feature of S3QL.</p> +<p>The syntax of <tt class="docutils literal"><span class="pre">s3qlcp</span></tt> is:</p> <div class="highlight-commandline"><div class="highlight"><pre><span class="l">s3qlcp </span><span class="ge">[options]</span><span class="l"> </span><span class="nv"><src></span><span class="l"> </span><span class="nv"><target></span><span class="l"></span> </pre></div> </div> -<p>This will replicate the contents of the directory <tt class=" docutils literal"><span class="pre"><src></span></tt> in the -directory <tt class=" docutils literal"><span class="pre"><target></span></tt>. <tt class=" docutils literal"><span class="pre"><src></span></tt> has to be an existing directory and -<tt class=" docutils literal"><span class="pre"><target></span></tt> must not exist. Moreover, both directories have to be +<p>This will replicate the contents of the directory <tt class="docutils literal"><span class="pre"><src></span></tt> in the +directory <tt class="docutils literal"><span class="pre"><target></span></tt>. <tt class="docutils literal"><span class="pre"><src></span></tt> has to be an existing directory and +<tt class="docutils literal"><span class="pre"><target></span></tt> must not exist. Moreover, both directories have to be within the same S3QL file system.</p> <p>The replication will not take any additional space. Only if one of directories is modified later on, the modified data will take additional storage space.</p> -<p><tt class=" docutils literal"><span class="pre">s3qlcp</span></tt> can only be called by the user that mounted the file system -and (if the file system was mounted with <tt class=" docutils literal"><span class="pre">--allow-other</span></tt> or <tt class=" docutils literal"><span class="pre">--allow-root</span></tt>) +<p><tt class="docutils literal"><span class="pre">s3qlcp</span></tt> can only be called by the user that mounted the file system +and (if the file system was mounted with <tt class="docutils literal"><span class="pre">--allow-other</span></tt> or <tt class="docutils literal"><span class="pre">--allow-root</span></tt>) the root user. This limitation might be removed in the future (see <a class="reference external" href="http://code.google.com/p/s3ql/issues/detail?id=155">issue 155</a>).</p> <p>Note that:</p> <ul class="simple"> <li>After the replication, both source and target directory will still -be completely ordinary directories. You can regard <tt class=" docutils literal"><span class="pre"><src></span></tt> as a -snapshot of <tt class=" docutils literal"><span class="pre"><target></span></tt> or vice versa. However, the most common -usage of <tt class=" docutils literal"><span class="pre">s3qlcp</span></tt> is to regularly duplicate the same source -directory, say <tt class=" docutils literal"><span class="pre">documents</span></tt>, to different target directories. For a +be completely ordinary directories. You can regard <tt class="docutils literal"><span class="pre"><src></span></tt> as a +snapshot of <tt class="docutils literal"><span class="pre"><target></span></tt> or vice versa. However, the most common +usage of <tt class="docutils literal"><span class="pre">s3qlcp</span></tt> is to regularly duplicate the same source +directory, say <tt class="docutils literal"><span class="pre">documents</span></tt>, to different target directories. For a e.g. monthly replication, the target directories would typically be -named something like <tt class=" docutils literal"><span class="pre">documents_Januray</span></tt> for the replication in -January, <tt class=" docutils literal"><span class="pre">documents_February</span></tt> for the replication in February etc. +named something like <tt class="docutils literal"><span class="pre">documents_January</span></tt> for the replication in +January, <tt class="docutils literal"><span class="pre">documents_February</span></tt> for the replication in February etc. In this case it is clear that the target directories should be regarded as snapshots of the source directory.</li> <li>Exactly the same effect could be achieved by an ordinary copy -program like <tt class=" docutils literal"><span class="pre">cp</span> <span class="pre">-a</span></tt>. However, this procedure would be orders of -magnitude slower, because <tt class=" docutils literal"><span class="pre">cp</span></tt> would have to read every file +program like <tt class="docutils literal"><span class="pre">cp</span> <span class="pre">-a</span></tt>. However, this procedure would be orders of +magnitude slower, because <tt class="docutils literal"><span class="pre">cp</span></tt> would have to read every file completely (so that S3QL had to fetch all the data over the network from the backend) before writing them into the destination folder.</li> -<li>Before starting with the replication, S3QL has to flush the local -cache. So if you just copied lots of new data into the file system -that has not yet been uploaded, replication will take longer than -usual.</li> </ul> <div class="section" id="snapshotting-vs-hardlinking"> <h3>Snapshotting vs Hardlinking<a class="headerlink" href="#snapshotting-vs-hardlinking" title="Permalink to this headline">¶</a></h3> @@ -159,17 +156,17 @@ any backup program.</p> <div class="section" id="getting-statistics"> <span id="s3qlstat"></span><h2>Getting Statistics<a class="headerlink" href="#getting-statistics" title="Permalink to this headline">¶</a></h2> <p>You can get more information about a mounted S3QL file system with the -<tt class=" docutils literal"><span class="pre">s3qlstat</span></tt> command. It has the following syntax:</p> +<tt class="docutils literal"><span class="pre">s3qlstat</span></tt> command. It has the following syntax:</p> <div class="highlight-commandline"><div class="highlight"><pre><span class="l">s3qlstat </span><span class="ge">[options]</span><span class="l"> </span><span class="nv"><mountpoint></span><span class="l"></span> </pre></div> </div> <p>Probably the most interesting numbers are the total size of your data, the total size after duplication, and the final size after de-duplication and compression.</p> -<p><tt class=" docutils literal"><span class="pre">s3qlstat</span></tt> can only be called by the user that mounted the file system -and (if the file system was mounted with <tt class=" docutils literal"><span class="pre">--allow-other</span></tt> or <tt class=" docutils literal"><span class="pre">--allow-root</span></tt>) +<p><tt class="docutils literal"><span class="pre">s3qlstat</span></tt> can only be called by the user that mounted the file system +and (if the file system was mounted with <tt class="docutils literal"><span class="pre">--allow-other</span></tt> or <tt class="docutils literal"><span class="pre">--allow-root</span></tt>) the root user. This limitation might be removed in the future (see <a class="reference external" href="http://code.google.com/p/s3ql/issues/detail?id=155">issue 155</a>).</p> -<p>For a full list of available options, run <tt class=" docutils literal"><span class="pre">s3qlstat</span> <span class="pre">--help</span></tt>.</p> +<p>For a full list of available options, run <tt class="docutils literal"><span class="pre">s3qlstat</span> <span class="pre">--help</span></tt>.</p> </div> <div class="section" id="immutable-trees"> <span id="s3qllock"></span><h2>Immutable Trees<a class="headerlink" href="#immutable-trees" title="Permalink to this headline">¶</a></h2> @@ -226,13 +223,13 @@ be removed entirely and immediately.</p> </div> <div class="section" id="runtime-configuration"> <span id="s3qlctrl"></span><h2>Runtime Configuration<a class="headerlink" href="#runtime-configuration" title="Permalink to this headline">¶</a></h2> -<p>The <tt class=" docutils literal"><span class="pre">s3qlctrl</span></tt> can be used to control a mounted S3QL file system. Its +<p>The <tt class="docutils literal"><span class="pre">s3qlctrl</span></tt> can be used to control a mounted S3QL file system. Its syntax is</p> <div class="highlight-commandline"><div class="highlight"><pre><span class="l">s3qlctrl </span><span class="ge">[options]</span><span class="l"> </span><span class="nv"><action></span><span class="l"> </span><span class="nv"><mountpoint></span><span class="l"> ...</span> </pre></div> </div> -<p><tt class=" docutils literal"><span class="pre"><mountpoint></span></tt> must be the location of a mounted S3QL file system. -For a list of valid options, run <tt class=" docutils literal"><span class="pre">s3qlctrl</span> <span class="pre">--help</span></tt>. <tt class=" docutils literal"><span class="pre"><action></span></tt> +<p><tt class="docutils literal"><span class="pre"><mountpoint></span></tt> must be the location of a mounted S3QL file system. +For a list of valid options, run <tt class="docutils literal"><span class="pre">s3qlctrl</span> <span class="pre">--help</span></tt>. <tt class="docutils literal"><span class="pre"><action></span></tt> may be either of:</p> <blockquote> <div><table class="docutils field-list" frame="void" rules="none"> @@ -269,7 +266,7 @@ been flushed.</td> <li class="right" > <a href="mount.html" title="Mounting" >previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/html/tips.html b/doc/html/tips.html index 6560a99..6f0d684 100644 --- a/doc/html/tips.html +++ b/doc/html/tips.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>Tips & Tricks — S3QL 1.0.1 documentation</title> + <title>Tips & Tricks — S3QL 1.2 documentation</title> <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="_static/underscore.js"></script> <script type="text/javascript" src="_static/doctools.js"></script> <link rel="author" title="About these documents" href="about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="index.html" /> <link rel="next" title="Known Issues" href="issues.html" /> <link rel="prev" title="Contributed Programs" href="contrib.html" /> </head> @@ -40,7 +40,7 @@ <li class="right" > <a href="contrib.html" title="Contributed Programs" accesskey="P">previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -49,6 +49,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="adm.html">Managing Buckets</a></li> @@ -58,6 +59,7 @@ <li class="toctree-l1"><a class="reference internal" href="fsck.html">Checking for Errors</a></li> <li class="toctree-l1"><a class="reference internal" href="contrib.html">Contributed Programs</a></li> <li class="toctree-l1 current"><a class="current reference internal" href="">Tips & Tricks</a><ul> +<li class="toctree-l2"><a class="reference internal" href="#ssh-backend">SSH Backend</a></li> <li class="toctree-l2"><a class="reference internal" href="#permanently-mounted-backup-file-system">Permanently mounted backup file system</a></li> <li class="toctree-l2"><a class="reference internal" href="#improving-copy-performance">Improving copy performance</a></li> </ul> @@ -91,8 +93,21 @@ <div class="section" id="tips-tricks"> <h1>Tips & Tricks<a class="headerlink" href="#tips-tricks" title="Permalink to this headline">¶</a></h1> +<div class="section" id="ssh-backend"> +<span id="ssh-tipp"></span><h2>SSH Backend<a class="headerlink" href="#ssh-backend" title="Permalink to this headline">¶</a></h2> +<p>By combining S3QL’s local backend with <a class="reference external" href="http://fuse.sourceforge.net/sshfs.html">sshfs</a>, it is possible to store an +S3QL file system on arbitrary SSH servers: first mount the remote +target directory into the local filesystem,</p> +<div class="highlight-commandline"><div class="highlight"><pre><span class="l">sshfs user@my.server.com:/mnt/s3ql /mnt/sshfs</span> +</pre></div> +</div> +<p>and then give the mountpoint to S3QL as a local destination:</p> +<div class="highlight-commandline"><div class="highlight"><pre><span class="l">mount.s3ql local:///mnt/sshfs/mybucket /mnt/s3ql</span> +</pre></div> +</div> +</div> <div class="section" id="permanently-mounted-backup-file-system"> -<span id="copy-performance"></span><h2>Permanently mounted backup file system<a class="headerlink" href="#permanently-mounted-backup-file-system" title="Permalink to this headline">¶</a></h2> +<h2>Permanently mounted backup file system<a class="headerlink" href="#permanently-mounted-backup-file-system" title="Permalink to this headline">¶</a></h2> <p>If you use S3QL as a backup file system, it can be useful to mount the file system permanently (rather than just mounting it for a backup and unmounting it afterwards). Especially if your file system becomes @@ -110,7 +125,12 @@ to zero).</li> </ul> </div> <div class="section" id="improving-copy-performance"> -<h2>Improving copy performance<a class="headerlink" href="#improving-copy-performance" title="Permalink to this headline">¶</a></h2> +<span id="copy-performance"></span><h2>Improving copy performance<a class="headerlink" href="#improving-copy-performance" title="Permalink to this headline">¶</a></h2> +<div class="admonition note"> +<p class="first admonition-title">Note</p> +<p class="last">The following applies only when copying data <strong>from</strong> an S3QL file +system, <strong>not</strong> when copying data <strong>to</strong> an S3QL file system.</p> +</div> <p>If you want to copy a lot of smaller files <em>from</em> an S3QL file system (e.g. for a system restore) you will probably notice that the performance is rather bad.</p> @@ -170,7 +190,7 @@ details.</p> <li class="right" > <a href="contrib.html" title="Contributed Programs" >previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/html/umount.html b/doc/html/umount.html index 23e4c45..32bd358 100644 --- a/doc/html/umount.html +++ b/doc/html/umount.html @@ -8,7 +8,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>Unmounting — S3QL 1.0.1 documentation</title> + <title>Unmounting — S3QL 1.2 documentation</title> <link rel="stylesheet" href="_static/sphinxdoc.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -16,7 +16,7 @@ <script type="text/javascript"> var DOCUMENTATION_OPTIONS = { URL_ROOT: '', - VERSION: '1.0.1', + VERSION: '1.2', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', HAS_SOURCE: true @@ -26,7 +26,7 @@ <script type="text/javascript" src="_static/underscore.js"></script> <script type="text/javascript" src="_static/doctools.js"></script> <link rel="author" title="About these documents" href="about.html" /> - <link rel="top" title="S3QL 1.0.1 documentation" href="index.html" /> + <link rel="top" title="S3QL 1.2 documentation" href="index.html" /> <link rel="next" title="Checking for Errors" href="fsck.html" /> <link rel="prev" title="Advanced S3QL Features" href="special.html" /> </head> @@ -40,7 +40,7 @@ <li class="right" > <a href="special.html" title="Advanced S3QL Features" accesskey="P">previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="sphinxsidebar"> @@ -49,6 +49,7 @@ <ul class="current"> <li class="toctree-l1"><a class="reference internal" href="about.html">About S3QL</a></li> <li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l1"><a class="reference internal" href="general.html">General Information</a></li> <li class="toctree-l1"><a class="reference internal" href="backends.html">Storage Backends</a></li> <li class="toctree-l1"><a class="reference internal" href="mkfs.html">File System Creation</a></li> <li class="toctree-l1"><a class="reference internal" href="adm.html">Managing Buckets</a></li> @@ -98,8 +99,8 @@ is able to unmount it again. If you are root and want to unmount an S3QL file system mounted by an ordinary user, you have to use the <strong class="command">fusermount -u</strong> or <strong class="command">umount</strong> command instead. Note that these commands do not block until all data has been uploaded, so -if you use them instead of <tt class=" docutils literal"><span class="pre">umount.s3ql</span></tt> then you should manually wait -for the <tt class=" docutils literal"><span class="pre">mount.s3ql</span></tt> process to terminate before shutting down the +if you use them instead of <tt class="docutils literal"><span class="pre">umount.s3ql</span></tt> then you should manually wait +for the <tt class="docutils literal"><span class="pre">mount.s3ql</span></tt> process to terminate before shutting down the system.</p> <p>The <strong class="command">umount.s3ql</strong> command accepts the following options:</p> <blockquote> @@ -124,8 +125,8 @@ background once all open files have been closed.</td></tr> </tbody> </table> </div></blockquote> -<p>If, for some reason, the <tt class=" docutils literal"><span class="pre">umount.sql</span></tt> command does not work, the file -system can also be unmounted with <tt class=" docutils literal"><span class="pre">fusermount</span> <span class="pre">-u</span> <span class="pre">-z</span></tt>. Note that this +<p>If, for some reason, the <tt class="docutils literal"><span class="pre">umount.sql</span></tt> command does not work, the file +system can also be unmounted with <tt class="docutils literal"><span class="pre">fusermount</span> <span class="pre">-u</span> <span class="pre">-z</span></tt>. Note that this command will return immediately and the file system may continue to upload data in the background for a while longer.</p> </div> @@ -145,7 +146,7 @@ upload data in the background for a while longer.</p> <li class="right" > <a href="special.html" title="Advanced S3QL Features" >previous</a> |</li> - <li><a href="index.html">S3QL 1.0.1 documentation</a> »</li> + <li><a href="index.html">S3QL 1.2 documentation</a> »</li> </ul> </div> <div class="footer"> diff --git a/doc/latex/manual.aux b/doc/latex/manual.aux index 4f41234..5c7507a 100644 --- a/doc/latex/manual.aux +++ b/doc/latex/manual.aux @@ -36,331 +36,312 @@ \@writefile{toc}{\contentsline {section}{\numberline {2.2}Installing S3QL}{4}{section.2.2}} \newlabel{installation:inst-s3ql}{{2.2}{4}{Installing S3QL\relax }{section.2.2}{}} \newlabel{installation:installing-s3ql}{{2.2}{4}{Installing S3QL\relax }{section.2.2}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {3}Storage Backends}{5}{chapter.3}} +\@writefile{toc}{\contentsline {chapter}{\numberline {3}General Information}{5}{chapter.3}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{backends::doc}{{3}{5}{Storage Backends\relax }{chapter.3}{}} -\newlabel{backends:storage-backends}{{3}{5}{Storage Backends\relax }{chapter.3}{}} -\@writefile{toc}{\contentsline {section}{\numberline {3.1}On Backend Reliability}{5}{section.3.1}} -\newlabel{backends:on-backend-reliability}{{3.1}{5}{On Backend Reliability\relax }{section.3.1}{}} -\@writefile{toc}{\contentsline {section}{\numberline {3.2}The \texttt {authinfo} file}{6}{section.3.2}} -\newlabel{backends:the-authinfo-file}{{3.2}{6}{The \texttt {authinfo} file\relax }{section.3.2}{}} -\@writefile{toc}{\contentsline {section}{\numberline {3.3}Consistency Guarantees}{6}{section.3.3}} -\newlabel{backends:consistency-guarantees}{{3.3}{6}{Consistency Guarantees\relax }{section.3.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {3.3.1}Dealing with Eventual Consistency}{6}{subsection.3.3.1}} -\newlabel{backends:dealing-with-eventual-consistency}{{3.3.1}{6}{Dealing with Eventual Consistency\relax }{subsection.3.3.1}{}} -\newlabel{backends:eventual-consistency}{{3.3.1}{6}{Dealing with Eventual Consistency\relax }{subsection.3.3.1}{}} -\@writefile{toc}{\contentsline {section}{\numberline {3.4}The Amazon S3 Backend}{7}{section.3.4}} -\newlabel{backends:the-amazon-s3-backend}{{3.4}{7}{The Amazon S3 Backend\relax }{section.3.4}{}} -\@writefile{toc}{\contentsline {section}{\numberline {3.5}The Local Backend}{8}{section.3.5}} -\newlabel{backends:the-local-backend}{{3.5}{8}{The Local Backend\relax }{section.3.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {3.6}The SFTP Backend}{8}{section.3.6}} -\newlabel{backends:the-sftp-backend}{{3.6}{8}{The SFTP Backend\relax }{section.3.6}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {4}File System Creation}{9}{chapter.4}} +\newlabel{general:general-information}{{3}{5}{General Information\relax }{chapter.3}{}} +\newlabel{general::doc}{{3}{5}{General Information\relax }{chapter.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {3.1}Terminology}{5}{section.3.1}} +\newlabel{general:terminology}{{3.1}{5}{Terminology\relax }{section.3.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {3.2}Storing Authentication Information}{5}{section.3.2}} +\newlabel{general:storing-authentication-information}{{3.2}{5}{Storing Authentication Information\relax }{section.3.2}{}} +\newlabel{general:bucket-pw}{{3.2}{5}{Storing Authentication Information\relax }{section.3.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {3.3}On Backend Reliability}{6}{section.3.3}} +\newlabel{general:backend-reliability}{{3.3}{6}{On Backend Reliability\relax }{section.3.3}{}} +\newlabel{general:on-backend-reliability}{{3.3}{6}{On Backend Reliability\relax }{section.3.3}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {4}Storage Backends}{9}{chapter.4}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{mkfs::doc}{{4}{9}{File System Creation\relax }{chapter.4}{}} -\newlabel{mkfs:file-system-creation}{{4}{9}{File System Creation\relax }{chapter.4}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {5}Managing Buckets}{11}{chapter.5}} +\newlabel{backends:id1}{{4}{9}{Storage Backends\relax }{chapter.4}{}} +\newlabel{backends::doc}{{4}{9}{Storage Backends\relax }{chapter.4}{}} +\newlabel{backends:storage-backends}{{4}{9}{Storage Backends\relax }{chapter.4}{}} +\@writefile{toc}{\contentsline {section}{\numberline {4.1}Google Storage}{9}{section.4.1}} +\newlabel{backends:google-storage}{{4.1}{9}{Google Storage\relax }{section.4.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {4.2}Amazon S3}{9}{section.4.2}} +\newlabel{backends:amazon-s3}{{4.2}{9}{Amazon S3\relax }{section.4.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {4.2.1}Reduced Redundancy Storage (RRS)}{10}{subsection.4.2.1}} +\newlabel{backends:reduced-redundancy-storage-rrs}{{4.2.1}{10}{Reduced Redundancy Storage (RRS)\relax }{subsection.4.2.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {4.2.2}Potential issues when using the US Standard storage region}{10}{subsection.4.2.2}} +\newlabel{backends:potential-issues-when-using-the-us-standard-storage-region}{{4.2.2}{10}{Potential issues when using the US Standard storage region\relax }{subsection.4.2.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {4.3}S3 compatible}{10}{section.4.3}} +\newlabel{backends:s3-compatible}{{4.3}{10}{S3 compatible\relax }{section.4.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {4.4}Local}{11}{section.4.4}} +\newlabel{backends:local}{{4.4}{11}{Local\relax }{section.4.4}{}} +\@writefile{toc}{\contentsline {section}{\numberline {4.5}SSH/SFTP}{11}{section.4.5}} +\newlabel{backends:ssh-sftp}{{4.5}{11}{SSH/SFTP\relax }{section.4.5}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {5}File System Creation}{13}{chapter.5}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{adm::doc}{{5}{11}{Managing Buckets\relax }{chapter.5}{}} -\newlabel{adm:managing-buckets}{{5}{11}{Managing Buckets\relax }{chapter.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {5.1}Changing the Passphrase}{11}{section.5.1}} -\newlabel{adm:changing-the-passphrase}{{5.1}{11}{Changing the Passphrase\relax }{section.5.1}{}} -\@writefile{toc}{\contentsline {section}{\numberline {5.2}Upgrading the file system}{11}{section.5.2}} -\newlabel{adm:upgrading-the-file-system}{{5.2}{11}{Upgrading the file system\relax }{section.5.2}{}} -\@writefile{toc}{\contentsline {section}{\numberline {5.3}Deleting a file system}{12}{section.5.3}} -\newlabel{adm:deleting-a-file-system}{{5.3}{12}{Deleting a file system\relax }{section.5.3}{}} -\@writefile{toc}{\contentsline {section}{\numberline {5.4}Restoring Metadata Backups}{12}{section.5.4}} -\newlabel{adm:restoring-metadata-backups}{{5.4}{12}{Restoring Metadata Backups\relax }{section.5.4}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {6}Mounting}{13}{chapter.6}} +\newlabel{mkfs::doc}{{5}{13}{File System Creation\relax }{chapter.5}{}} +\newlabel{mkfs:file-system-creation}{{5}{13}{File System Creation\relax }{chapter.5}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {6}Managing Buckets}{15}{chapter.6}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{mount:mounting}{{6}{13}{Mounting\relax }{chapter.6}{}} -\newlabel{mount::doc}{{6}{13}{Mounting\relax }{chapter.6}{}} -\@writefile{toc}{\contentsline {section}{\numberline {6.1}Storing Encryption Passwords}{14}{section.6.1}} -\newlabel{mount:bucket-pw}{{6.1}{14}{Storing Encryption Passwords\relax }{section.6.1}{}} -\newlabel{mount:storing-encryption-passwords}{{6.1}{14}{Storing Encryption Passwords\relax }{section.6.1}{}} -\@writefile{toc}{\contentsline {section}{\numberline {6.2}Compression Algorithms}{14}{section.6.2}} -\newlabel{mount:compression-algorithms}{{6.2}{14}{Compression Algorithms\relax }{section.6.2}{}} -\@writefile{toc}{\contentsline {section}{\numberline {6.3}Parallel Compression}{15}{section.6.3}} -\newlabel{mount:parallel-compression}{{6.3}{15}{Parallel Compression\relax }{section.6.3}{}} -\@writefile{toc}{\contentsline {section}{\numberline {6.4}Notes about Caching}{15}{section.6.4}} -\newlabel{mount:notes-about-caching}{{6.4}{15}{Notes about Caching\relax }{section.6.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {6.4.1}Maximum Number of Cache Entries}{15}{subsection.6.4.1}} -\newlabel{mount:maximum-number-of-cache-entries}{{6.4.1}{15}{Maximum Number of Cache Entries\relax }{subsection.6.4.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {6.4.2}Cache Flushing and Expiration}{15}{subsection.6.4.2}} -\newlabel{mount:cache-flushing-and-expiration}{{6.4.2}{15}{Cache Flushing and Expiration\relax }{subsection.6.4.2}{}} -\@writefile{toc}{\contentsline {section}{\numberline {6.5}Automatic Mounting}{15}{section.6.5}} -\newlabel{mount:automatic-mounting}{{6.5}{15}{Automatic Mounting\relax }{section.6.5}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {7}Advanced S3QL Features}{17}{chapter.7}} +\newlabel{adm::doc}{{6}{15}{Managing Buckets\relax }{chapter.6}{}} +\newlabel{adm:managing-buckets}{{6}{15}{Managing Buckets\relax }{chapter.6}{}} +\@writefile{toc}{\contentsline {section}{\numberline {6.1}Changing the Passphrase}{15}{section.6.1}} +\newlabel{adm:changing-the-passphrase}{{6.1}{15}{Changing the Passphrase\relax }{section.6.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {6.2}Upgrading the file system}{15}{section.6.2}} +\newlabel{adm:upgrading-the-file-system}{{6.2}{15}{Upgrading the file system\relax }{section.6.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {6.3}Deleting a file system}{16}{section.6.3}} +\newlabel{adm:deleting-a-file-system}{{6.3}{16}{Deleting a file system\relax }{section.6.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {6.4}Restoring Metadata Backups}{16}{section.6.4}} +\newlabel{adm:restoring-metadata-backups}{{6.4}{16}{Restoring Metadata Backups\relax }{section.6.4}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {7}Mounting}{17}{chapter.7}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{special:advanced-s3ql-features}{{7}{17}{Advanced S3QL Features\relax }{chapter.7}{}} -\newlabel{special::doc}{{7}{17}{Advanced S3QL Features\relax }{chapter.7}{}} -\@writefile{toc}{\contentsline {section}{\numberline {7.1}Snapshotting and Copy-on-Write}{17}{section.7.1}} -\newlabel{special:snapshotting-and-copy-on-write}{{7.1}{17}{Snapshotting and Copy-on-Write\relax }{section.7.1}{}} -\newlabel{special:s3qlcp}{{7.1}{17}{Snapshotting and Copy-on-Write\relax }{section.7.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {7.1.1}Snapshotting vs Hardlinking}{17}{subsection.7.1.1}} -\newlabel{special:snapshotting-vs-hardlinking}{{7.1.1}{17}{Snapshotting vs Hardlinking\relax }{subsection.7.1.1}{}} -\@writefile{toc}{\contentsline {section}{\numberline {7.2}Getting Statistics}{18}{section.7.2}} -\newlabel{special:s3qlstat}{{7.2}{18}{Getting Statistics\relax }{section.7.2}{}} -\newlabel{special:getting-statistics}{{7.2}{18}{Getting Statistics\relax }{section.7.2}{}} -\@writefile{toc}{\contentsline {section}{\numberline {7.3}Immutable Trees}{18}{section.7.3}} -\newlabel{special:immutable-trees}{{7.3}{18}{Immutable Trees\relax }{section.7.3}{}} -\newlabel{special:s3qllock}{{7.3}{18}{Immutable Trees\relax }{section.7.3}{}} -\@writefile{toc}{\contentsline {section}{\numberline {7.4}Fast Recursive Removal}{19}{section.7.4}} -\newlabel{special:s3qlrm}{{7.4}{19}{Fast Recursive Removal\relax }{section.7.4}{}} -\newlabel{special:fast-recursive-removal}{{7.4}{19}{Fast Recursive Removal\relax }{section.7.4}{}} -\@writefile{toc}{\contentsline {section}{\numberline {7.5}Runtime Configuration}{19}{section.7.5}} -\newlabel{special:runtime-configuration}{{7.5}{19}{Runtime Configuration\relax }{section.7.5}{}} -\newlabel{special:s3qlctrl}{{7.5}{19}{Runtime Configuration\relax }{section.7.5}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {8}Unmounting}{21}{chapter.8}} +\newlabel{mount:mounting}{{7}{17}{Mounting\relax }{chapter.7}{}} +\newlabel{mount::doc}{{7}{17}{Mounting\relax }{chapter.7}{}} +\@writefile{toc}{\contentsline {section}{\numberline {7.1}Compression Algorithms}{18}{section.7.1}} +\newlabel{mount:compression-algorithms}{{7.1}{18}{Compression Algorithms\relax }{section.7.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {7.2}Parallel Compression}{18}{section.7.2}} +\newlabel{mount:parallel-compression}{{7.2}{18}{Parallel Compression\relax }{section.7.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {7.3}Notes about Caching}{19}{section.7.3}} +\newlabel{mount:notes-about-caching}{{7.3}{19}{Notes about Caching\relax }{section.7.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {7.3.1}Maximum Number of Cache Entries}{19}{subsection.7.3.1}} +\newlabel{mount:maximum-number-of-cache-entries}{{7.3.1}{19}{Maximum Number of Cache Entries\relax }{subsection.7.3.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {7.3.2}Cache Flushing and Expiration}{19}{subsection.7.3.2}} +\newlabel{mount:cache-flushing-and-expiration}{{7.3.2}{19}{Cache Flushing and Expiration\relax }{subsection.7.3.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {7.4}Automatic Mounting}{19}{section.7.4}} +\newlabel{mount:automatic-mounting}{{7.4}{19}{Automatic Mounting\relax }{section.7.4}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {8}Advanced S3QL Features}{21}{chapter.8}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{umount::doc}{{8}{21}{Unmounting\relax }{chapter.8}{}} -\newlabel{umount:unmounting}{{8}{21}{Unmounting\relax }{chapter.8}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {9}Checking for Errors}{23}{chapter.9}} +\newlabel{special:advanced-s3ql-features}{{8}{21}{Advanced S3QL Features\relax }{chapter.8}{}} +\newlabel{special::doc}{{8}{21}{Advanced S3QL Features\relax }{chapter.8}{}} +\@writefile{toc}{\contentsline {section}{\numberline {8.1}Snapshotting and Copy-on-Write}{21}{section.8.1}} +\newlabel{special:snapshotting-and-copy-on-write}{{8.1}{21}{Snapshotting and Copy-on-Write\relax }{section.8.1}{}} +\newlabel{special:s3qlcp}{{8.1}{21}{Snapshotting and Copy-on-Write\relax }{section.8.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {8.1.1}Snapshotting vs Hardlinking}{21}{subsection.8.1.1}} +\newlabel{special:snapshotting-vs-hardlinking}{{8.1.1}{21}{Snapshotting vs Hardlinking\relax }{subsection.8.1.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {8.2}Getting Statistics}{22}{section.8.2}} +\newlabel{special:s3qlstat}{{8.2}{22}{Getting Statistics\relax }{section.8.2}{}} +\newlabel{special:getting-statistics}{{8.2}{22}{Getting Statistics\relax }{section.8.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {8.3}Immutable Trees}{22}{section.8.3}} +\newlabel{special:immutable-trees}{{8.3}{22}{Immutable Trees\relax }{section.8.3}{}} +\newlabel{special:s3qllock}{{8.3}{22}{Immutable Trees\relax }{section.8.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {8.4}Fast Recursive Removal}{23}{section.8.4}} +\newlabel{special:s3qlrm}{{8.4}{23}{Fast Recursive Removal\relax }{section.8.4}{}} +\newlabel{special:fast-recursive-removal}{{8.4}{23}{Fast Recursive Removal\relax }{section.8.4}{}} +\@writefile{toc}{\contentsline {section}{\numberline {8.5}Runtime Configuration}{23}{section.8.5}} +\newlabel{special:runtime-configuration}{{8.5}{23}{Runtime Configuration\relax }{section.8.5}{}} +\newlabel{special:s3qlctrl}{{8.5}{23}{Runtime Configuration\relax }{section.8.5}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {9}Unmounting}{25}{chapter.9}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{fsck:checking-for-errors}{{9}{23}{Checking for Errors\relax }{chapter.9}{}} -\newlabel{fsck::doc}{{9}{23}{Checking for Errors\relax }{chapter.9}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {10}Contributed Programs}{25}{chapter.10}} +\newlabel{umount::doc}{{9}{25}{Unmounting\relax }{chapter.9}{}} +\newlabel{umount:unmounting}{{9}{25}{Unmounting\relax }{chapter.9}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {10}Checking for Errors}{27}{chapter.10}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{contrib:contributed-programs}{{10}{25}{Contributed Programs\relax }{chapter.10}{}} -\newlabel{contrib::doc}{{10}{25}{Contributed Programs\relax }{chapter.10}{}} -\@writefile{toc}{\contentsline {section}{\numberline {10.1}benchmark.py}{25}{section.10.1}} -\newlabel{contrib:benchmark-py}{{10.1}{25}{benchmark.py\relax }{section.10.1}{}} -\@writefile{toc}{\contentsline {section}{\numberline {10.2}s3\_copy.py}{25}{section.10.2}} -\newlabel{contrib:s3-copy-py}{{10.2}{25}{s3\_copy.py\relax }{section.10.2}{}} -\@writefile{toc}{\contentsline {section}{\numberline {10.3}pcp.py}{25}{section.10.3}} -\newlabel{contrib:pcp-py}{{10.3}{25}{pcp.py\relax }{section.10.3}{}} -\newlabel{contrib:pcp}{{10.3}{25}{pcp.py\relax }{section.10.3}{}} -\@writefile{toc}{\contentsline {section}{\numberline {10.4}s3\_backup.sh}{25}{section.10.4}} -\newlabel{contrib:s3-backup-sh}{{10.4}{25}{s3\_backup.sh\relax }{section.10.4}{}} -\@writefile{toc}{\contentsline {section}{\numberline {10.5}expire\_backups.py}{26}{section.10.5}} -\newlabel{contrib:expire-backups-py}{{10.5}{26}{expire\_backups.py\relax }{section.10.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {10.6}s3ql.conf}{27}{section.10.6}} -\newlabel{contrib:s3ql-conf}{{10.6}{27}{s3ql.conf\relax }{section.10.6}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {11}Tips \& Tricks}{29}{chapter.11}} +\newlabel{fsck:checking-for-errors}{{10}{27}{Checking for Errors\relax }{chapter.10}{}} +\newlabel{fsck::doc}{{10}{27}{Checking for Errors\relax }{chapter.10}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {11}Contributed Programs}{29}{chapter.11}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{tips:tips-tricks}{{11}{29}{Tips \& Tricks\relax }{chapter.11}{}} -\newlabel{tips::doc}{{11}{29}{Tips \& Tricks\relax }{chapter.11}{}} -\@writefile{toc}{\contentsline {section}{\numberline {11.1}Permanently mounted backup file system}{29}{section.11.1}} -\newlabel{tips:copy-performance}{{11.1}{29}{Permanently mounted backup file system\relax }{section.11.1}{}} -\newlabel{tips:permanently-mounted-backup-file-system}{{11.1}{29}{Permanently mounted backup file system\relax }{section.11.1}{}} -\@writefile{toc}{\contentsline {section}{\numberline {11.2}Improving copy performance}{29}{section.11.2}} -\newlabel{tips:improving-copy-performance}{{11.2}{29}{Improving copy performance\relax }{section.11.2}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {12}Known Issues}{31}{chapter.12}} +\newlabel{contrib:contributed-programs}{{11}{29}{Contributed Programs\relax }{chapter.11}{}} +\newlabel{contrib::doc}{{11}{29}{Contributed Programs\relax }{chapter.11}{}} +\@writefile{toc}{\contentsline {section}{\numberline {11.1}benchmark.py}{29}{section.11.1}} +\newlabel{contrib:benchmark-py}{{11.1}{29}{benchmark.py\relax }{section.11.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {11.2}s3\_copy.py}{29}{section.11.2}} +\newlabel{contrib:s3-copy-py}{{11.2}{29}{s3\_copy.py\relax }{section.11.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {11.3}pcp.py}{29}{section.11.3}} +\newlabel{contrib:pcp-py}{{11.3}{29}{pcp.py\relax }{section.11.3}{}} +\newlabel{contrib:pcp}{{11.3}{29}{pcp.py\relax }{section.11.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {11.4}s3\_backup.sh}{29}{section.11.4}} +\newlabel{contrib:s3-backup-sh}{{11.4}{29}{s3\_backup.sh\relax }{section.11.4}{}} +\@writefile{toc}{\contentsline {section}{\numberline {11.5}expire\_backups.py}{30}{section.11.5}} +\newlabel{contrib:expire-backups-py}{{11.5}{30}{expire\_backups.py\relax }{section.11.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {11.6}s3ql.conf}{31}{section.11.6}} +\newlabel{contrib:s3ql-conf}{{11.6}{31}{s3ql.conf\relax }{section.11.6}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {12}Tips \& Tricks}{33}{chapter.12}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{issues:known-issues}{{12}{31}{Known Issues\relax }{chapter.12}{}} -\newlabel{issues::doc}{{12}{31}{Known Issues\relax }{chapter.12}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {13}Manpages}{33}{chapter.13}} +\newlabel{tips:tips-tricks}{{12}{33}{Tips \& Tricks\relax }{chapter.12}{}} +\newlabel{tips::doc}{{12}{33}{Tips \& Tricks\relax }{chapter.12}{}} +\@writefile{toc}{\contentsline {section}{\numberline {12.1}SSH Backend}{33}{section.12.1}} +\newlabel{tips:ssh-tipp}{{12.1}{33}{SSH Backend\relax }{section.12.1}{}} +\newlabel{tips:ssh-backend}{{12.1}{33}{SSH Backend\relax }{section.12.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {12.2}Permanently mounted backup file system}{33}{section.12.2}} +\newlabel{tips:permanently-mounted-backup-file-system}{{12.2}{33}{Permanently mounted backup file system\relax }{section.12.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {12.3}Improving copy performance}{33}{section.12.3}} +\newlabel{tips:copy-performance}{{12.3}{33}{Improving copy performance\relax }{section.12.3}{}} +\newlabel{tips:improving-copy-performance}{{12.3}{33}{Improving copy performance\relax }{section.12.3}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {13}Known Issues}{35}{chapter.13}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{man/index:manpages}{{13}{33}{Manpages\relax }{chapter.13}{}} -\newlabel{man/index::doc}{{13}{33}{Manpages\relax }{chapter.13}{}} -\@writefile{toc}{\contentsline {section}{\numberline {13.1}The \textbf {mkfs.s3ql} command}{33}{section.13.1}} -\newlabel{man/mkfs:the-mkfs-s3ql-command}{{13.1}{33}{The \textbf {mkfs.s3ql} command\relax }{section.13.1}{}} -\newlabel{man/mkfs::doc}{{13.1}{33}{The \textbf {mkfs.s3ql} command\relax }{section.13.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.1.1}Synopsis}{33}{subsection.13.1.1}} -\newlabel{man/mkfs:synopsis}{{13.1.1}{33}{Synopsis\relax }{subsection.13.1.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.1.2}Description}{33}{subsection.13.1.2}} -\newlabel{man/mkfs:description}{{13.1.2}{33}{Description\relax }{subsection.13.1.2}{}} -\@writefile{toc}{\contentsline {subsubsection}{Amazon S3}{33}{subsubsection*.3}} -\newlabel{man/mkfs:amazon-s3}{{13.1.2}{33}{Amazon S3\relax }{subsubsection*.3}{}} -\@writefile{toc}{\contentsline {subsubsection}{Local}{33}{subsubsection*.4}} -\newlabel{man/mkfs:local}{{13.1.2}{33}{Local\relax }{subsubsection*.4}{}} -\@writefile{toc}{\contentsline {subsubsection}{SFTP}{33}{subsubsection*.5}} -\newlabel{man/mkfs:sftp}{{13.1.2}{33}{SFTP\relax }{subsubsection*.5}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.1.3}Options}{34}{subsection.13.1.3}} -\newlabel{man/mkfs:options}{{13.1.3}{34}{Options\relax }{subsection.13.1.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.1.4}Files}{34}{subsection.13.1.4}} -\newlabel{man/mkfs:files}{{13.1.4}{34}{Files\relax }{subsection.13.1.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.1.5}Exit Status}{34}{subsection.13.1.5}} -\newlabel{man/mkfs:exit-status}{{13.1.5}{34}{Exit Status\relax }{subsection.13.1.5}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.1.6}See Also}{34}{subsection.13.1.6}} -\newlabel{man/mkfs:see-also}{{13.1.6}{34}{See Also\relax }{subsection.13.1.6}{}} -\@writefile{toc}{\contentsline {section}{\numberline {13.2}The \textbf {s3qladm} command}{34}{section.13.2}} -\newlabel{man/adm::doc}{{13.2}{34}{The \textbf {s3qladm} command\relax }{section.13.2}{}} -\newlabel{man/adm:the-s3qladm-command}{{13.2}{34}{The \textbf {s3qladm} command\relax }{section.13.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.2.1}Synopsis}{34}{subsection.13.2.1}} -\newlabel{man/adm:synopsis}{{13.2.1}{34}{Synopsis\relax }{subsection.13.2.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.2.2}Description}{35}{subsection.13.2.2}} -\newlabel{man/adm:description}{{13.2.2}{35}{Description\relax }{subsection.13.2.2}{}} -\@writefile{toc}{\contentsline {subsubsection}{Amazon S3}{35}{subsubsection*.6}} -\newlabel{man/adm:amazon-s3}{{13.2.2}{35}{Amazon S3\relax }{subsubsection*.6}{}} -\@writefile{toc}{\contentsline {subsubsection}{Local}{35}{subsubsection*.7}} -\newlabel{man/adm:local}{{13.2.2}{35}{Local\relax }{subsubsection*.7}{}} -\@writefile{toc}{\contentsline {subsubsection}{SFTP}{35}{subsubsection*.8}} -\newlabel{man/adm:sftp}{{13.2.2}{35}{SFTP\relax }{subsubsection*.8}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.2.3}Options}{35}{subsection.13.2.3}} -\newlabel{man/adm:options}{{13.2.3}{35}{Options\relax }{subsection.13.2.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.2.4}Actions}{35}{subsection.13.2.4}} -\newlabel{man/adm:actions}{{13.2.4}{35}{Actions\relax }{subsection.13.2.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.2.5}Files}{36}{subsection.13.2.5}} -\newlabel{man/adm:files}{{13.2.5}{36}{Files\relax }{subsection.13.2.5}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.2.6}Exit Status}{36}{subsection.13.2.6}} -\newlabel{man/adm:exit-status}{{13.2.6}{36}{Exit Status\relax }{subsection.13.2.6}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.2.7}See Also}{36}{subsection.13.2.7}} -\newlabel{man/adm:see-also}{{13.2.7}{36}{See Also\relax }{subsection.13.2.7}{}} -\@writefile{toc}{\contentsline {section}{\numberline {13.3}The \textbf {mount.s3ql} command}{36}{section.13.3}} -\newlabel{man/mount::doc}{{13.3}{36}{The \textbf {mount.s3ql} command\relax }{section.13.3}{}} -\newlabel{man/mount:the-mount-s3ql-command}{{13.3}{36}{The \textbf {mount.s3ql} command\relax }{section.13.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.3.1}Synopsis}{36}{subsection.13.3.1}} -\newlabel{man/mount:synopsis}{{13.3.1}{36}{Synopsis\relax }{subsection.13.3.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.3.2}Description}{36}{subsection.13.3.2}} -\newlabel{man/mount:description}{{13.3.2}{36}{Description\relax }{subsection.13.3.2}{}} -\@writefile{toc}{\contentsline {subsubsection}{Amazon S3}{36}{subsubsection*.9}} -\newlabel{man/mount:amazon-s3}{{13.3.2}{36}{Amazon S3\relax }{subsubsection*.9}{}} -\@writefile{toc}{\contentsline {subsubsection}{Local}{36}{subsubsection*.10}} -\newlabel{man/mount:local}{{13.3.2}{36}{Local\relax }{subsubsection*.10}{}} -\@writefile{toc}{\contentsline {subsubsection}{SFTP}{36}{subsubsection*.11}} -\newlabel{man/mount:sftp}{{13.3.2}{36}{SFTP\relax }{subsubsection*.11}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.3.3}Options}{37}{subsection.13.3.3}} -\newlabel{man/mount:options}{{13.3.3}{37}{Options\relax }{subsection.13.3.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.3.4}Files}{38}{subsection.13.3.4}} -\newlabel{man/mount:files}{{13.3.4}{38}{Files\relax }{subsection.13.3.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.3.5}Exit Status}{38}{subsection.13.3.5}} -\newlabel{man/mount:exit-status}{{13.3.5}{38}{Exit Status\relax }{subsection.13.3.5}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.3.6}See Also}{38}{subsection.13.3.6}} -\newlabel{man/mount:see-also}{{13.3.6}{38}{See Also\relax }{subsection.13.3.6}{}} -\@writefile{toc}{\contentsline {section}{\numberline {13.4}The \textbf {s3qlstat} command}{38}{section.13.4}} -\newlabel{man/stat:the-s3qlstat-command}{{13.4}{38}{The \textbf {s3qlstat} command\relax }{section.13.4}{}} -\newlabel{man/stat::doc}{{13.4}{38}{The \textbf {s3qlstat} command\relax }{section.13.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.4.1}Synopsis}{38}{subsection.13.4.1}} -\newlabel{man/stat:synopsis}{{13.4.1}{38}{Synopsis\relax }{subsection.13.4.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.4.2}Description}{38}{subsection.13.4.2}} -\newlabel{man/stat:description}{{13.4.2}{38}{Description\relax }{subsection.13.4.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.4.3}Options}{38}{subsection.13.4.3}} -\newlabel{man/stat:options}{{13.4.3}{38}{Options\relax }{subsection.13.4.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.4.4}Exit Status}{38}{subsection.13.4.4}} -\newlabel{man/stat:exit-status}{{13.4.4}{38}{Exit Status\relax }{subsection.13.4.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.4.5}See Also}{38}{subsection.13.4.5}} -\newlabel{man/stat:see-also}{{13.4.5}{38}{See Also\relax }{subsection.13.4.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {13.5}The \textbf {s3qlctrl} command}{39}{section.13.5}} -\newlabel{man/ctrl:the-s3qlctrl-command}{{13.5}{39}{The \textbf {s3qlctrl} command\relax }{section.13.5}{}} -\newlabel{man/ctrl::doc}{{13.5}{39}{The \textbf {s3qlctrl} command\relax }{section.13.5}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.5.1}Synopsis}{39}{subsection.13.5.1}} -\newlabel{man/ctrl:synopsis}{{13.5.1}{39}{Synopsis\relax }{subsection.13.5.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.5.2}Description}{39}{subsection.13.5.2}} -\newlabel{man/ctrl:description}{{13.5.2}{39}{Description\relax }{subsection.13.5.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.5.3}Options}{39}{subsection.13.5.3}} -\newlabel{man/ctrl:options}{{13.5.3}{39}{Options\relax }{subsection.13.5.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.5.4}Exit Status}{39}{subsection.13.5.4}} -\newlabel{man/ctrl:exit-status}{{13.5.4}{39}{Exit Status\relax }{subsection.13.5.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.5.5}See Also}{39}{subsection.13.5.5}} -\newlabel{man/ctrl:see-also}{{13.5.5}{39}{See Also\relax }{subsection.13.5.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {13.6}The \textbf {s3qlcp} command}{40}{section.13.6}} -\newlabel{man/cp:the-s3qlcp-command}{{13.6}{40}{The \textbf {s3qlcp} command\relax }{section.13.6}{}} -\newlabel{man/cp::doc}{{13.6}{40}{The \textbf {s3qlcp} command\relax }{section.13.6}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.6.1}Synopsis}{40}{subsection.13.6.1}} -\newlabel{man/cp:synopsis}{{13.6.1}{40}{Synopsis\relax }{subsection.13.6.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.6.2}Description}{40}{subsection.13.6.2}} -\newlabel{man/cp:description}{{13.6.2}{40}{Description\relax }{subsection.13.6.2}{}} -\@writefile{toc}{\contentsline {subsubsection}{Snapshotting vs Hardlinking}{40}{subsubsection*.12}} -\newlabel{man/cp:snapshotting-vs-hardlinking}{{13.6.2}{40}{Snapshotting vs Hardlinking\relax }{subsubsection*.12}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.6.3}Options}{40}{subsection.13.6.3}} -\newlabel{man/cp:options}{{13.6.3}{40}{Options\relax }{subsection.13.6.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.6.4}Exit Status}{41}{subsection.13.6.4}} -\newlabel{man/cp:exit-status}{{13.6.4}{41}{Exit Status\relax }{subsection.13.6.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.6.5}See Also}{41}{subsection.13.6.5}} -\newlabel{man/cp:see-also}{{13.6.5}{41}{See Also\relax }{subsection.13.6.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {13.7}The \textbf {s3qlrm} command}{41}{section.13.7}} -\newlabel{man/rm::doc}{{13.7}{41}{The \textbf {s3qlrm} command\relax }{section.13.7}{}} -\newlabel{man/rm:the-s3qlrm-command}{{13.7}{41}{The \textbf {s3qlrm} command\relax }{section.13.7}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.7.1}Synopsis}{41}{subsection.13.7.1}} -\newlabel{man/rm:synopsis}{{13.7.1}{41}{Synopsis\relax }{subsection.13.7.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.7.2}Description}{41}{subsection.13.7.2}} -\newlabel{man/rm:description}{{13.7.2}{41}{Description\relax }{subsection.13.7.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.7.3}Options}{41}{subsection.13.7.3}} -\newlabel{man/rm:options}{{13.7.3}{41}{Options\relax }{subsection.13.7.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.7.4}Exit Status}{41}{subsection.13.7.4}} -\newlabel{man/rm:exit-status}{{13.7.4}{41}{Exit Status\relax }{subsection.13.7.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.7.5}See Also}{41}{subsection.13.7.5}} -\newlabel{man/rm:see-also}{{13.7.5}{41}{See Also\relax }{subsection.13.7.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {13.8}The \textbf {s3qllock} command}{42}{section.13.8}} -\newlabel{man/lock:the-s3qllock-command}{{13.8}{42}{The \textbf {s3qllock} command\relax }{section.13.8}{}} -\newlabel{man/lock::doc}{{13.8}{42}{The \textbf {s3qllock} command\relax }{section.13.8}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.8.1}Synopsis}{42}{subsection.13.8.1}} -\newlabel{man/lock:synopsis}{{13.8.1}{42}{Synopsis\relax }{subsection.13.8.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.8.2}Description}{42}{subsection.13.8.2}} -\newlabel{man/lock:description}{{13.8.2}{42}{Description\relax }{subsection.13.8.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.8.3}Rationale}{42}{subsection.13.8.3}} -\newlabel{man/lock:rationale}{{13.8.3}{42}{Rationale\relax }{subsection.13.8.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.8.4}Options}{42}{subsection.13.8.4}} -\newlabel{man/lock:options}{{13.8.4}{42}{Options\relax }{subsection.13.8.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.8.5}Exit Status}{42}{subsection.13.8.5}} -\newlabel{man/lock:exit-status}{{13.8.5}{42}{Exit Status\relax }{subsection.13.8.5}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.8.6}See Also}{43}{subsection.13.8.6}} -\newlabel{man/lock:see-also}{{13.8.6}{43}{See Also\relax }{subsection.13.8.6}{}} -\@writefile{toc}{\contentsline {section}{\numberline {13.9}The \textbf {umount.s3ql} command}{43}{section.13.9}} -\newlabel{man/umount::doc}{{13.9}{43}{The \textbf {umount.s3ql} command\relax }{section.13.9}{}} -\newlabel{man/umount:the-umount-s3ql-command}{{13.9}{43}{The \textbf {umount.s3ql} command\relax }{section.13.9}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.9.1}Synopsis}{43}{subsection.13.9.1}} -\newlabel{man/umount:synopsis}{{13.9.1}{43}{Synopsis\relax }{subsection.13.9.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.9.2}Description}{43}{subsection.13.9.2}} -\newlabel{man/umount:description}{{13.9.2}{43}{Description\relax }{subsection.13.9.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.9.3}Options}{43}{subsection.13.9.3}} -\newlabel{man/umount:options}{{13.9.3}{43}{Options\relax }{subsection.13.9.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.9.4}Exit Status}{43}{subsection.13.9.4}} -\newlabel{man/umount:exit-status}{{13.9.4}{43}{Exit Status\relax }{subsection.13.9.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.9.5}See Also}{43}{subsection.13.9.5}} -\newlabel{man/umount:see-also}{{13.9.5}{43}{See Also\relax }{subsection.13.9.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {13.10}The \textbf {fsck.s3ql} command}{44}{section.13.10}} -\newlabel{man/fsck::doc}{{13.10}{44}{The \textbf {fsck.s3ql} command\relax }{section.13.10}{}} -\newlabel{man/fsck:the-fsck-s3ql-command}{{13.10}{44}{The \textbf {fsck.s3ql} command\relax }{section.13.10}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.10.1}Synopsis}{44}{subsection.13.10.1}} -\newlabel{man/fsck:synopsis}{{13.10.1}{44}{Synopsis\relax }{subsection.13.10.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.10.2}Description}{44}{subsection.13.10.2}} -\newlabel{man/fsck:description}{{13.10.2}{44}{Description\relax }{subsection.13.10.2}{}} -\@writefile{toc}{\contentsline {subsubsection}{Amazon S3}{44}{subsubsection*.13}} -\newlabel{man/fsck:amazon-s3}{{13.10.2}{44}{Amazon S3\relax }{subsubsection*.13}{}} -\@writefile{toc}{\contentsline {subsubsection}{Local}{44}{subsubsection*.14}} -\newlabel{man/fsck:local}{{13.10.2}{44}{Local\relax }{subsubsection*.14}{}} -\@writefile{toc}{\contentsline {subsubsection}{SFTP}{44}{subsubsection*.15}} -\newlabel{man/fsck:sftp}{{13.10.2}{44}{SFTP\relax }{subsubsection*.15}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.10.3}Options}{44}{subsection.13.10.3}} -\newlabel{man/fsck:options}{{13.10.3}{44}{Options\relax }{subsection.13.10.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.10.4}Files}{45}{subsection.13.10.4}} -\newlabel{man/fsck:files}{{13.10.4}{45}{Files\relax }{subsection.13.10.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.10.5}Exit Status}{45}{subsection.13.10.5}} -\newlabel{man/fsck:exit-status}{{13.10.5}{45}{Exit Status\relax }{subsection.13.10.5}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.10.6}See Also}{45}{subsection.13.10.6}} -\newlabel{man/fsck:see-also}{{13.10.6}{45}{See Also\relax }{subsection.13.10.6}{}} -\@writefile{toc}{\contentsline {section}{\numberline {13.11}The \textbf {pcp} command}{45}{section.13.11}} -\newlabel{man/pcp:the-pcp-command}{{13.11}{45}{The \textbf {pcp} command\relax }{section.13.11}{}} -\newlabel{man/pcp::doc}{{13.11}{45}{The \textbf {pcp} command\relax }{section.13.11}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.11.1}Synopsis}{45}{subsection.13.11.1}} -\newlabel{man/pcp:synopsis}{{13.11.1}{45}{Synopsis\relax }{subsection.13.11.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.11.2}Description}{45}{subsection.13.11.2}} -\newlabel{man/pcp:description}{{13.11.2}{45}{Description\relax }{subsection.13.11.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.11.3}Options}{45}{subsection.13.11.3}} -\newlabel{man/pcp:options}{{13.11.3}{45}{Options\relax }{subsection.13.11.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.11.4}Exit Status}{45}{subsection.13.11.4}} -\newlabel{man/pcp:exit-status}{{13.11.4}{45}{Exit Status\relax }{subsection.13.11.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.11.5}See Also}{45}{subsection.13.11.5}} -\newlabel{man/pcp:see-also}{{13.11.5}{45}{See Also\relax }{subsection.13.11.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {13.12}The \textbf {expire\_backups} command}{46}{section.13.12}} -\newlabel{man/expire_backups::doc}{{13.12}{46}{The \textbf {expire\_backups} command\relax }{section.13.12}{}} -\newlabel{man/expire_backups:the-expire-backups-command}{{13.12}{46}{The \textbf {expire\_backups} command\relax }{section.13.12}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.12.1}Synopsis}{46}{subsection.13.12.1}} -\newlabel{man/expire_backups:synopsis}{{13.12.1}{46}{Synopsis\relax }{subsection.13.12.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.12.2}Description}{46}{subsection.13.12.2}} -\newlabel{man/expire_backups:description}{{13.12.2}{46}{Description\relax }{subsection.13.12.2}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.12.3}Options}{47}{subsection.13.12.3}} -\newlabel{man/expire_backups:options}{{13.12.3}{47}{Options\relax }{subsection.13.12.3}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.12.4}Exit Status}{47}{subsection.13.12.4}} -\newlabel{man/expire_backups:exit-status}{{13.12.4}{47}{Exit Status\relax }{subsection.13.12.4}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {13.12.5}See Also}{47}{subsection.13.12.5}} -\newlabel{man/expire_backups:see-also}{{13.12.5}{47}{See Also\relax }{subsection.13.12.5}{}} -\@writefile{toc}{\contentsline {chapter}{\numberline {14}Further Resources / Getting Help}{49}{chapter.14}} +\newlabel{issues:known-issues}{{13}{35}{Known Issues\relax }{chapter.13}{}} +\newlabel{issues::doc}{{13}{35}{Known Issues\relax }{chapter.13}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {14}Manpages}{37}{chapter.14}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{resources::doc}{{14}{49}{Further Resources / Getting Help\relax }{chapter.14}{}} -\newlabel{resources:further-resources-getting-help}{{14}{49}{Further Resources / Getting Help\relax }{chapter.14}{}} -\newlabel{resources:resources}{{14}{49}{Further Resources / Getting Help\relax }{chapter.14}{}} +\newlabel{man/index:manpages}{{14}{37}{Manpages\relax }{chapter.14}{}} +\newlabel{man/index::doc}{{14}{37}{Manpages\relax }{chapter.14}{}} +\@writefile{toc}{\contentsline {section}{\numberline {14.1}The \textbf {mkfs.s3ql} command}{37}{section.14.1}} +\newlabel{man/mkfs:the-mkfs-s3ql-command}{{14.1}{37}{The \textbf {mkfs.s3ql} command\relax }{section.14.1}{}} +\newlabel{man/mkfs::doc}{{14.1}{37}{The \textbf {mkfs.s3ql} command\relax }{section.14.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.1.1}Synopsis}{37}{subsection.14.1.1}} +\newlabel{man/mkfs:synopsis}{{14.1.1}{37}{Synopsis\relax }{subsection.14.1.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.1.2}Description}{37}{subsection.14.1.2}} +\newlabel{man/mkfs:description}{{14.1.2}{37}{Description\relax }{subsection.14.1.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.1.3}Options}{37}{subsection.14.1.3}} +\newlabel{man/mkfs:options}{{14.1.3}{37}{Options\relax }{subsection.14.1.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.1.4}Exit Status}{38}{subsection.14.1.4}} +\newlabel{man/mkfs:exit-status}{{14.1.4}{38}{Exit Status\relax }{subsection.14.1.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.1.5}See Also}{38}{subsection.14.1.5}} +\newlabel{man/mkfs:see-also}{{14.1.5}{38}{See Also\relax }{subsection.14.1.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {14.2}The \textbf {s3qladm} command}{38}{section.14.2}} +\newlabel{man/adm::doc}{{14.2}{38}{The \textbf {s3qladm} command\relax }{section.14.2}{}} +\newlabel{man/adm:the-s3qladm-command}{{14.2}{38}{The \textbf {s3qladm} command\relax }{section.14.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.2.1}Synopsis}{38}{subsection.14.2.1}} +\newlabel{man/adm:synopsis}{{14.2.1}{38}{Synopsis\relax }{subsection.14.2.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.2.2}Description}{38}{subsection.14.2.2}} +\newlabel{man/adm:description}{{14.2.2}{38}{Description\relax }{subsection.14.2.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.2.3}Options}{38}{subsection.14.2.3}} +\newlabel{man/adm:options}{{14.2.3}{38}{Options\relax }{subsection.14.2.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.2.4}Actions}{39}{subsection.14.2.4}} +\newlabel{man/adm:actions}{{14.2.4}{39}{Actions\relax }{subsection.14.2.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.2.5}Exit Status}{39}{subsection.14.2.5}} +\newlabel{man/adm:exit-status}{{14.2.5}{39}{Exit Status\relax }{subsection.14.2.5}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.2.6}See Also}{39}{subsection.14.2.6}} +\newlabel{man/adm:see-also}{{14.2.6}{39}{See Also\relax }{subsection.14.2.6}{}} +\@writefile{toc}{\contentsline {section}{\numberline {14.3}The \textbf {mount.s3ql} command}{39}{section.14.3}} +\newlabel{man/mount::doc}{{14.3}{39}{The \textbf {mount.s3ql} command\relax }{section.14.3}{}} +\newlabel{man/mount:the-mount-s3ql-command}{{14.3}{39}{The \textbf {mount.s3ql} command\relax }{section.14.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.3.1}Synopsis}{39}{subsection.14.3.1}} +\newlabel{man/mount:synopsis}{{14.3.1}{39}{Synopsis\relax }{subsection.14.3.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.3.2}Description}{39}{subsection.14.3.2}} +\newlabel{man/mount:description}{{14.3.2}{39}{Description\relax }{subsection.14.3.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.3.3}Options}{39}{subsection.14.3.3}} +\newlabel{man/mount:options}{{14.3.3}{39}{Options\relax }{subsection.14.3.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.3.4}Exit Status}{40}{subsection.14.3.4}} +\newlabel{man/mount:exit-status}{{14.3.4}{40}{Exit Status\relax }{subsection.14.3.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.3.5}See Also}{40}{subsection.14.3.5}} +\newlabel{man/mount:see-also}{{14.3.5}{40}{See Also\relax }{subsection.14.3.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {14.4}The \textbf {s3qlstat} command}{41}{section.14.4}} +\newlabel{man/stat:the-s3qlstat-command}{{14.4}{41}{The \textbf {s3qlstat} command\relax }{section.14.4}{}} +\newlabel{man/stat::doc}{{14.4}{41}{The \textbf {s3qlstat} command\relax }{section.14.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.4.1}Synopsis}{41}{subsection.14.4.1}} +\newlabel{man/stat:synopsis}{{14.4.1}{41}{Synopsis\relax }{subsection.14.4.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.4.2}Description}{41}{subsection.14.4.2}} +\newlabel{man/stat:description}{{14.4.2}{41}{Description\relax }{subsection.14.4.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.4.3}Options}{41}{subsection.14.4.3}} +\newlabel{man/stat:options}{{14.4.3}{41}{Options\relax }{subsection.14.4.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.4.4}Exit Status}{41}{subsection.14.4.4}} +\newlabel{man/stat:exit-status}{{14.4.4}{41}{Exit Status\relax }{subsection.14.4.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.4.5}See Also}{41}{subsection.14.4.5}} +\newlabel{man/stat:see-also}{{14.4.5}{41}{See Also\relax }{subsection.14.4.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {14.5}The \textbf {s3qlctrl} command}{41}{section.14.5}} +\newlabel{man/ctrl:the-s3qlctrl-command}{{14.5}{41}{The \textbf {s3qlctrl} command\relax }{section.14.5}{}} +\newlabel{man/ctrl::doc}{{14.5}{41}{The \textbf {s3qlctrl} command\relax }{section.14.5}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.5.1}Synopsis}{41}{subsection.14.5.1}} +\newlabel{man/ctrl:synopsis}{{14.5.1}{41}{Synopsis\relax }{subsection.14.5.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.5.2}Description}{41}{subsection.14.5.2}} +\newlabel{man/ctrl:description}{{14.5.2}{41}{Description\relax }{subsection.14.5.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.5.3}Options}{42}{subsection.14.5.3}} +\newlabel{man/ctrl:options}{{14.5.3}{42}{Options\relax }{subsection.14.5.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.5.4}Exit Status}{42}{subsection.14.5.4}} +\newlabel{man/ctrl:exit-status}{{14.5.4}{42}{Exit Status\relax }{subsection.14.5.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.5.5}See Also}{42}{subsection.14.5.5}} +\newlabel{man/ctrl:see-also}{{14.5.5}{42}{See Also\relax }{subsection.14.5.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {14.6}The \textbf {s3qlcp} command}{42}{section.14.6}} +\newlabel{man/cp:the-s3qlcp-command}{{14.6}{42}{The \textbf {s3qlcp} command\relax }{section.14.6}{}} +\newlabel{man/cp::doc}{{14.6}{42}{The \textbf {s3qlcp} command\relax }{section.14.6}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.6.1}Synopsis}{42}{subsection.14.6.1}} +\newlabel{man/cp:synopsis}{{14.6.1}{42}{Synopsis\relax }{subsection.14.6.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.6.2}Description}{42}{subsection.14.6.2}} +\newlabel{man/cp:description}{{14.6.2}{42}{Description\relax }{subsection.14.6.2}{}} +\@writefile{toc}{\contentsline {subsubsection}{Snapshotting vs Hardlinking}{43}{subsubsection*.3}} +\newlabel{man/cp:snapshotting-vs-hardlinking}{{14.6.2}{43}{Snapshotting vs Hardlinking\relax }{subsubsection*.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.6.3}Options}{43}{subsection.14.6.3}} +\newlabel{man/cp:options}{{14.6.3}{43}{Options\relax }{subsection.14.6.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.6.4}Exit Status}{43}{subsection.14.6.4}} +\newlabel{man/cp:exit-status}{{14.6.4}{43}{Exit Status\relax }{subsection.14.6.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.6.5}See Also}{43}{subsection.14.6.5}} +\newlabel{man/cp:see-also}{{14.6.5}{43}{See Also\relax }{subsection.14.6.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {14.7}The \textbf {s3qlrm} command}{44}{section.14.7}} +\newlabel{man/rm::doc}{{14.7}{44}{The \textbf {s3qlrm} command\relax }{section.14.7}{}} +\newlabel{man/rm:the-s3qlrm-command}{{14.7}{44}{The \textbf {s3qlrm} command\relax }{section.14.7}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.7.1}Synopsis}{44}{subsection.14.7.1}} +\newlabel{man/rm:synopsis}{{14.7.1}{44}{Synopsis\relax }{subsection.14.7.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.7.2}Description}{44}{subsection.14.7.2}} +\newlabel{man/rm:description}{{14.7.2}{44}{Description\relax }{subsection.14.7.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.7.3}Options}{44}{subsection.14.7.3}} +\newlabel{man/rm:options}{{14.7.3}{44}{Options\relax }{subsection.14.7.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.7.4}Exit Status}{44}{subsection.14.7.4}} +\newlabel{man/rm:exit-status}{{14.7.4}{44}{Exit Status\relax }{subsection.14.7.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.7.5}See Also}{44}{subsection.14.7.5}} +\newlabel{man/rm:see-also}{{14.7.5}{44}{See Also\relax }{subsection.14.7.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {14.8}The \textbf {s3qllock} command}{44}{section.14.8}} +\newlabel{man/lock:the-s3qllock-command}{{14.8}{44}{The \textbf {s3qllock} command\relax }{section.14.8}{}} +\newlabel{man/lock::doc}{{14.8}{44}{The \textbf {s3qllock} command\relax }{section.14.8}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.8.1}Synopsis}{44}{subsection.14.8.1}} +\newlabel{man/lock:synopsis}{{14.8.1}{44}{Synopsis\relax }{subsection.14.8.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.8.2}Description}{45}{subsection.14.8.2}} +\newlabel{man/lock:description}{{14.8.2}{45}{Description\relax }{subsection.14.8.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.8.3}Rationale}{45}{subsection.14.8.3}} +\newlabel{man/lock:rationale}{{14.8.3}{45}{Rationale\relax }{subsection.14.8.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.8.4}Options}{45}{subsection.14.8.4}} +\newlabel{man/lock:options}{{14.8.4}{45}{Options\relax }{subsection.14.8.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.8.5}Exit Status}{45}{subsection.14.8.5}} +\newlabel{man/lock:exit-status}{{14.8.5}{45}{Exit Status\relax }{subsection.14.8.5}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.8.6}See Also}{45}{subsection.14.8.6}} +\newlabel{man/lock:see-also}{{14.8.6}{45}{See Also\relax }{subsection.14.8.6}{}} +\@writefile{toc}{\contentsline {section}{\numberline {14.9}The \textbf {umount.s3ql} command}{46}{section.14.9}} +\newlabel{man/umount::doc}{{14.9}{46}{The \textbf {umount.s3ql} command\relax }{section.14.9}{}} +\newlabel{man/umount:the-umount-s3ql-command}{{14.9}{46}{The \textbf {umount.s3ql} command\relax }{section.14.9}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.9.1}Synopsis}{46}{subsection.14.9.1}} +\newlabel{man/umount:synopsis}{{14.9.1}{46}{Synopsis\relax }{subsection.14.9.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.9.2}Description}{46}{subsection.14.9.2}} +\newlabel{man/umount:description}{{14.9.2}{46}{Description\relax }{subsection.14.9.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.9.3}Options}{46}{subsection.14.9.3}} +\newlabel{man/umount:options}{{14.9.3}{46}{Options\relax }{subsection.14.9.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.9.4}Exit Status}{46}{subsection.14.9.4}} +\newlabel{man/umount:exit-status}{{14.9.4}{46}{Exit Status\relax }{subsection.14.9.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.9.5}See Also}{46}{subsection.14.9.5}} +\newlabel{man/umount:see-also}{{14.9.5}{46}{See Also\relax }{subsection.14.9.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {14.10}The \textbf {fsck.s3ql} command}{46}{section.14.10}} +\newlabel{man/fsck::doc}{{14.10}{46}{The \textbf {fsck.s3ql} command\relax }{section.14.10}{}} +\newlabel{man/fsck:the-fsck-s3ql-command}{{14.10}{46}{The \textbf {fsck.s3ql} command\relax }{section.14.10}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.10.1}Synopsis}{46}{subsection.14.10.1}} +\newlabel{man/fsck:synopsis}{{14.10.1}{46}{Synopsis\relax }{subsection.14.10.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.10.2}Description}{47}{subsection.14.10.2}} +\newlabel{man/fsck:description}{{14.10.2}{47}{Description\relax }{subsection.14.10.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.10.3}Options}{47}{subsection.14.10.3}} +\newlabel{man/fsck:options}{{14.10.3}{47}{Options\relax }{subsection.14.10.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.10.4}Exit Status}{47}{subsection.14.10.4}} +\newlabel{man/fsck:exit-status}{{14.10.4}{47}{Exit Status\relax }{subsection.14.10.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.10.5}See Also}{47}{subsection.14.10.5}} +\newlabel{man/fsck:see-also}{{14.10.5}{47}{See Also\relax }{subsection.14.10.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {14.11}The \textbf {pcp} command}{47}{section.14.11}} +\newlabel{man/pcp:the-pcp-command}{{14.11}{47}{The \textbf {pcp} command\relax }{section.14.11}{}} +\newlabel{man/pcp::doc}{{14.11}{47}{The \textbf {pcp} command\relax }{section.14.11}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.11.1}Synopsis}{47}{subsection.14.11.1}} +\newlabel{man/pcp:synopsis}{{14.11.1}{47}{Synopsis\relax }{subsection.14.11.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.11.2}Description}{48}{subsection.14.11.2}} +\newlabel{man/pcp:description}{{14.11.2}{48}{Description\relax }{subsection.14.11.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.11.3}Options}{48}{subsection.14.11.3}} +\newlabel{man/pcp:options}{{14.11.3}{48}{Options\relax }{subsection.14.11.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.11.4}Exit Status}{48}{subsection.14.11.4}} +\newlabel{man/pcp:exit-status}{{14.11.4}{48}{Exit Status\relax }{subsection.14.11.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.11.5}See Also}{48}{subsection.14.11.5}} +\newlabel{man/pcp:see-also}{{14.11.5}{48}{See Also\relax }{subsection.14.11.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {14.12}The \textbf {expire\_backups} command}{48}{section.14.12}} +\newlabel{man/expire_backups::doc}{{14.12}{48}{The \textbf {expire\_backups} command\relax }{section.14.12}{}} +\newlabel{man/expire_backups:the-expire-backups-command}{{14.12}{48}{The \textbf {expire\_backups} command\relax }{section.14.12}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.12.1}Synopsis}{48}{subsection.14.12.1}} +\newlabel{man/expire_backups:synopsis}{{14.12.1}{48}{Synopsis\relax }{subsection.14.12.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.12.2}Description}{48}{subsection.14.12.2}} +\newlabel{man/expire_backups:description}{{14.12.2}{48}{Description\relax }{subsection.14.12.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.12.3}Options}{49}{subsection.14.12.3}} +\newlabel{man/expire_backups:options}{{14.12.3}{49}{Options\relax }{subsection.14.12.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.12.4}Exit Status}{49}{subsection.14.12.4}} +\newlabel{man/expire_backups:exit-status}{{14.12.4}{49}{Exit Status\relax }{subsection.14.12.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {14.12.5}See Also}{49}{subsection.14.12.5}} +\newlabel{man/expire_backups:see-also}{{14.12.5}{49}{See Also\relax }{subsection.14.12.5}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {15}Further Resources / Getting Help}{51}{chapter.15}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{resources::doc}{{15}{51}{Further Resources / Getting Help\relax }{chapter.15}{}} +\newlabel{resources:further-resources-getting-help}{{15}{51}{Further Resources / Getting Help\relax }{chapter.15}{}} +\newlabel{resources:resources}{{15}{51}{Further Resources / Getting Help\relax }{chapter.15}{}} diff --git a/doc/latex/manual.log b/doc/latex/manual.log index 20029d7..d97b674 100644 --- a/doc/latex/manual.log +++ b/doc/latex/manual.log @@ -1,4 +1,4 @@ -This is pdfTeX, Version 3.1415926-1.40.10 (TeX Live 2009/Debian) (format=pdflatex 2011.4.17) 20 MAY 2011 12:20 +This is pdfTeX, Version 3.1415926-1.40.10 (TeX Live 2009/Debian) (format=pdflatex 2011.9.6) 28 SEP 2011 11:35 entering extended mode %&-line parsing enabled. **manual.tex @@ -793,17 +793,13 @@ p/pdftex.map): fontmap entry for `ugqbo8r' already exists, duplicates ignored ] (./manual.toc LaTeX Font Info: Font shape `T1/ptm/bx/n' in size <10> not available (Font) Font shape `T1/ptm/b/n' tried instead on input line 2. -LaTeX Font Info: Try loading font information for T1+pcr on input line 10. -(/usr/share/texmf-texlive/tex/latex/psnfss/t1pcr.fd -File: t1pcr.fd 2001/06/04 font definitions for T1/pcr. -) LaTeX Font Info: Font shape `T1/phv/bx/n' in size <10> not available -(Font) Font shape `T1/phv/b/n' tried instead on input line 38. +(Font) Font shape `T1/phv/b/n' tried instead on input line 40. pdfTeX warning (ext4): destination with the same identifier (name{page.i}) has been already used, duplicate ignored <to be read again> \relax -l.38 ...ine {9}Checking for Errors}{23}{chapter.9} +l.40 ...{\numberline {9}Unmounting}{25}{chapter.9} [1 ]) @@ -819,7 +815,7 @@ l.112 \tableofcontents Chapter 1. LaTeX Font Info: Font shape `T1/phv/bx/n' in size <14.4> not available (Font) Font shape `T1/phv/b/n' tried instead on input line 117. -LaTeX Font Info: Try loading font information for TS1+ptm on input line 140. +LaTeX Font Info: Try loading font information for TS1+ptm on input line 139. (/usr/share/texmf-texlive/tex/latex/psnfss/ts1ptm.fd File: ts1ptm.fd 2001/06/04 font definitions for TS1/ptm. @@ -828,107 +824,158 @@ File: ts1ptm.fd 2001/06/04 font definitions for TS1/ptm. ] [2] Chapter 2. -[3 +LaTeX Font Info: Try loading font information for T1+pcr on input line 241. +(/usr/share/texmf-texlive/tex/latex/psnfss/t1pcr.fd +File: t1pcr.fd 2001/06/04 font definitions for T1/pcr. +) [3 ] [4] Chapter 3. +LaTeX Font Info: Font shape `T1/pcr/m/it' in size <9> not available +(Font) Font shape `T1/pcr/m/sl' tried instead on input line 368. +[5 -Underfull \hbox (badness 10000) in paragraph at lines 346--347 +] +Underfull \hbox (badness 10000) in paragraph at lines 413--414 [] -[5 +[6] [7] [8 ] -LaTeX Font Info: Font shape `T1/pcr/bx/n' in size <14.4> not available -(Font) Font shape `T1/pcr/b/n' tried instead on input line 402. -LaTeX Font Info: Font shape `T1/phv/bx/n' in size <12> not available -(Font) Font shape `T1/phv/b/n' tried instead on input line 462. -[6] [7] -LaTeX Font Info: Font shape `T1/pcr/m/it' in size <9> not available -(Font) Font shape `T1/pcr/m/sl' tried instead on input line 607. -[8] Chapter 4. -[9 - -] [10 +[9] +LaTeX Font Info: Font shape `T1/phv/bx/n' in size <12> not available +(Font) Font shape `T1/phv/b/n' tried instead on input line 548. +[10] [11] [12 ] Chapter 5. -[11] -Underfull \hbox (badness 10000) in paragraph at lines 777--778 +Underfull \hbox (badness 10000) in paragraph at lines 684--686 +[]\T1/ptm/m/n/10 Read au-then-ti-ca-tion cre-den-tials from this file (de-fault +: [] -[12] +[13] [14 + +] Chapter 6. -[13 -] [14] [15] [16] +Underfull \hbox (badness 10000) in paragraph at lines 742--744 +[]\T1/ptm/m/n/10 Read au-then-ti-ca-tion cre-den-tials from this file (de-fault +: + [] + +[15] +Underfull \hbox (badness 10000) in paragraph at lines 817--818 + + [] + +[16] Chapter 7. -[17 -] [18] [19] [20 +Underfull \hbox (badness 10000) in paragraph at lines 847--849 +[]\T1/ptm/m/n/10 Read au-then-ti-ca-tion cre-den-tials from this file (de-fault +: + [] -] +[17 + +] [18] [19] [20] Chapter 8. -[21] [22 +[21 + +] [22] [23] [24 ] Chapter 9. -[23] [24 +[25] [26 ] Chapter 10. -[25] -Underfull \hbox (badness 10000) in paragraph at lines 1439--1444 + +Underfull \hbox (badness 10000) in paragraph at lines 1312--1314 +[]\T1/ptm/m/n/10 Read au-then-ti-ca-tion cre-den-tials from this file (de-fault +: + [] + +[27] [28 + +] +Chapter 11. +[29] +Underfull \hbox (badness 10000) in paragraph at lines 1451--1456 []\T1/ptm/b/n/10 expire_backups \T1/ptm/m/n/10 us-age is sim-ple. It re-quires back-ups to have names of the forms [] -[26] [27] [28 +[30] [31] [32 ] -Chapter 11. -[29] [30] Chapter 12. -[31 - -] [32] +[33] [34] Chapter 13. -[33 +[35 + +] [36] +Chapter 14. -] [34] [35] [36] [37] [38] [39] [40] [41] [42] [43] [44] [45] -Underfull \hbox (badness 10000) in paragraph at lines 2672--2677 +Underfull \hbox (badness 10000) in paragraph at lines 1738--1740 +[]\T1/ptm/m/n/10 Read au-then-ti-ca-tion cre-den-tials from this file (de-fault +: + [] + +[37 + +] +Underfull \hbox (badness 10000) in paragraph at lines 1815--1817 +[]\T1/ptm/m/n/10 Read au-then-ti-ca-tion cre-den-tials from this file (de-fault +: + [] + +[38] +Underfull \hbox (badness 10000) in paragraph at lines 1894--1896 +[]\T1/ptm/m/n/10 Read au-then-ti-ca-tion cre-den-tials from this file (de-fault +: + [] + +[39] [40] [41] [42] [43] [44] [45] [46] +Underfull \hbox (badness 10000) in paragraph at lines 2454--2456 +[]\T1/ptm/m/n/10 Read au-then-ti-ca-tion cre-den-tials from this file (de-fault +: + [] + +[47] [48] +Underfull \hbox (badness 10000) in paragraph at lines 2597--2602 []\T1/ptm/b/n/10 expire_backups \T1/ptm/m/n/10 us-age is sim-ple. It re-quires back-ups to have names of the forms [] -[46] [47] [48 +[49] [50 ] -Chapter 14. +Chapter 15. No file manual.ind. -[49] (./manual.aux) ) +[51] (./manual.aux) ) Here is how much of TeX's memory you used: - 8345 strings out of 495021 - 113597 string characters out of 1181035 - 200143 words of memory out of 3000000 - 11122 multiletter control sequences out of 15000+50000 - 59136 words of font info for 67 fonts, out of 3000000 for 9000 + 8325 strings out of 495021 + 113261 string characters out of 1181036 + 199733 words of memory out of 3000000 + 11114 multiletter control sequences out of 15000+50000 + 58515 words of font info for 65 fonts, out of 3000000 for 9000 29 hyphenation exceptions out of 8191 - 45i,12n,48p,275b,492s stack positions out of 5000i,500n,10000p,200000b,50000s + 45i,12n,48p,278b,492s stack positions out of 5000i,500n,10000p,200000b,50000s {/usr/share/texmf-texlive/fonts/enc/dvips/base/8r.enc}</us -r/share/texmf-texlive/fonts/type1/urw/courier/ucrb8a.pfb></usr/share/texmf-texl -ive/fonts/type1/urw/courier/ucrr8a.pfb></usr/share/texmf-texlive/fonts/type1/ur -w/courier/ucrro8a.pfb></usr/share/texmf-texlive/fonts/type1/urw/helvetic/uhvb8a -.pfb></usr/share/texmf-texlive/fonts/type1/urw/helvetic/uhvbo8a.pfb></usr/share -/texmf-texlive/fonts/type1/urw/times/utmb8a.pfb></usr/share/texmf-texlive/fonts -/type1/urw/times/utmr8a.pfb></usr/share/texmf-texlive/fonts/type1/urw/times/utm -ri8a.pfb> -Output written on manual.pdf (53 pages, 289155 bytes). +r/share/texmf-texlive/fonts/type1/urw/courier/ucrr8a.pfb></usr/share/texmf-texl +ive/fonts/type1/urw/courier/ucrro8a.pfb></usr/share/texmf-texlive/fonts/type1/u +rw/helvetic/uhvb8a.pfb></usr/share/texmf-texlive/fonts/type1/urw/helvetic/uhvbo +8a.pfb></usr/share/texmf-texlive/fonts/type1/urw/times/utmb8a.pfb></usr/share/t +exmf-texlive/fonts/type1/urw/times/utmr8a.pfb></usr/share/texmf-texlive/fonts/t +ype1/urw/times/utmri8a.pfb> +Output written on manual.pdf (55 pages, 286628 bytes). PDF statistics: - 781 PDF objects out of 1000 (max. 8388607) - 224 named destinations out of 1000 (max. 500000) - 465 words of extra memory for PDF output out of 10000 (max. 10000000) + 804 PDF objects out of 1000 (max. 8388607) + 214 named destinations out of 1000 (max. 500000) + 489 words of extra memory for PDF output out of 10000 (max. 10000000) diff --git a/doc/latex/manual.out b/doc/latex/manual.out index 16ea0d1..e1db91b 100644 --- a/doc/latex/manual.out +++ b/doc/latex/manual.out @@ -4,55 +4,58 @@ \BOOKMARK [0][-]{chapter.2}{Installation}{} \BOOKMARK [1][-]{section.2.1}{Dependencies}{chapter.2} \BOOKMARK [1][-]{section.2.2}{Installing S3QL}{chapter.2} -\BOOKMARK [0][-]{chapter.3}{Storage Backends}{} -\BOOKMARK [1][-]{section.3.1}{On Backend Reliability}{chapter.3} -\BOOKMARK [1][-]{section.3.2}{The authinfo file}{chapter.3} -\BOOKMARK [1][-]{section.3.3}{Consistency Guarantees}{chapter.3} -\BOOKMARK [1][-]{section.3.4}{The Amazon S3 Backend}{chapter.3} -\BOOKMARK [1][-]{section.3.5}{The Local Backend}{chapter.3} -\BOOKMARK [1][-]{section.3.6}{The SFTP Backend}{chapter.3} -\BOOKMARK [0][-]{chapter.4}{File System Creation}{} -\BOOKMARK [0][-]{chapter.5}{Managing Buckets}{} -\BOOKMARK [1][-]{section.5.1}{Changing the Passphrase}{chapter.5} -\BOOKMARK [1][-]{section.5.2}{Upgrading the file system}{chapter.5} -\BOOKMARK [1][-]{section.5.3}{Deleting a file system}{chapter.5} -\BOOKMARK [1][-]{section.5.4}{Restoring Metadata Backups}{chapter.5} -\BOOKMARK [0][-]{chapter.6}{Mounting}{} -\BOOKMARK [1][-]{section.6.1}{Storing Encryption Passwords}{chapter.6} -\BOOKMARK [1][-]{section.6.2}{Compression Algorithms}{chapter.6} -\BOOKMARK [1][-]{section.6.3}{Parallel Compression}{chapter.6} -\BOOKMARK [1][-]{section.6.4}{Notes about Caching}{chapter.6} -\BOOKMARK [1][-]{section.6.5}{Automatic Mounting}{chapter.6} -\BOOKMARK [0][-]{chapter.7}{Advanced S3QL Features}{} -\BOOKMARK [1][-]{section.7.1}{Snapshotting and Copy-on-Write}{chapter.7} -\BOOKMARK [1][-]{section.7.2}{Getting Statistics}{chapter.7} -\BOOKMARK [1][-]{section.7.3}{Immutable Trees}{chapter.7} -\BOOKMARK [1][-]{section.7.4}{Fast Recursive Removal}{chapter.7} -\BOOKMARK [1][-]{section.7.5}{Runtime Configuration}{chapter.7} -\BOOKMARK [0][-]{chapter.8}{Unmounting}{} -\BOOKMARK [0][-]{chapter.9}{Checking for Errors}{} -\BOOKMARK [0][-]{chapter.10}{Contributed Programs}{} -\BOOKMARK [1][-]{section.10.1}{benchmark.py}{chapter.10} -\BOOKMARK [1][-]{section.10.2}{s3\137copy.py}{chapter.10} -\BOOKMARK [1][-]{section.10.3}{pcp.py}{chapter.10} -\BOOKMARK [1][-]{section.10.4}{s3\137backup.sh}{chapter.10} -\BOOKMARK [1][-]{section.10.5}{expire\137backups.py}{chapter.10} -\BOOKMARK [1][-]{section.10.6}{s3ql.conf}{chapter.10} -\BOOKMARK [0][-]{chapter.11}{Tips \046 Tricks}{} -\BOOKMARK [1][-]{section.11.1}{Permanently mounted backup file system}{chapter.11} -\BOOKMARK [1][-]{section.11.2}{Improving copy performance}{chapter.11} -\BOOKMARK [0][-]{chapter.12}{Known Issues}{} -\BOOKMARK [0][-]{chapter.13}{Manpages}{} -\BOOKMARK [1][-]{section.13.1}{The mkfs.s3ql command}{chapter.13} -\BOOKMARK [1][-]{section.13.2}{The s3qladm command}{chapter.13} -\BOOKMARK [1][-]{section.13.3}{The mount.s3ql command}{chapter.13} -\BOOKMARK [1][-]{section.13.4}{The s3qlstat command}{chapter.13} -\BOOKMARK [1][-]{section.13.5}{The s3qlctrl command}{chapter.13} -\BOOKMARK [1][-]{section.13.6}{The s3qlcp command}{chapter.13} -\BOOKMARK [1][-]{section.13.7}{The s3qlrm command}{chapter.13} -\BOOKMARK [1][-]{section.13.8}{The s3qllock command}{chapter.13} -\BOOKMARK [1][-]{section.13.9}{The umount.s3ql command}{chapter.13} -\BOOKMARK [1][-]{section.13.10}{The fsck.s3ql command}{chapter.13} -\BOOKMARK [1][-]{section.13.11}{The pcp command}{chapter.13} -\BOOKMARK [1][-]{section.13.12}{The expire\137backups command}{chapter.13} -\BOOKMARK [0][-]{chapter.14}{Further Resources / Getting Help}{} +\BOOKMARK [0][-]{chapter.3}{General Information}{} +\BOOKMARK [1][-]{section.3.1}{Terminology}{chapter.3} +\BOOKMARK [1][-]{section.3.2}{Storing Authentication Information}{chapter.3} +\BOOKMARK [1][-]{section.3.3}{On Backend Reliability}{chapter.3} +\BOOKMARK [0][-]{chapter.4}{Storage Backends}{} +\BOOKMARK [1][-]{section.4.1}{Google Storage}{chapter.4} +\BOOKMARK [1][-]{section.4.2}{Amazon S3}{chapter.4} +\BOOKMARK [1][-]{section.4.3}{S3 compatible}{chapter.4} +\BOOKMARK [1][-]{section.4.4}{Local}{chapter.4} +\BOOKMARK [1][-]{section.4.5}{SSH/SFTP}{chapter.4} +\BOOKMARK [0][-]{chapter.5}{File System Creation}{} +\BOOKMARK [0][-]{chapter.6}{Managing Buckets}{} +\BOOKMARK [1][-]{section.6.1}{Changing the Passphrase}{chapter.6} +\BOOKMARK [1][-]{section.6.2}{Upgrading the file system}{chapter.6} +\BOOKMARK [1][-]{section.6.3}{Deleting a file system}{chapter.6} +\BOOKMARK [1][-]{section.6.4}{Restoring Metadata Backups}{chapter.6} +\BOOKMARK [0][-]{chapter.7}{Mounting}{} +\BOOKMARK [1][-]{section.7.1}{Compression Algorithms}{chapter.7} +\BOOKMARK [1][-]{section.7.2}{Parallel Compression}{chapter.7} +\BOOKMARK [1][-]{section.7.3}{Notes about Caching}{chapter.7} +\BOOKMARK [1][-]{section.7.4}{Automatic Mounting}{chapter.7} +\BOOKMARK [0][-]{chapter.8}{Advanced S3QL Features}{} +\BOOKMARK [1][-]{section.8.1}{Snapshotting and Copy-on-Write}{chapter.8} +\BOOKMARK [1][-]{section.8.2}{Getting Statistics}{chapter.8} +\BOOKMARK [1][-]{section.8.3}{Immutable Trees}{chapter.8} +\BOOKMARK [1][-]{section.8.4}{Fast Recursive Removal}{chapter.8} +\BOOKMARK [1][-]{section.8.5}{Runtime Configuration}{chapter.8} +\BOOKMARK [0][-]{chapter.9}{Unmounting}{} +\BOOKMARK [0][-]{chapter.10}{Checking for Errors}{} +\BOOKMARK [0][-]{chapter.11}{Contributed Programs}{} +\BOOKMARK [1][-]{section.11.1}{benchmark.py}{chapter.11} +\BOOKMARK [1][-]{section.11.2}{s3\137copy.py}{chapter.11} +\BOOKMARK [1][-]{section.11.3}{pcp.py}{chapter.11} +\BOOKMARK [1][-]{section.11.4}{s3\137backup.sh}{chapter.11} +\BOOKMARK [1][-]{section.11.5}{expire\137backups.py}{chapter.11} +\BOOKMARK [1][-]{section.11.6}{s3ql.conf}{chapter.11} +\BOOKMARK [0][-]{chapter.12}{Tips \046 Tricks}{} +\BOOKMARK [1][-]{section.12.1}{SSH Backend}{chapter.12} +\BOOKMARK [1][-]{section.12.2}{Permanently mounted backup file system}{chapter.12} +\BOOKMARK [1][-]{section.12.3}{Improving copy performance}{chapter.12} +\BOOKMARK [0][-]{chapter.13}{Known Issues}{} +\BOOKMARK [0][-]{chapter.14}{Manpages}{} +\BOOKMARK [1][-]{section.14.1}{The mkfs.s3ql command}{chapter.14} +\BOOKMARK [1][-]{section.14.2}{The s3qladm command}{chapter.14} +\BOOKMARK [1][-]{section.14.3}{The mount.s3ql command}{chapter.14} +\BOOKMARK [1][-]{section.14.4}{The s3qlstat command}{chapter.14} +\BOOKMARK [1][-]{section.14.5}{The s3qlctrl command}{chapter.14} +\BOOKMARK [1][-]{section.14.6}{The s3qlcp command}{chapter.14} +\BOOKMARK [1][-]{section.14.7}{The s3qlrm command}{chapter.14} +\BOOKMARK [1][-]{section.14.8}{The s3qllock command}{chapter.14} +\BOOKMARK [1][-]{section.14.9}{The umount.s3ql command}{chapter.14} +\BOOKMARK [1][-]{section.14.10}{The fsck.s3ql command}{chapter.14} +\BOOKMARK [1][-]{section.14.11}{The pcp command}{chapter.14} +\BOOKMARK [1][-]{section.14.12}{The expire\137backups command}{chapter.14} +\BOOKMARK [0][-]{chapter.15}{Further Resources / Getting Help}{} diff --git a/doc/latex/manual.tex b/doc/latex/manual.tex index fe2532b..250ef72 100644 --- a/doc/latex/manual.tex +++ b/doc/latex/manual.tex @@ -12,8 +12,8 @@ \title{S3QL Documentation} -\date{May 20, 2011} -\release{1.0.1} +\date{September 28, 2011} +\release{1.2} \author{Nikolaus Rath} \newcommand{\sphinxlogo}{} \renewcommand{\releasename}{Release} @@ -116,13 +116,12 @@ \chapter{About S3QL} \label{about:about-s3ql}\label{about::doc}\label{about:s3ql-user-s-guide} -S3QL is a file system that stores all its data online. It supports -\href{http://aws.amazon.com/s3AmazonS3}{Amazon S3} as well as arbitrary -SFTP servers and effectively provides you with a hard disk of dynamic, -infinite capacity that can be accessed from any computer with internet -access. +S3QL is a file system that stores all its data online using storage +services like \href{http://code.google.com/apis/storage/}{Google Storage}, \href{http://aws.amazon.com/s3AmazonS3}{Amazon S3} or \href{http://openstack.org/projects/storage/}{OpenStack}. S3QL effectively provides +a hard disk of dynamic, infinite capacity that can be accessed from +any computer with internet access. -S3QL is providing a standard, full featured UNIX file system that is +S3QL is a standard conforming, full featured UNIX file system that is conceptually indistinguishable from any local file system. Furthermore, S3QL has additional features like compression, encryption, data de-duplication, immutable trees and snapshotting @@ -232,14 +231,10 @@ already provides a suitable packages and only install from source if that is not the case. \begin{itemize} \item {} -Kernel version 2.6.9 or newer. Starting with kernel 2.6.26 -you will get significantly better write performance, so you should -actually use \emph{2.6.26 or newer whenever possible}. - -\item {} -The \href{http://fuse.sourceforge.net/}{FUSE Library} should already be -installed on your system. However, you have to make sure that you -have at least version 2.8.0. +Kernel: Linux 2.6.9 or newer or FreeBSD with \href{http://www.freshports.org/sysutils/fusefs-kmod/}{FUSE4BSD}. Starting with +kernel 2.6.26 you will get significantly better write performance, +so under Linux you should actually use \emph{2.6.26 or newer whenever +possible}. \item {} The \href{http://pypi.python.org/pypi/pycryptopp}{PyCrypto++ Python Module}. To check if this module @@ -278,12 +273,6 @@ Note that earlier S3QL versions shipped with a builtin version of this module. If you are upgrading from such a version, make sure to completely remove the old S3QL version first. -\item {} -If you want to use the SFTP backend, then you also need the -\href{http://www.lag.net/paramiko/}{Paramiko Python Module}. To check -if this module is installed, try to execute \code{python -c 'import -paramiko'}. - \end{itemize} @@ -321,17 +310,95 @@ setup.py install -{-}user}. In this case you should make sure that \end{itemize} -\chapter{Storage Backends} -\label{backends::doc}\label{backends:storage-backends} -S3QL can use different protocols to store the file system data. -Independent of the backend that you use, the place where your file -system data is being stored is called a \emph{bucket}. (This is mostly for -historical reasons, since initially S3QL supported only the Amazon S3 -backend). +\chapter{General Information} +\label{general:general-information}\label{general::doc} + +\section{Terminology} +\label{general:terminology} +S3QL can store data at different service providers and using different +protocols. The term \emph{backend} refers to both the part of S3QL that +implements communication with a specific storage service and the +storage service itself. Most backends can hold more than one S3QL file +system and thus require some additional information that specifies the +file system location within the backend. This location is called a +\emph{bucket} (for historical reasons). + +Many S3QL commands expect a \emph{storage url} as a parameter. A storage +url specifies both the backend and the bucket and thus uniquely +identifies an S3QL file system. The form of the storage url depends on +the backend and is described together with the +{\hyperref[backends:storage-backends]{\emph{Storage Backends}}}. + + +\section{Storing Authentication Information} +\label{general:storing-authentication-information}\label{general:bucket-pw} +Normally, S3QL reads username and password for the backend as well as +an encryption passphrase for the bucket from the terminal. Most +commands also accept an \code{-{-}authfile} parameter that can be +used to read this information from a file instead. + +The authentication file consists of sections, led by a \code{{[}section{]}} +header and followed by \code{name: value} entries. The section headers +themselves are not used by S3QL but have to be unique within the file. + +In each section, the following entries can be defined: +\begin{quote}\begin{description} +\item[{storage-url}] \leavevmode +Specifies the storage url to which this section applies. If a +storage url starts with the value of this entry, the section is +considered applicable. + +\item[{backend-login}] \leavevmode +Specifies the username to use for authentication with the backend. + +\item[{backend-password}] \leavevmode +Specifies the password to use for authentication with the backend. + +\item[{bucket-passphrase}] \leavevmode +Specifies the passphrase to use to decrypt the bucket (if it is +encrypted). + +\end{description}\end{quote} + +When reading the authentication file, S3QL considers every applicable +section in order and uses the last value that it found for each entry. +For example, consider the following authentication file: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{g+ge}{[s3]} +\PYG{l}{storage-url: s3://} +\PYG{l}{backend-login: joe} +\PYG{l}{backend-password: notquitesecret} + +\PYG{g+ge}{[bucket1]} +\PYG{l}{storage-url: s3://joes-first-bucket} +\PYG{l}{bucket-passphrase: neitheristhis} + +\PYG{g+ge}{[bucket2]} +\PYG{l}{storage-url: s3://joes-second-bucket} +\PYG{l}{bucket-passphrase: swordfish} + +\PYG{g+ge}{[bucket3]} +\PYG{l}{storage-url: s3://joes-second-bucket/with-prefix} +\PYG{l}{backend-login: bill} +\PYG{l}{backend-password: bi23ll} +\PYG{l}{bucket-passphrase: ll23bi} +\end{Verbatim} + +With this authentication file, S3QL would try to log in as ``joe'' +whenever the s3 backend is used, except when accessing a storage url +that begins with ``s3://joes-second-bucket/with-prefix''. In that case, +the last section becomes active and S3QL would use the ``bill'' +credentials. Furthermore, bucket encryption passphrases will be used +for storage urls that start with ``s3://joes-first-bucket'' or +``s3://joes-second-bucket''. + +The authentication file is parsed by the \href{http://docs.python.org/library/configparser.html}{Python ConfigParser +module}. \section{On Backend Reliability} -\label{backends:on-backend-reliability} +\label{general:backend-reliability}\label{general:on-backend-reliability} S3QL has been designed for use with a storage backend where data loss is so infrequent that it can be completely neglected (e.g. the Amazon S3 backend). If you decide to use a less reliable backend, you should @@ -394,229 +461,209 @@ a ``checkpoint'': data loss in the backend that occurred before running \code{fsck.s3ql} can not affect any file system operations performed after running \code{fsck.s3ql}. -Nevertheless, (as said at the beginning) the recommended way to use -S3QL is in combination with a sufficiently reliable storage backend. -In that case none of the above will ever be a concern. +Nevertheless, the recommended way to use S3QL is in combination with a +sufficiently reliable storage backend. In that case none of the above +will ever be a concern. + + +\chapter{Storage Backends} +\label{backends:id1}\label{backends::doc}\label{backends:storage-backends} +The following backends are currently available in S3QL: + +\section{Google Storage} +\label{backends:google-storage} +\href{http://code.google.com/apis/storage/}{Google Storage} is an online +storage service offered by Google. It is the most feature-rich service +supported by S3QL and S3QL offers the best performance when used with +the Google Storage backend. -\section{The \texttt{authinfo} file} -\label{backends:the-authinfo-file} -Most backends first try to read the file \code{\textasciitilde{}/.s3ql/authinfo} to determine -the username and password for connecting to the remote host. If this -fails, both username and password are read from the terminal. +To use the Google Storage backend, you need to have (or sign up for) a +Google account, and then \href{http://code.google.com/apis/storage/docs/signup.html}{activate Google Storage} for your +account. The account is free, you will pay only for the amount of +storage and traffic that you actually use. Once you have created the +account, make sure to \href{http://code.google.com/apis/storage/docs/reference/v1/apiversion1.html\#enabling}{activate legacy access}. -The \code{authinfo} file has to contain entries of the form +To create a Google Storage bucket, you can use e.g. the \href{https://sandbox.google.com/storage/}{Google +Storage Manager}. The +storage URL for accessing the bucket in S3QL is then \begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{backend }\PYG{n+nv}{\textless{}backend\textgreater{}}\PYG{l}{ machine }\PYG{n+nv}{\textless{}host\textgreater{}}\PYG{l}{ login }\PYG{n+nv}{\textless{}user\textgreater{}}\PYG{l}{ password }\PYG{n+nv}{\textless{}password\textgreater{}} +\PYG{l}{gs://}\PYG{n+nv}{\textless{}bucketname\textgreater{}}\PYG{l}{/}\PYG{n+nv}{\textless{}prefix\textgreater{}} \end{Verbatim} -So to use the login \code{joe} with password \code{jibbadup} when using the FTP -backend to connect to the host \code{backups.joesdomain.com}, you would -specify +Here \emph{bucketname} is the name of the bucket, and \emph{prefix} can be +an arbitrary prefix that will be prepended to all object names used by +S3QL. This allows you to store several S3QL file systems in the same +Google Storage bucket. + +Note that the backend login and password for accessing your Google +Storage bucket are not your Google account name and password, but the +\emph{Google Storage developer access key} and \emph{Google Storage developer +secret} that you can manage with the \href{https://code.google.com/apis/console/\#:storage:legacy}{Google Storage key management +tool}. + +If you would like S3QL to connect using HTTPS instead of standard +HTTP, start the storage url with \code{gss://} instead of \code{gs://}. Note +that at this point S3QL does not perform any server certificate +validation (see \href{http://code.google.com/p/s3ql/issues/detail?id=267}{issue 267}). + + +\section{Amazon S3} +\label{backends:amazon-s3} +\href{http://aws.amazon.com/s3}{Amazon S3} is the online storage service +offered by \href{http://aws.amazon.com/}{Amazon Web Services (AWS)}. To +use the S3 backend, you first need to sign up for an AWS account. The +account is free, you will pay only for the amount of storage and +traffic that you actually use. After that, you need to create a bucket +that will hold the S3QL file system, e.g. using the \href{https://console.aws.amazon.com/s3/home}{AWS Management +Console}. For best +performance, it is recommend to create the bucket in the +geographically closest storage region, but not the US Standard +region (see below). + +The storage URL for accessing S3 buckets in S3QL has the form \begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{backend ftp machine backups.joesdomain.com login joe password jibbadup} +\PYG{l}{s3://}\PYG{n+nv}{\textless{}bucketname\textgreater{}}\PYG{l}{/}\PYG{n+nv}{\textless{}prefix\textgreater{}} \end{Verbatim} +Here \emph{bucketname} is the name of the bucket, and \emph{prefix} can be +an arbitrary prefix that will be prepended to all object names used by +S3QL. This allows you to store several S3QL file systems in the same +S3 bucket. + +Note that the backend login and password for accessing S3 are not the +user id and password that you use to log into the Amazon Webpage, but +the \emph{AWS access key id} and \emph{AWS secret access key} shown under \href{https://aws-portal.amazon.com/gp/aws/developer/account/index.html?ie=UTF8\&action=access-key}{My +Account/Access Identifiers}. + +If you would like S3QL to connect using HTTPS instead of standard +HTTP, start the storage url with \code{s3s://} instead of \code{s3://}. Note +that, as of May 2011, Amazon S3 is faster when accessed using a +standard HTTP connection, and that S3QL does not perform any server +certificate validation (see \href{http://code.google.com/p/s3ql/issues/detail?id=267}{issue 267}). -\section{Consistency Guarantees} -\label{backends:consistency-guarantees} -The different backends provide different types of \emph{consistency -guarantees}. Informally, a consistency guarantee tells you how fast -the backend will apply changes to the stored data. -S3QL defines the following three levels: +\subsection{Reduced Redundancy Storage (RRS)} +\label{backends:reduced-redundancy-storage-rrs} +S3QL does not allow the use of \href{http://aws.amazon.com/s3/\#protecting}{reduced redundancy storage}. The reason for that is a +combination of three factors: \begin{itemize} \item {} -\textbf{Read-after-Write Consistency.} This is the strongest consistency -guarantee. If a backend offers read-after-write consistency, it -guarantees that as soon as you have committed any changes to the -backend, subsequent requests will take into account these changes. +RRS has a relatively low reliability, on average you loose one +out of every ten-thousand objects a year. So you can expect to +occasionally loose some data. \item {} -\textbf{Read-after-Create Consistency.} If a backend provides only -read-after-create consistency, only the creation of a new object is -guaranteed to be taken into account for subsequent requests. This -means that, for example, if you overwrite data in an existing -object, subsequent requests may still return the old data for a -certain period of time. +When \code{fsck.s3ql} asks S3 for a list of the stored objects, this list +includes even those objects that have been lost. Therefore +\code{fsck.s3ql} \emph{can not detect lost objects} and lost data will only +become apparent when you try to actually read from a file whose data +has been lost. This is a (very unfortunate) peculiarity of Amazon +S3. \item {} -\textbf{Eventual consistency.} This is the lowest consistency level. -Basically, any changes that you make to the backend may not be -visible for a certain amount of time after the change has been made. -However, you are guaranteed that no change will be lost. All changes -will \emph{eventually} become visible. - -. +Due to the data de-duplication feature of S3QL, unnoticed lost +objects may cause subsequent data loss later in time (see +{\hyperref[general:backend-reliability]{\emph{On Backend Reliability}}} for details). \end{itemize} -As long as your backend provides read-after-write or read-after-create -consistency, you do not have to worry about consistency guarantees at -all. However, if you plan to use a backend with only eventual -consistency, you have to be a bit careful in some situations. - - -\subsection{Dealing with Eventual Consistency} -\label{backends:dealing-with-eventual-consistency}\label{backends:eventual-consistency} -\begin{notice}{note}{Note:} -The following applies only to storage backends that do not provide -read-after-create or read-after-write consistency. Currently, -this is only the Amazon S3 backend \emph{if used with the US-Standard -storage region}. If you use a different storage backend, or the S3 -backend with a different storage region, this section does not apply -to you. -\end{notice} -While the file system is mounted, S3QL is able to automatically handle -all issues related to the weak eventual consistency guarantee. -However, some issues may arise during the mount process and when the -file system is checked. +\subsection{Potential issues when using the US Standard storage region} +\label{backends:potential-issues-when-using-the-us-standard-storage-region} +In the US Standard storage region, Amazon S3 does not guarantee read +after create consistency. This means that after a new object has been +stored, requests to read this object may still fail for a little +while. While the file system is mounted, S3QL is able to automatically +handle all issues related to this so-called eventual consistency. +However, problems may arise during the mount process and when the file +system is checked: Suppose that you mount the file system, store some new data, delete -some old data and unmount it again. Now remember that eventual -consistency means that there is no guarantee that these changes will -be visible immediately. At least in theory it is therefore possible -that if you mount the file system again, S3QL does not see any of the -changes that you have done and presents you an ``old version'' of the -file system without them. Even worse, if you notice the problem and -unmount the file system, S3QL will upload the old status (which S3QL -necessarily has to consider as current) and thereby permanently -override the newer version (even though this change may not become -immediately visible either). - -The same problem applies when checking the file system. If the backend +some old data and unmount it again. Now there is no guarantee that +these changes will be visible immediately. At least in theory it is +therefore possible that if you mount the file system again, S3QL +does not see any of the changes that you have done and presents you +an ``old version'' of the file system without them. Even worse, if you +notice the problem and unmount the file system, S3QL will upload the +old status (which S3QL necessarily has to consider as current) and +thereby permanently override the newer version (even though this +change may not become immediately visible either). + +The same problem applies when checking the file system. If S3 provides S3QL with only partially updated data, S3QL has no way to find out if this a real consistency problem that needs to be fixed or if it is only a temporary problem that will resolve itself automatically (because there are still changes that have not become visible yet). -While this may seem to be a rather big problem, the likelihood of it -to occur is rather low. In practice, most storage providers rarely -need more than a few seconds to apply incoming changes, so to trigger -this problem one would have to unmount and remount the file system in -a very short time window. Many people therefore make sure that they -wait a few minutes between successive mounts (or file system checks) -and decide that the remaining risk is negligible. - -Nevertheless, the eventual consistency guarantee does not impose an -upper limit on the time that it may take for change to become visible. -Therefore there is no ``totally safe'' waiting time that would totally -eliminate this problem; a theoretical possibility always remains. - - -\section{The Amazon S3 Backend} -\label{backends:the-amazon-s3-backend} -To store your file system in an Amazon S3 bucket, use a storage URL of -the form \code{s3://\textless{}bucketname\textgreater{}}. Bucket names must conform to the \href{http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/BucketRestrictions.html}{S3 -Bucket Name Restrictions}. - -The S3 backend offers exceptionally strong reliability guarantees. As -of August 2010, Amazon guarantees a durability of 99.999999999\% per -year. In other words, if you store a thousand million objects then on -average you would loose less than one object in a hundred years. - -The Amazon S3 backend provides read-after-create consistency for the -EU, Asia-Pacific and US-West storage regions. \emph{For the US-Standard -storage region, Amazon S3 provides only eventual consistency} (please -refer to {\hyperref[backends:eventual-consistency]{\emph{Dealing with Eventual Consistency}}} for information about -what this entails). - -When connecting to Amazon S3, S3QL uses an unencrypted HTTP -connection, so if you want your data to stay confidential, you have -to create the S3QL file system with encryption (this is also the default). - -When reading the authentication information for the S3 backend from -the \code{authinfo} file, the \code{host} field is ignored, i.e. the first entry -with \code{s3} as a backend will be used. For example - -\begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{backend s3 machine any login myAWSaccessKeyId password myAwsSecretAccessKey} -\end{Verbatim} - -Note that the bucket names come from a global pool, so chances are -that your favorite name has already been taken by another S3 user. -Usually a longer bucket name containing some random numbers, like -\code{19283712\_yourname\_s3ql}, will work better. - -If you do not already have one, you need to obtain an Amazon S3 -account from \href{http://aws.amazon.com/}{Amazon AWS}. The account is -free, you will pay only for the amount of storage that you actually -use. - -Note that the login and password for accessing S3 are not the user id -and password that you use to log into the Amazon Webpage, but the ``AWS -access key id'' and ``AWS secret access key'' shown under \href{https://aws-portal.amazon.com/gp/aws/developer/account/index.html?ie=UTF8\&action=access-key}{My -Account/Access Identifiers}. - -\begin{notice}{note}{Note:} -S3QL also allows you to use \href{http://aws.amazon.com/s3/\#protecting}{reduced redundancy storage} by using \code{s3rr://} -instead of \code{s3://} in the storage url. However, this not -recommended. The reason is a combination of three factors: -\begin{itemize} -\item {} -RRS has a relatively low reliability, on average you loose one -out of every ten-thousand objects a year. So you can expect to -occasionally loose some data. - -\item {} -When \code{fsck.s3ql} asks Amazon S3 for a list of the stored objects, -this list includes even those objects that have been lost. -Therefore \code{fsck.s3ql} \emph{can not detect lost objects} and lost data -will only become apparent when you try to actually read from a -file whose data has been lost. This is a (very unfortunate) -peculiarity of Amazon S3. +The likelihood of this to happen is rather low. In practice, most +objects are ready for retrieval just a few seconds after they have +been stored, so to trigger this problem one would have to unmount and +remount the file system in a very short time window. However, since S3 +does not place any upper limit on the length of this window, it is +recommended to not place S3QL buckets in the US Standard storage +region. As of May 2011, all other storage regions provide stronger +consistency guarantees that completely eliminate any of the described +problems. -\item {} -Due to the data de-duplication feature of S3QL, unnoticed lost -objects may cause subsequent data loss later in time (see {\hyperref[backends:on-backend-reliability]{On -Backend Reliability}} for details). -\end{itemize} +\section{S3 compatible} +\label{backends:s3-compatible} +S3QL is also able to access other, S3 compatible storage services for +which no specific backend exists. Note that when accessing such +services, only the lowest common denominator of available features can +be used, so it is generally recommended to use a service specific +backend instead. -In other words, you should really only store an S3QL file system -using RRS if you know exactly what you are getting into. -\end{notice} +The storage URL for accessing an arbitrary S3 compatible storage +service is +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3c://}\PYG{n+nv}{\textless{}hostname\textgreater{}}\PYG{l}{:}\PYG{n+nv}{\textless{}port\textgreater{}}\PYG{l}{/}\PYG{n+nv}{\textless{}bucketname\textgreater{}}\PYG{l}{/}\PYG{n+nv}{\textless{}prefix\textgreater{}} +\end{Verbatim} -\section{The Local Backend} -\label{backends:the-local-backend} -The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -\code{local://\textless{}path\textgreater{}}. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. \code{local:///var/archive}. +or -The local backend provides read-after-write consistency. +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3cs://}\PYG{n+nv}{\textless{}hostname\textgreater{}}\PYG{l}{:}\PYG{n+nv}{\textless{}port\textgreater{}}\PYG{l}{/}\PYG{n+nv}{\textless{}bucketname\textgreater{}}\PYG{l}{/}\PYG{n+nv}{\textless{}prefix\textgreater{}} +\end{Verbatim} +to use HTTPS connections. Note, however, that at this point S3QL does +not verify the server certificate (cf. \href{http://code.google.com/p/s3ql/issues/detail?id=267}{issue 267}). -\section{The SFTP Backend} -\label{backends:the-sftp-backend} -The SFTP backend uses the SFTP protocol, which is a file transfer -protocol similar to ftp, but uses an encrypted SSH connection. -It provides read-after-write consistency. -Note that the SFTP backend is rather slow and has not been tested -as extensively as the S3 and Local backends. +\section{Local} +\label{backends:local} +S3QL is also able to store its data on the local file system. This can +be used to backup data on external media, or to access external +services that S3QL can not talk to directly (e.g., it is possible to +store data over SSH by first mounting the remote system using +\href{http://fuse.sourceforge.net/sshfs.html}{sshfs}, then using the local backend to store the data in the sshfs +mountpoint). -The storage URL for SFTP connections has the form +The storage URL for local storage is \begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{sftp://}\PYG{n+nv}{\textless{}host\textgreater{}}\PYG{g+ge}{[:port]}\PYG{l}{/}\PYG{n+nv}{\textless{}path\textgreater{}} +\PYG{l}{local://}\PYG{n+nv}{\textless{}path\textgreater{}} \end{Verbatim} -The SFTP backend will always ask you for a password if you haven't -defined one in \code{\textasciitilde{}/.s3ql/authinfo}. However, public key authentication -is tried first and the password will only be used if the public key -authentication fails. +Note that you have to write three consecutive slashes to specify an +absolute path, e.g. \code{local:///var/archive}. Also, relative paths will +automatically be converted to absolute paths before the authentication +file is read, i.e. if you are in the \code{/home/john} directory and try to +mount \code{local://bucket}, the corresponding section in the +authentication file must match the storage url +\code{local:///home/john/bucket}. + -The public and private keys will be read from the standard files in -\code{\textasciitilde{}/.ssh/}. Note that S3QL will refuse to connect to a computer with -unknown host key; to add the key to your local keyring you have to -establish a connection to that computer with the standard SSH command -line programs first. +\section{SSH/SFTP} +\label{backends:ssh-sftp} +Previous versions of S3QL included an SSH/SFTP backend. With newer +S3QL versions, it is recommended to instead combine the local backend +with \href{http://fuse.sourceforge.net/sshfs.html}{sshfs} (cf. {\hyperref[tips:ssh-tipp]{\emph{SSH Backend}}}). \chapter{File System Creation} @@ -631,27 +678,19 @@ following syntax: This command accepts the following options: \begin{quote} \begin{optionlist}{3cm} -\item [-{-}homedir \textless{}path\textgreater{}] -Directory for log files, cache and authentication -info. (default: \code{\textasciitilde{}/.s3ql)} +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file (default: +\code{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug \textless{}module\textgreater{}] activate debugging output from \textless{}module\textgreater{}. Use \code{all} to -get debug messages from all modules. This option can -be specified multiple times. +get debug messages from all modules. This option can be +specified multiple times. \item [-{-}quiet] be really quiet \item [-{-}version] just print program version and exit -\item [-{-}ssl] -Use SSL when connecting to remote servers. This option -is not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext -even for unencrypted file systems. -\item [-{-}s3-location \textless{}name\textgreater{}] -Storage location for new S3 buckets. Allowed values: -\code{EU}, \code{us-west-1}, \code{ap-southeast-1}, or \code{us-standard}. -(default: EU) \item [-L \textless{}name\textgreater{}] Filesystem label \item [-{-}blocksize \textless{}size\textgreater{}] @@ -663,10 +702,10 @@ Overwrite any existing data. \end{optionlist} \end{quote} -Unless you have specified the \code{-{-}plain} option, \code{mkfs.s3ql} will ask you -to enter an encryption password. If you do not want to enter this -password every time that you mount the file system, you can store it -in the \code{\textasciitilde{}/.s3ql/authinfo} file, see {\hyperref[mount:bucket-pw]{\emph{Storing Encryption Passwords}}}. +Unless you have specified the \code{-{-}plain} option, \code{mkfs.s3ql} will ask +you to enter an encryption password. This password will \emph{not} be read +from an authentication file specified with the \code{-{-}authfile} +option to prevent accidental creation of an encrypted bucket. \chapter{Managing Buckets} @@ -682,7 +721,7 @@ The syntax is \end{Verbatim} where \code{action} may be either of \textbf{passphrase}, -\textbf{upgrade}, \textbf{delete} or \textbf{download-metadata}. +\textbf{upgrade}, \textbf{clear} or \textbf{download-metadata}. The \textbf{s3qladm} accepts the following general options, no matter what specific action is being invoked: @@ -694,17 +733,18 @@ debug messages from all modules. This option can be specified multiple times. \item [-{-}quiet] be really quiet -\item [-{-}homedir \textless{}path\textgreater{}] -Directory for log files, cache and authentication info. -(default: \code{\textasciitilde{}/.s3ql)} +\item [-{-}log \textless{}target\textgreater{}] +Write logging info into this file. File will be rotated +when it reaches 1 MB, and at most 5 old log files will be +kept. Specify \code{none} to disable logging. Default: +\code{none} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file (default: +\code{\textasciitilde{}/.s3ql/authinfo2)} +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)} \item [-{-}version] just print program version and exit -\item [-{-}ssl] -Use SSL when connecting to remote servers. This option is -not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext -even for unencrypted file systems. \end{optionlist} \end{quote} @@ -747,7 +787,7 @@ execute A file system can be deleted with: \begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{s3qladm delete }\PYG{n+nv}{\textless{}storage url\textgreater{}} +\PYG{l}{s3qladm clear }\PYG{n+nv}{\textless{}storage url\textgreater{}} \end{Verbatim} This physically deletes all the data and file system structures. @@ -795,9 +835,17 @@ mounted on one computer at a time. This command accepts the following options: \begin{quote} \begin{optionlist}{3cm} -\item [-{-}homedir \textless{}path\textgreater{}] -Directory for log files, cache and authentication -info. (default: \code{\textasciitilde{}/.s3ql)} +\item [-{-}log \textless{}target\textgreater{}] +Write logging info into this file. File will be +rotated when it reaches 1 MB, and at most 5 old log +files will be kept. Specify \code{none} to disable +logging. Default: \code{\textasciitilde{}/.s3ql/mount.log} +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: +\code{\textasciitilde{}/.s3ql)} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \code{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug \textless{}module\textgreater{}] activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get debug messages from all modules. This option can @@ -806,12 +854,6 @@ be specified multiple times. be really quiet \item [-{-}version] just print program version and exit -\item [-{-}ssl] -Use SSL when connecting to remote servers. This option -is not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext -even for unencrypted file systems. \item [-{-}cachesize \textless{}size\textgreater{}] Cache size in kb (default: 102400 (100 MB)). Should be at least 10 times the blocksize of the filesystem, @@ -854,42 +896,16 @@ Allowed values: \code{lzma}, \code{bzip2}, \code{zlib}, none. \item [-{-}metadata-upload-interval \textless{}seconds\textgreater{}] Interval in seconds between complete metadata uploads. Set to 0 to disable. Default: 24h. -\item [-{-}compression-threads \textless{}no\textgreater{}] -Number of parallel compression and encryption threads -to use (default: 1). +\item [-{-}threads \textless{}no\textgreater{}] +Number of parallel upload threads to use (default: +auto). +\item [-{-}nfs] +Support export of S3QL file systems over NFS (default: +False) \end{optionlist} \end{quote} -\section{Storing Encryption Passwords} -\label{mount:bucket-pw}\label{mount:storing-encryption-passwords} -If you are trying to mount an encrypted bucket, \code{mount.s3ql} will first -try to read the password from the \code{.s3ql/authinfo} file (the same file -that is used to read the backend authentication data) and prompt the -user to enter the password only if this fails. - -The \code{authinfo} entries to specify bucket passwords are of the form - -\begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{storage-url }\PYG{n+nv}{\textless{}storage-url\textgreater{}}\PYG{l}{ password }\PYG{n+nv}{\textless{}password\textgreater{}} -\end{Verbatim} - -So to always use the password \code{topsecret} when mounting \code{s3://joes\_bucket}, -the entry would be - -\begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{storage-url s3://joes\PYGZus{}bucket password topsecret} -\end{Verbatim} - -\begin{notice}{note}{Note:} -If you are using the local backend, the storage url will -always be converted to an absolute path. So if you are in the -\code{/home/john} directory and try to mount \code{local://bucket}, the matching -\code{authinfo} entry has to have a storage url of -\code{local:///home/john/bucket}. -\end{notice} - - \section{Compression Algorithms} \label{mount:compression-algorithms} S3QL supports three compression algorithms, LZMA, Bzip2 and zlib (with @@ -921,9 +937,8 @@ videos at the same time). \section{Parallel Compression} \label{mount:parallel-compression} If you are running S3QL on a system with multiple cores, you might -want to set \code{-{-}compression-threads} to a value bigger than one. This -will instruct S3QL to compress and encrypt several blocks at the same -time. +want to set the \code{-{-}threads} value larger than one. This will +instruct S3QL to compress and encrypt several blocks at the same time. If you want to do this in combination with using the LZMA compression algorithm, you should keep an eye on memory usage though. Every @@ -1045,8 +1060,8 @@ mounted. \section{Snapshotting and Copy-on-Write} \label{special:snapshotting-and-copy-on-write}\label{special:s3qlcp} The command \code{s3qlcp} can be used to duplicate a directory tree without -physically copying the file contents. This is possible due to the data -de-duplication feature of S3QL. +physically copying the file contents. This is made possible by the +data de-duplication feature of S3QL. The syntax of \code{s3qlcp} is: @@ -1076,7 +1091,7 @@ snapshot of \code{\textless{}target\textgreater{}} or vice versa. However, the m usage of \code{s3qlcp} is to regularly duplicate the same source directory, say \code{documents}, to different target directories. For a e.g. monthly replication, the target directories would typically be -named something like \code{documents\_Januray} for the replication in +named something like \code{documents\_January} for the replication in January, \code{documents\_February} for the replication in February etc. In this case it is clear that the target directories should be regarded as snapshots of the source directory. @@ -1088,12 +1103,6 @@ magnitude slower, because \code{cp} would have to read every file completely (so that S3QL had to fetch all the data over the network from the backend) before writing them into the destination folder. -\item {} -Before starting with the replication, S3QL has to flush the local -cache. So if you just copied lots of new data into the file system -that has not yet been uploaded, replication will take longer than -usual. - \end{itemize} @@ -1292,9 +1301,16 @@ or if you suspect that there might be errors, you should run the This command accepts the following options: \begin{quote} \begin{optionlist}{3cm} -\item [-{-}homedir \textless{}path\textgreater{}] -Directory for log files, cache and authentication info. -(default: \code{\textasciitilde{}/.s3ql)} +\item [-{-}log \textless{}target\textgreater{}] +Write logging info into this file. File will be rotated +when it reaches 1 MB, and at most 5 old log files will be +kept. Specify \code{none} to disable logging. Default: +\code{\textasciitilde{}/.s3ql/fsck.log} +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file (default: +\code{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug \textless{}module\textgreater{}] activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get debug messages from all modules. This option can be @@ -1303,12 +1319,6 @@ specified multiple times. be really quiet \item [-{-}version] just print program version and exit -\item [-{-}ssl] -Use SSL when connecting to remote servers. This option is -not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext even -for unencrypted file systems. \item [-{-}batch] If user input is required, exit without prompting. \item [-{-}force] @@ -1328,8 +1338,10 @@ the \code{contrib} directory of the source distribution or in \section{benchmark.py} \label{contrib:benchmark-py} -This program measures your uplink bandwidth and compression speed and -recommends a compression algorithm for optimal throughput. +This program measures S3QL write performance, uplink bandwidth and +compression speed to determine the limiting factor. It also gives +recommendation for compression algorithm and number of upload threads +to achieve maximum performance. \section{s3\_copy.py} @@ -1344,7 +1356,7 @@ migrate buckets to a different storage region or storage class \code{pcp.py} is a wrapper program that starts several rsync processes to copy directory trees in parallel. This is important because transferring files in parallel significantly enhances performance when -copying data from an S3QL file system (see {\hyperref[tips:copy-performance]{\emph{Permanently mounted backup file system}}} for +copying data from an S3QL file system (see {\hyperref[tips:copy-performance]{\emph{Improving copy performance}}} for details). To recursively copy the directory \code{/mnt/home-backup} into @@ -1478,8 +1490,25 @@ properly unmounts it when the system is shut down. \chapter{Tips \& Tricks} \label{tips:tips-tricks}\label{tips::doc} +\section{SSH Backend} +\label{tips:ssh-tipp}\label{tips:ssh-backend} +By combining S3QL's local backend with \href{http://fuse.sourceforge.net/sshfs.html}{sshfs}, it is possible to store an +S3QL file system on arbitrary SSH servers: first mount the remote +target directory into the local filesystem, + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{sshfs user@my.server.com:/mnt/s3ql /mnt/sshfs} +\end{Verbatim} + +and then give the mountpoint to S3QL as a local destination: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{mount.s3ql local:///mnt/sshfs/mybucket /mnt/s3ql} +\end{Verbatim} + + \section{Permanently mounted backup file system} -\label{tips:copy-performance}\label{tips:permanently-mounted-backup-file-system} +\label{tips:permanently-mounted-backup-file-system} If you use S3QL as a backup file system, it can be useful to mount the file system permanently (rather than just mounting it for a backup and unmounting it afterwards). Especially if your file system becomes @@ -1503,7 +1532,12 @@ to zero). \section{Improving copy performance} -\label{tips:improving-copy-performance} +\label{tips:copy-performance}\label{tips:improving-copy-performance} +\begin{notice}{note}{Note:} +The following applies only when copying data \textbf{from} an S3QL file +system, \textbf{not} when copying data \textbf{to} an S3QL file system. +\end{notice} + If you want to copy a lot of smaller files \emph{from} an S3QL file system (e.g. for a system restore) you will probably notice that the performance is rather bad. @@ -1557,6 +1591,11 @@ details. \chapter{Known Issues} \label{issues:known-issues}\label{issues::doc}\begin{itemize} \item {} +S3QL does not verify TLS/SSL server certificates, so a +man-in-the-middle attack is principally possible. See \href{http://code.google.com/p/s3ql/issues/detail?id=267}{issue 267} for more +details. + +\item {} S3QL is rather slow when an application tries to write data in unreasonably small chunks. If a 1 MB file is copied in chunks of 1 KB, this will take more than 10 times as long as when it's copied @@ -1641,7 +1680,7 @@ use a dedicated init script instead). \item {} S3QL relies on the backends not to run out of space. This is a given for big storage providers like Amazon S3, but you may stumble upon -this if you store buckets e.g. on a small sftp server. +this if you store buckets e.g. on smaller servers or servies. If there is no space left in the backend, attempts to write more data into the S3QL file system will fail and the file system will be @@ -1678,34 +1717,14 @@ here in the User's Guide. \subsection{Description} \label{man/mkfs:description} The \textbf{mkfs.s3ql} command creates a new file system in the location -specified by \emph{storage url}. - -The form of the storage url depends on the backend that is used. The -following backends are supported: - - -\subsubsection{Amazon S3} -\label{man/mkfs:amazon-s3} -To store your file system in an Amazon S3 bucket, use a storage URL of -the form \code{s3://\textless{}bucketname\textgreater{}}. Bucket names must conform to the S3 Bucket -Name Restrictions. +specified by \emph{storage url}. The storage url depends on the backend +that is used. The S3QL User's Guide should be consulted for a +description of the available backends. - -\subsubsection{Local} -\label{man/mkfs:local} -The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -\code{local://\textless{}path\textgreater{}}. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. \code{local:///var/archive}. - - -\subsubsection{SFTP} -\label{man/mkfs:sftp} -The storage URL for SFTP connections has the form - -\begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{sftp://}\PYG{n+nv}{\textless{}host\textgreater{}}\PYG{g+ge}{[:port]}\PYG{l}{/}\PYG{n+nv}{\textless{}path\textgreater{}} -\end{Verbatim} +Unless you have specified the \code{-{-}plain} option, \code{mkfs.s3ql} will ask +you to enter an encryption password. This password will \emph{not} be read +from an authentication file specified with the \code{-{-}authfile} +option to prevent accidental creation of an encrypted bucket. \subsection{Options} @@ -1713,27 +1732,19 @@ The storage URL for SFTP connections has the form The \textbf{mkfs.s3ql} command accepts the following options. \begin{quote} \begin{optionlist}{3cm} -\item [-{-}homedir \textless{}path\textgreater{}] -Directory for log files, cache and authentication -info. (default: \code{\textasciitilde{}/.s3ql)} +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file (default: +\code{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug \textless{}module\textgreater{}] activate debugging output from \textless{}module\textgreater{}. Use \code{all} to -get debug messages from all modules. This option can -be specified multiple times. +get debug messages from all modules. This option can be +specified multiple times. \item [-{-}quiet] be really quiet \item [-{-}version] just print program version and exit -\item [-{-}ssl] -Use SSL when connecting to remote servers. This option -is not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext -even for unencrypted file systems. -\item [-{-}s3-location \textless{}name\textgreater{}] -Storage location for new S3 buckets. Allowed values: -\code{EU}, \code{us-west-1}, \code{ap-southeast-1}, or \code{us-standard}. -(default: EU) \item [-L \textless{}name\textgreater{}] Filesystem label \item [-{-}blocksize \textless{}size\textgreater{}] @@ -1746,14 +1757,6 @@ Overwrite any existing data. \end{quote} -\subsection{Files} -\label{man/mkfs:files} -Authentication data for backends and bucket encryption passphrases are -read from \code{authinfo} in \code{\textasciitilde{}/.s3ql} or the directory -specified with \code{-{-}homedir}. Log files are placed in the same -directory. - - \subsection{Exit Status} \label{man/mkfs:exit-status} \textbf{mkfs.s3ql} returns exit code 0 if the operation succeeded and 1 if some @@ -1788,32 +1791,8 @@ The \textbf{s3qladm} command performs various operations on S3QL buckets. The file system contained in the bucket \emph{must not be mounted} when using \textbf{s3qladm} or things will go wrong badly. -The form of the storage url depends on the backend that is used. The -following backends are supported: - - -\subsubsection{Amazon S3} -\label{man/adm:amazon-s3} -To store your file system in an Amazon S3 bucket, use a storage URL of -the form \code{s3://\textless{}bucketname\textgreater{}}. Bucket names must conform to the S3 Bucket -Name Restrictions. - - -\subsubsection{Local} -\label{man/adm:local} -The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -\code{local://\textless{}path\textgreater{}}. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. \code{local:///var/archive}. - - -\subsubsection{SFTP} -\label{man/adm:sftp} -The storage URL for SFTP connections has the form - -\begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{sftp://}\PYG{n+nv}{\textless{}host\textgreater{}}\PYG{g+ge}{[:port]}\PYG{l}{/}\PYG{n+nv}{\textless{}path\textgreater{}} -\end{Verbatim} +The storage url depends on the backend that is used. The S3QL User's +Guide should be consulted for a description of the available backends. \subsection{Options} @@ -1827,17 +1806,18 @@ debug messages from all modules. This option can be specified multiple times. \item [-{-}quiet] be really quiet -\item [-{-}homedir \textless{}path\textgreater{}] -Directory for log files, cache and authentication info. -(default: \code{\textasciitilde{}/.s3ql)} +\item [-{-}log \textless{}target\textgreater{}] +Write logging info into this file. File will be rotated +when it reaches 1 MB, and at most 5 old log files will be +kept. Specify \code{none} to disable logging. Default: +\code{none} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file (default: +\code{\textasciitilde{}/.s3ql/authinfo2)} +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)} \item [-{-}version] just print program version and exit -\item [-{-}ssl] -Use SSL when connecting to remote servers. This option is -not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext -even for unencrypted file systems. \end{optionlist} \end{quote} @@ -1864,14 +1844,6 @@ Interactively download backups of the file system metadata. \end{description} -\subsection{Files} -\label{man/adm:files} -Authentication data for backends and bucket encryption passphrases are -read from \code{authinfo} in \code{\textasciitilde{}/.s3ql} or the directory -specified with \code{-{-}homedir}. Log files are placed in the same -directory. - - \subsection{Exit Status} \label{man/adm:exit-status} \textbf{s3qladm} returns exit code 0 if the operation succeeded and 1 if some @@ -1900,34 +1872,9 @@ system, conventional locations are \code{/usr/share/doc/s3ql} or \subsection{Description} \label{man/mount:description} The \textbf{mount.s3ql} command mounts the S3QL file system stored in \emph{storage -url} in the directory \emph{mount point}. - -The form of the storage url depends on the backend that is used. The -following backends are supported: - - -\subsubsection{Amazon S3} -\label{man/mount:amazon-s3} -To store your file system in an Amazon S3 bucket, use a storage URL of -the form \code{s3://\textless{}bucketname\textgreater{}}. Bucket names must conform to the S3 Bucket -Name Restrictions. - - -\subsubsection{Local} -\label{man/mount:local} -The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -\code{local://\textless{}path\textgreater{}}. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. \code{local:///var/archive}. - - -\subsubsection{SFTP} -\label{man/mount:sftp} -The storage URL for SFTP connections has the form - -\begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{sftp://}\PYG{n+nv}{\textless{}host\textgreater{}}\PYG{g+ge}{[:port]}\PYG{l}{/}\PYG{n+nv}{\textless{}path\textgreater{}} -\end{Verbatim} +url} in the directory \emph{mount point}. The storage url depends on the +backend that is used. The S3QL User's Guide should be consulted for a +description of the available backends. \subsection{Options} @@ -1935,9 +1882,17 @@ The storage URL for SFTP connections has the form The \textbf{mount.s3ql} command accepts the following options. \begin{quote} \begin{optionlist}{3cm} -\item [-{-}homedir \textless{}path\textgreater{}] -Directory for log files, cache and authentication -info. (default: \code{\textasciitilde{}/.s3ql)} +\item [-{-}log \textless{}target\textgreater{}] +Write logging info into this file. File will be +rotated when it reaches 1 MB, and at most 5 old log +files will be kept. Specify \code{none} to disable +logging. Default: \code{\textasciitilde{}/.s3ql/mount.log} +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: +\code{\textasciitilde{}/.s3ql)} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \code{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug \textless{}module\textgreater{}] activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get debug messages from all modules. This option can @@ -1946,12 +1901,6 @@ be specified multiple times. be really quiet \item [-{-}version] just print program version and exit -\item [-{-}ssl] -Use SSL when connecting to remote servers. This option -is not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext -even for unencrypted file systems. \item [-{-}cachesize \textless{}size\textgreater{}] Cache size in kb (default: 102400 (100 MB)). Should be at least 10 times the blocksize of the filesystem, @@ -1994,21 +1943,16 @@ Allowed values: \code{lzma}, \code{bzip2}, \code{zlib}, none. \item [-{-}metadata-upload-interval \textless{}seconds\textgreater{}] Interval in seconds between complete metadata uploads. Set to 0 to disable. Default: 24h. -\item [-{-}compression-threads \textless{}no\textgreater{}] -Number of parallel compression and encryption threads -to use (default: 1). +\item [-{-}threads \textless{}no\textgreater{}] +Number of parallel upload threads to use (default: +auto). +\item [-{-}nfs] +Support export of S3QL file systems over NFS (default: +False) \end{optionlist} \end{quote} -\subsection{Files} -\label{man/mount:files} -Authentication data for backends and bucket encryption passphrases are -read from \code{authinfo} in \code{\textasciitilde{}/.s3ql} or the directory -specified with \code{-{-}homedir}. Log files are placed in the same -directory. - - \subsection{Exit Status} \label{man/mount:exit-status} \textbf{mount.s3ql} returns exit code 0 if the operation succeeded and 1 if some @@ -2094,6 +2038,11 @@ where \code{action} may be either of \textbf{flushcache}, The \textbf{s3qlctrl} command performs various actions on the S3QL file system mounted in \code{mountpoint}. +\textbf{s3qlctrl} can only be called by the user that mounted the file system +and (if the file system was mounted with \code{-{-}allow-other} or +\code{-{-}allow-root}) the root user. This limitation might be +removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}). + The following actions may be specified: \begin{description} \item[{flushcache}] \leavevmode @@ -2195,7 +2144,7 @@ snapshot of \code{\textless{}target\textgreater{}} or vice versa. However, the m usage of \code{s3qlcp} is to regularly duplicate the same source directory, say \code{documents}, to different target directories. For a e.g. monthly replication, the target directories would typically be -named something like \code{documents\_Januray} for the replication in +named something like \code{documents\_January} for the replication in January, \code{documents\_February} for the replication in February etc. In this case it is clear that the target directories should be regarded as snapshots of the source directory. @@ -2207,12 +2156,6 @@ magnitude slower, because \code{cp} would have to read every file completely (so that S3QL had to fetch all the data over the network from the backend) before writing them into the destination folder. -\item {} -Before starting with the replication, S3QL has to flush the local -cache. So if you just copied lots of new data into the file system -that has not yet been uploaded, replication will take longer than -usual. - \end{itemize} @@ -2293,6 +2236,11 @@ you to delete immutable trees (which can be created with Be warned that there is no additional confirmation. The directory will be removed entirely and immediately. +\textbf{s3qlrm} can only be called by the user that mounted the file system +and (if the file system was mounted with \code{-{-}allow-other} or +\code{-{-}allow-root}) the root user. This limitation might be +removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}). + \subsection{Options} \label{man/rm:options} @@ -2342,6 +2290,11 @@ whatsoever. You can not add new files or directories and you can not change or delete existing files and directories. The only way to get rid of an immutable tree is to use the \textbf{s3qlrm} command. +\textbf{s3qllock} can only be called by the user that mounted the file system +and (if the file system was mounted with \code{-{-}allow-other} or +\code{-{-}allow-root}) the root user. This limitation might be +removed in the future (see \href{http://code.google.com/p/s3ql/issues/detail?id=155}{issue 155}). + \subsection{Rationale} \label{man/lock:rationale} @@ -2480,34 +2433,9 @@ system, conventional locations are \code{/usr/share/doc/s3ql} or \label{man/fsck:description} The \textbf{mkfs.s3ql} command checks the new file system in the location specified by \emph{storage url} for errors and attempts to repair any -problems. - -The form of the storage url depends on the backend that is used. The -following backends are supported: - - -\subsubsection{Amazon S3} -\label{man/fsck:amazon-s3} -To store your file system in an Amazon S3 bucket, use a storage URL of -the form \code{s3://\textless{}bucketname\textgreater{}}. Bucket names must conform to the S3 Bucket -Name Restrictions. - - -\subsubsection{Local} -\label{man/fsck:local} -The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -\code{local://\textless{}path\textgreater{}}. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. \code{local:///var/archive}. - - -\subsubsection{SFTP} -\label{man/fsck:sftp} -The storage URL for SFTP connections has the form - -\begin{Verbatim}[commandchars=\\\{\}] -\PYG{l}{sftp://}\PYG{n+nv}{\textless{}host\textgreater{}}\PYG{g+ge}{[:port]}\PYG{l}{/}\PYG{n+nv}{\textless{}path\textgreater{}} -\end{Verbatim} +problems. The storage url depends on the backend that is used. The +S3QL User's Guide should be consulted for a description of the +available backends. \subsection{Options} @@ -2515,9 +2443,16 @@ The storage URL for SFTP connections has the form The \textbf{mkfs.s3ql} command accepts the following options. \begin{quote} \begin{optionlist}{3cm} -\item [-{-}homedir \textless{}path\textgreater{}] -Directory for log files, cache and authentication info. -(default: \code{\textasciitilde{}/.s3ql)} +\item [-{-}log \textless{}target\textgreater{}] +Write logging info into this file. File will be rotated +when it reaches 1 MB, and at most 5 old log files will be +kept. Specify \code{none} to disable logging. Default: +\code{\textasciitilde{}/.s3ql/fsck.log} +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: \code{\textasciitilde{}/.s3ql)} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file (default: +\code{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug \textless{}module\textgreater{}] activate debugging output from \textless{}module\textgreater{}. Use \code{all} to get debug messages from all modules. This option can be @@ -2526,12 +2461,6 @@ specified multiple times. be really quiet \item [-{-}version] just print program version and exit -\item [-{-}ssl] -Use SSL when connecting to remote servers. This option is -not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext even -for unencrypted file systems. \item [-{-}batch] If user input is required, exit without prompting. \item [-{-}force] @@ -2540,14 +2469,6 @@ Force checking even if file system is marked clean. \end{quote} -\subsection{Files} -\label{man/fsck:files} -Authentication data for backends and bucket encryption passphrases are -read from \code{authinfo} in \code{\textasciitilde{}/.s3ql} or the directory -specified with \code{-{-}homedir}. Log files are placed in the same -directory. - - \subsection{Exit Status} \label{man/fsck:exit-status} \textbf{mkfs.s3ql} returns exit code 0 if the operation succeeded and 1 if some @@ -2580,6 +2501,10 @@ The \textbf{pcp} command is a is a wrapper that starts several allows much better copying performance on file system that have relatively high latency when retrieving individual files like S3QL. +\textbf{Note}: Using this program only improves performance when copying +\emph{from} an S3QL file system. When copying \emph{to} an S3QL file system, +using \textbf{pcp} is more likely to \emph{decrease} performance. + \subsection{Options} \label{man/pcp:options} diff --git a/doc/latex/manual.toc b/doc/latex/manual.toc index 2996158..d52b4fc 100644 --- a/doc/latex/manual.toc +++ b/doc/latex/manual.toc @@ -5,138 +5,126 @@ \contentsline {chapter}{\numberline {2}Installation}{3}{chapter.2} \contentsline {section}{\numberline {2.1}Dependencies}{3}{section.2.1} \contentsline {section}{\numberline {2.2}Installing S3QL}{4}{section.2.2} -\contentsline {chapter}{\numberline {3}Storage Backends}{5}{chapter.3} -\contentsline {section}{\numberline {3.1}On Backend Reliability}{5}{section.3.1} -\contentsline {section}{\numberline {3.2}The \texttt {authinfo} file}{6}{section.3.2} -\contentsline {section}{\numberline {3.3}Consistency Guarantees}{6}{section.3.3} -\contentsline {subsection}{\numberline {3.3.1}Dealing with Eventual Consistency}{6}{subsection.3.3.1} -\contentsline {section}{\numberline {3.4}The Amazon S3 Backend}{7}{section.3.4} -\contentsline {section}{\numberline {3.5}The Local Backend}{8}{section.3.5} -\contentsline {section}{\numberline {3.6}The SFTP Backend}{8}{section.3.6} -\contentsline {chapter}{\numberline {4}File System Creation}{9}{chapter.4} -\contentsline {chapter}{\numberline {5}Managing Buckets}{11}{chapter.5} -\contentsline {section}{\numberline {5.1}Changing the Passphrase}{11}{section.5.1} -\contentsline {section}{\numberline {5.2}Upgrading the file system}{11}{section.5.2} -\contentsline {section}{\numberline {5.3}Deleting a file system}{12}{section.5.3} -\contentsline {section}{\numberline {5.4}Restoring Metadata Backups}{12}{section.5.4} -\contentsline {chapter}{\numberline {6}Mounting}{13}{chapter.6} -\contentsline {section}{\numberline {6.1}Storing Encryption Passwords}{14}{section.6.1} -\contentsline {section}{\numberline {6.2}Compression Algorithms}{14}{section.6.2} -\contentsline {section}{\numberline {6.3}Parallel Compression}{15}{section.6.3} -\contentsline {section}{\numberline {6.4}Notes about Caching}{15}{section.6.4} -\contentsline {subsection}{\numberline {6.4.1}Maximum Number of Cache Entries}{15}{subsection.6.4.1} -\contentsline {subsection}{\numberline {6.4.2}Cache Flushing and Expiration}{15}{subsection.6.4.2} -\contentsline {section}{\numberline {6.5}Automatic Mounting}{15}{section.6.5} -\contentsline {chapter}{\numberline {7}Advanced S3QL Features}{17}{chapter.7} -\contentsline {section}{\numberline {7.1}Snapshotting and Copy-on-Write}{17}{section.7.1} -\contentsline {subsection}{\numberline {7.1.1}Snapshotting vs Hardlinking}{17}{subsection.7.1.1} -\contentsline {section}{\numberline {7.2}Getting Statistics}{18}{section.7.2} -\contentsline {section}{\numberline {7.3}Immutable Trees}{18}{section.7.3} -\contentsline {section}{\numberline {7.4}Fast Recursive Removal}{19}{section.7.4} -\contentsline {section}{\numberline {7.5}Runtime Configuration}{19}{section.7.5} -\contentsline {chapter}{\numberline {8}Unmounting}{21}{chapter.8} -\contentsline {chapter}{\numberline {9}Checking for Errors}{23}{chapter.9} -\contentsline {chapter}{\numberline {10}Contributed Programs}{25}{chapter.10} -\contentsline {section}{\numberline {10.1}benchmark.py}{25}{section.10.1} -\contentsline {section}{\numberline {10.2}s3\_copy.py}{25}{section.10.2} -\contentsline {section}{\numberline {10.3}pcp.py}{25}{section.10.3} -\contentsline {section}{\numberline {10.4}s3\_backup.sh}{25}{section.10.4} -\contentsline {section}{\numberline {10.5}expire\_backups.py}{26}{section.10.5} -\contentsline {section}{\numberline {10.6}s3ql.conf}{27}{section.10.6} -\contentsline {chapter}{\numberline {11}Tips \& Tricks}{29}{chapter.11} -\contentsline {section}{\numberline {11.1}Permanently mounted backup file system}{29}{section.11.1} -\contentsline {section}{\numberline {11.2}Improving copy performance}{29}{section.11.2} -\contentsline {chapter}{\numberline {12}Known Issues}{31}{chapter.12} -\contentsline {chapter}{\numberline {13}Manpages}{33}{chapter.13} -\contentsline {section}{\numberline {13.1}The \textbf {mkfs.s3ql} command}{33}{section.13.1} -\contentsline {subsection}{\numberline {13.1.1}Synopsis}{33}{subsection.13.1.1} -\contentsline {subsection}{\numberline {13.1.2}Description}{33}{subsection.13.1.2} -\contentsline {subsubsection}{Amazon S3}{33}{subsubsection*.3} -\contentsline {subsubsection}{Local}{33}{subsubsection*.4} -\contentsline {subsubsection}{SFTP}{33}{subsubsection*.5} -\contentsline {subsection}{\numberline {13.1.3}Options}{34}{subsection.13.1.3} -\contentsline {subsection}{\numberline {13.1.4}Files}{34}{subsection.13.1.4} -\contentsline {subsection}{\numberline {13.1.5}Exit Status}{34}{subsection.13.1.5} -\contentsline {subsection}{\numberline {13.1.6}See Also}{34}{subsection.13.1.6} -\contentsline {section}{\numberline {13.2}The \textbf {s3qladm} command}{34}{section.13.2} -\contentsline {subsection}{\numberline {13.2.1}Synopsis}{34}{subsection.13.2.1} -\contentsline {subsection}{\numberline {13.2.2}Description}{35}{subsection.13.2.2} -\contentsline {subsubsection}{Amazon S3}{35}{subsubsection*.6} -\contentsline {subsubsection}{Local}{35}{subsubsection*.7} -\contentsline {subsubsection}{SFTP}{35}{subsubsection*.8} -\contentsline {subsection}{\numberline {13.2.3}Options}{35}{subsection.13.2.3} -\contentsline {subsection}{\numberline {13.2.4}Actions}{35}{subsection.13.2.4} -\contentsline {subsection}{\numberline {13.2.5}Files}{36}{subsection.13.2.5} -\contentsline {subsection}{\numberline {13.2.6}Exit Status}{36}{subsection.13.2.6} -\contentsline {subsection}{\numberline {13.2.7}See Also}{36}{subsection.13.2.7} -\contentsline {section}{\numberline {13.3}The \textbf {mount.s3ql} command}{36}{section.13.3} -\contentsline {subsection}{\numberline {13.3.1}Synopsis}{36}{subsection.13.3.1} -\contentsline {subsection}{\numberline {13.3.2}Description}{36}{subsection.13.3.2} -\contentsline {subsubsection}{Amazon S3}{36}{subsubsection*.9} -\contentsline {subsubsection}{Local}{36}{subsubsection*.10} -\contentsline {subsubsection}{SFTP}{36}{subsubsection*.11} -\contentsline {subsection}{\numberline {13.3.3}Options}{37}{subsection.13.3.3} -\contentsline {subsection}{\numberline {13.3.4}Files}{38}{subsection.13.3.4} -\contentsline {subsection}{\numberline {13.3.5}Exit Status}{38}{subsection.13.3.5} -\contentsline {subsection}{\numberline {13.3.6}See Also}{38}{subsection.13.3.6} -\contentsline {section}{\numberline {13.4}The \textbf {s3qlstat} command}{38}{section.13.4} -\contentsline {subsection}{\numberline {13.4.1}Synopsis}{38}{subsection.13.4.1} -\contentsline {subsection}{\numberline {13.4.2}Description}{38}{subsection.13.4.2} -\contentsline {subsection}{\numberline {13.4.3}Options}{38}{subsection.13.4.3} -\contentsline {subsection}{\numberline {13.4.4}Exit Status}{38}{subsection.13.4.4} -\contentsline {subsection}{\numberline {13.4.5}See Also}{38}{subsection.13.4.5} -\contentsline {section}{\numberline {13.5}The \textbf {s3qlctrl} command}{39}{section.13.5} -\contentsline {subsection}{\numberline {13.5.1}Synopsis}{39}{subsection.13.5.1} -\contentsline {subsection}{\numberline {13.5.2}Description}{39}{subsection.13.5.2} -\contentsline {subsection}{\numberline {13.5.3}Options}{39}{subsection.13.5.3} -\contentsline {subsection}{\numberline {13.5.4}Exit Status}{39}{subsection.13.5.4} -\contentsline {subsection}{\numberline {13.5.5}See Also}{39}{subsection.13.5.5} -\contentsline {section}{\numberline {13.6}The \textbf {s3qlcp} command}{40}{section.13.6} -\contentsline {subsection}{\numberline {13.6.1}Synopsis}{40}{subsection.13.6.1} -\contentsline {subsection}{\numberline {13.6.2}Description}{40}{subsection.13.6.2} -\contentsline {subsubsection}{Snapshotting vs Hardlinking}{40}{subsubsection*.12} -\contentsline {subsection}{\numberline {13.6.3}Options}{40}{subsection.13.6.3} -\contentsline {subsection}{\numberline {13.6.4}Exit Status}{41}{subsection.13.6.4} -\contentsline {subsection}{\numberline {13.6.5}See Also}{41}{subsection.13.6.5} -\contentsline {section}{\numberline {13.7}The \textbf {s3qlrm} command}{41}{section.13.7} -\contentsline {subsection}{\numberline {13.7.1}Synopsis}{41}{subsection.13.7.1} -\contentsline {subsection}{\numberline {13.7.2}Description}{41}{subsection.13.7.2} -\contentsline {subsection}{\numberline {13.7.3}Options}{41}{subsection.13.7.3} -\contentsline {subsection}{\numberline {13.7.4}Exit Status}{41}{subsection.13.7.4} -\contentsline {subsection}{\numberline {13.7.5}See Also}{41}{subsection.13.7.5} -\contentsline {section}{\numberline {13.8}The \textbf {s3qllock} command}{42}{section.13.8} -\contentsline {subsection}{\numberline {13.8.1}Synopsis}{42}{subsection.13.8.1} -\contentsline {subsection}{\numberline {13.8.2}Description}{42}{subsection.13.8.2} -\contentsline {subsection}{\numberline {13.8.3}Rationale}{42}{subsection.13.8.3} -\contentsline {subsection}{\numberline {13.8.4}Options}{42}{subsection.13.8.4} -\contentsline {subsection}{\numberline {13.8.5}Exit Status}{42}{subsection.13.8.5} -\contentsline {subsection}{\numberline {13.8.6}See Also}{43}{subsection.13.8.6} -\contentsline {section}{\numberline {13.9}The \textbf {umount.s3ql} command}{43}{section.13.9} -\contentsline {subsection}{\numberline {13.9.1}Synopsis}{43}{subsection.13.9.1} -\contentsline {subsection}{\numberline {13.9.2}Description}{43}{subsection.13.9.2} -\contentsline {subsection}{\numberline {13.9.3}Options}{43}{subsection.13.9.3} -\contentsline {subsection}{\numberline {13.9.4}Exit Status}{43}{subsection.13.9.4} -\contentsline {subsection}{\numberline {13.9.5}See Also}{43}{subsection.13.9.5} -\contentsline {section}{\numberline {13.10}The \textbf {fsck.s3ql} command}{44}{section.13.10} -\contentsline {subsection}{\numberline {13.10.1}Synopsis}{44}{subsection.13.10.1} -\contentsline {subsection}{\numberline {13.10.2}Description}{44}{subsection.13.10.2} -\contentsline {subsubsection}{Amazon S3}{44}{subsubsection*.13} -\contentsline {subsubsection}{Local}{44}{subsubsection*.14} -\contentsline {subsubsection}{SFTP}{44}{subsubsection*.15} -\contentsline {subsection}{\numberline {13.10.3}Options}{44}{subsection.13.10.3} -\contentsline {subsection}{\numberline {13.10.4}Files}{45}{subsection.13.10.4} -\contentsline {subsection}{\numberline {13.10.5}Exit Status}{45}{subsection.13.10.5} -\contentsline {subsection}{\numberline {13.10.6}See Also}{45}{subsection.13.10.6} -\contentsline {section}{\numberline {13.11}The \textbf {pcp} command}{45}{section.13.11} -\contentsline {subsection}{\numberline {13.11.1}Synopsis}{45}{subsection.13.11.1} -\contentsline {subsection}{\numberline {13.11.2}Description}{45}{subsection.13.11.2} -\contentsline {subsection}{\numberline {13.11.3}Options}{45}{subsection.13.11.3} -\contentsline {subsection}{\numberline {13.11.4}Exit Status}{45}{subsection.13.11.4} -\contentsline {subsection}{\numberline {13.11.5}See Also}{45}{subsection.13.11.5} -\contentsline {section}{\numberline {13.12}The \textbf {expire\_backups} command}{46}{section.13.12} -\contentsline {subsection}{\numberline {13.12.1}Synopsis}{46}{subsection.13.12.1} -\contentsline {subsection}{\numberline {13.12.2}Description}{46}{subsection.13.12.2} -\contentsline {subsection}{\numberline {13.12.3}Options}{47}{subsection.13.12.3} -\contentsline {subsection}{\numberline {13.12.4}Exit Status}{47}{subsection.13.12.4} -\contentsline {subsection}{\numberline {13.12.5}See Also}{47}{subsection.13.12.5} -\contentsline {chapter}{\numberline {14}Further Resources / Getting Help}{49}{chapter.14} +\contentsline {chapter}{\numberline {3}General Information}{5}{chapter.3} +\contentsline {section}{\numberline {3.1}Terminology}{5}{section.3.1} +\contentsline {section}{\numberline {3.2}Storing Authentication Information}{5}{section.3.2} +\contentsline {section}{\numberline {3.3}On Backend Reliability}{6}{section.3.3} +\contentsline {chapter}{\numberline {4}Storage Backends}{9}{chapter.4} +\contentsline {section}{\numberline {4.1}Google Storage}{9}{section.4.1} +\contentsline {section}{\numberline {4.2}Amazon S3}{9}{section.4.2} +\contentsline {subsection}{\numberline {4.2.1}Reduced Redundancy Storage (RRS)}{10}{subsection.4.2.1} +\contentsline {subsection}{\numberline {4.2.2}Potential issues when using the US Standard storage region}{10}{subsection.4.2.2} +\contentsline {section}{\numberline {4.3}S3 compatible}{10}{section.4.3} +\contentsline {section}{\numberline {4.4}Local}{11}{section.4.4} +\contentsline {section}{\numberline {4.5}SSH/SFTP}{11}{section.4.5} +\contentsline {chapter}{\numberline {5}File System Creation}{13}{chapter.5} +\contentsline {chapter}{\numberline {6}Managing Buckets}{15}{chapter.6} +\contentsline {section}{\numberline {6.1}Changing the Passphrase}{15}{section.6.1} +\contentsline {section}{\numberline {6.2}Upgrading the file system}{15}{section.6.2} +\contentsline {section}{\numberline {6.3}Deleting a file system}{16}{section.6.3} +\contentsline {section}{\numberline {6.4}Restoring Metadata Backups}{16}{section.6.4} +\contentsline {chapter}{\numberline {7}Mounting}{17}{chapter.7} +\contentsline {section}{\numberline {7.1}Compression Algorithms}{18}{section.7.1} +\contentsline {section}{\numberline {7.2}Parallel Compression}{18}{section.7.2} +\contentsline {section}{\numberline {7.3}Notes about Caching}{19}{section.7.3} +\contentsline {subsection}{\numberline {7.3.1}Maximum Number of Cache Entries}{19}{subsection.7.3.1} +\contentsline {subsection}{\numberline {7.3.2}Cache Flushing and Expiration}{19}{subsection.7.3.2} +\contentsline {section}{\numberline {7.4}Automatic Mounting}{19}{section.7.4} +\contentsline {chapter}{\numberline {8}Advanced S3QL Features}{21}{chapter.8} +\contentsline {section}{\numberline {8.1}Snapshotting and Copy-on-Write}{21}{section.8.1} +\contentsline {subsection}{\numberline {8.1.1}Snapshotting vs Hardlinking}{21}{subsection.8.1.1} +\contentsline {section}{\numberline {8.2}Getting Statistics}{22}{section.8.2} +\contentsline {section}{\numberline {8.3}Immutable Trees}{22}{section.8.3} +\contentsline {section}{\numberline {8.4}Fast Recursive Removal}{23}{section.8.4} +\contentsline {section}{\numberline {8.5}Runtime Configuration}{23}{section.8.5} +\contentsline {chapter}{\numberline {9}Unmounting}{25}{chapter.9} +\contentsline {chapter}{\numberline {10}Checking for Errors}{27}{chapter.10} +\contentsline {chapter}{\numberline {11}Contributed Programs}{29}{chapter.11} +\contentsline {section}{\numberline {11.1}benchmark.py}{29}{section.11.1} +\contentsline {section}{\numberline {11.2}s3\_copy.py}{29}{section.11.2} +\contentsline {section}{\numberline {11.3}pcp.py}{29}{section.11.3} +\contentsline {section}{\numberline {11.4}s3\_backup.sh}{29}{section.11.4} +\contentsline {section}{\numberline {11.5}expire\_backups.py}{30}{section.11.5} +\contentsline {section}{\numberline {11.6}s3ql.conf}{31}{section.11.6} +\contentsline {chapter}{\numberline {12}Tips \& Tricks}{33}{chapter.12} +\contentsline {section}{\numberline {12.1}SSH Backend}{33}{section.12.1} +\contentsline {section}{\numberline {12.2}Permanently mounted backup file system}{33}{section.12.2} +\contentsline {section}{\numberline {12.3}Improving copy performance}{33}{section.12.3} +\contentsline {chapter}{\numberline {13}Known Issues}{35}{chapter.13} +\contentsline {chapter}{\numberline {14}Manpages}{37}{chapter.14} +\contentsline {section}{\numberline {14.1}The \textbf {mkfs.s3ql} command}{37}{section.14.1} +\contentsline {subsection}{\numberline {14.1.1}Synopsis}{37}{subsection.14.1.1} +\contentsline {subsection}{\numberline {14.1.2}Description}{37}{subsection.14.1.2} +\contentsline {subsection}{\numberline {14.1.3}Options}{37}{subsection.14.1.3} +\contentsline {subsection}{\numberline {14.1.4}Exit Status}{38}{subsection.14.1.4} +\contentsline {subsection}{\numberline {14.1.5}See Also}{38}{subsection.14.1.5} +\contentsline {section}{\numberline {14.2}The \textbf {s3qladm} command}{38}{section.14.2} +\contentsline {subsection}{\numberline {14.2.1}Synopsis}{38}{subsection.14.2.1} +\contentsline {subsection}{\numberline {14.2.2}Description}{38}{subsection.14.2.2} +\contentsline {subsection}{\numberline {14.2.3}Options}{38}{subsection.14.2.3} +\contentsline {subsection}{\numberline {14.2.4}Actions}{39}{subsection.14.2.4} +\contentsline {subsection}{\numberline {14.2.5}Exit Status}{39}{subsection.14.2.5} +\contentsline {subsection}{\numberline {14.2.6}See Also}{39}{subsection.14.2.6} +\contentsline {section}{\numberline {14.3}The \textbf {mount.s3ql} command}{39}{section.14.3} +\contentsline {subsection}{\numberline {14.3.1}Synopsis}{39}{subsection.14.3.1} +\contentsline {subsection}{\numberline {14.3.2}Description}{39}{subsection.14.3.2} +\contentsline {subsection}{\numberline {14.3.3}Options}{39}{subsection.14.3.3} +\contentsline {subsection}{\numberline {14.3.4}Exit Status}{40}{subsection.14.3.4} +\contentsline {subsection}{\numberline {14.3.5}See Also}{40}{subsection.14.3.5} +\contentsline {section}{\numberline {14.4}The \textbf {s3qlstat} command}{41}{section.14.4} +\contentsline {subsection}{\numberline {14.4.1}Synopsis}{41}{subsection.14.4.1} +\contentsline {subsection}{\numberline {14.4.2}Description}{41}{subsection.14.4.2} +\contentsline {subsection}{\numberline {14.4.3}Options}{41}{subsection.14.4.3} +\contentsline {subsection}{\numberline {14.4.4}Exit Status}{41}{subsection.14.4.4} +\contentsline {subsection}{\numberline {14.4.5}See Also}{41}{subsection.14.4.5} +\contentsline {section}{\numberline {14.5}The \textbf {s3qlctrl} command}{41}{section.14.5} +\contentsline {subsection}{\numberline {14.5.1}Synopsis}{41}{subsection.14.5.1} +\contentsline {subsection}{\numberline {14.5.2}Description}{41}{subsection.14.5.2} +\contentsline {subsection}{\numberline {14.5.3}Options}{42}{subsection.14.5.3} +\contentsline {subsection}{\numberline {14.5.4}Exit Status}{42}{subsection.14.5.4} +\contentsline {subsection}{\numberline {14.5.5}See Also}{42}{subsection.14.5.5} +\contentsline {section}{\numberline {14.6}The \textbf {s3qlcp} command}{42}{section.14.6} +\contentsline {subsection}{\numberline {14.6.1}Synopsis}{42}{subsection.14.6.1} +\contentsline {subsection}{\numberline {14.6.2}Description}{42}{subsection.14.6.2} +\contentsline {subsubsection}{Snapshotting vs Hardlinking}{43}{subsubsection*.3} +\contentsline {subsection}{\numberline {14.6.3}Options}{43}{subsection.14.6.3} +\contentsline {subsection}{\numberline {14.6.4}Exit Status}{43}{subsection.14.6.4} +\contentsline {subsection}{\numberline {14.6.5}See Also}{43}{subsection.14.6.5} +\contentsline {section}{\numberline {14.7}The \textbf {s3qlrm} command}{44}{section.14.7} +\contentsline {subsection}{\numberline {14.7.1}Synopsis}{44}{subsection.14.7.1} +\contentsline {subsection}{\numberline {14.7.2}Description}{44}{subsection.14.7.2} +\contentsline {subsection}{\numberline {14.7.3}Options}{44}{subsection.14.7.3} +\contentsline {subsection}{\numberline {14.7.4}Exit Status}{44}{subsection.14.7.4} +\contentsline {subsection}{\numberline {14.7.5}See Also}{44}{subsection.14.7.5} +\contentsline {section}{\numberline {14.8}The \textbf {s3qllock} command}{44}{section.14.8} +\contentsline {subsection}{\numberline {14.8.1}Synopsis}{44}{subsection.14.8.1} +\contentsline {subsection}{\numberline {14.8.2}Description}{45}{subsection.14.8.2} +\contentsline {subsection}{\numberline {14.8.3}Rationale}{45}{subsection.14.8.3} +\contentsline {subsection}{\numberline {14.8.4}Options}{45}{subsection.14.8.4} +\contentsline {subsection}{\numberline {14.8.5}Exit Status}{45}{subsection.14.8.5} +\contentsline {subsection}{\numberline {14.8.6}See Also}{45}{subsection.14.8.6} +\contentsline {section}{\numberline {14.9}The \textbf {umount.s3ql} command}{46}{section.14.9} +\contentsline {subsection}{\numberline {14.9.1}Synopsis}{46}{subsection.14.9.1} +\contentsline {subsection}{\numberline {14.9.2}Description}{46}{subsection.14.9.2} +\contentsline {subsection}{\numberline {14.9.3}Options}{46}{subsection.14.9.3} +\contentsline {subsection}{\numberline {14.9.4}Exit Status}{46}{subsection.14.9.4} +\contentsline {subsection}{\numberline {14.9.5}See Also}{46}{subsection.14.9.5} +\contentsline {section}{\numberline {14.10}The \textbf {fsck.s3ql} command}{46}{section.14.10} +\contentsline {subsection}{\numberline {14.10.1}Synopsis}{46}{subsection.14.10.1} +\contentsline {subsection}{\numberline {14.10.2}Description}{47}{subsection.14.10.2} +\contentsline {subsection}{\numberline {14.10.3}Options}{47}{subsection.14.10.3} +\contentsline {subsection}{\numberline {14.10.4}Exit Status}{47}{subsection.14.10.4} +\contentsline {subsection}{\numberline {14.10.5}See Also}{47}{subsection.14.10.5} +\contentsline {section}{\numberline {14.11}The \textbf {pcp} command}{47}{section.14.11} +\contentsline {subsection}{\numberline {14.11.1}Synopsis}{47}{subsection.14.11.1} +\contentsline {subsection}{\numberline {14.11.2}Description}{48}{subsection.14.11.2} +\contentsline {subsection}{\numberline {14.11.3}Options}{48}{subsection.14.11.3} +\contentsline {subsection}{\numberline {14.11.4}Exit Status}{48}{subsection.14.11.4} +\contentsline {subsection}{\numberline {14.11.5}See Also}{48}{subsection.14.11.5} +\contentsline {section}{\numberline {14.12}The \textbf {expire\_backups} command}{48}{section.14.12} +\contentsline {subsection}{\numberline {14.12.1}Synopsis}{48}{subsection.14.12.1} +\contentsline {subsection}{\numberline {14.12.2}Description}{48}{subsection.14.12.2} +\contentsline {subsection}{\numberline {14.12.3}Options}{49}{subsection.14.12.3} +\contentsline {subsection}{\numberline {14.12.4}Exit Status}{49}{subsection.14.12.4} +\contentsline {subsection}{\numberline {14.12.5}See Also}{49}{subsection.14.12.5} +\contentsline {chapter}{\numberline {15}Further Resources / Getting Help}{51}{chapter.15} diff --git a/doc/man/fsck.s3ql.1 b/doc/man/fsck.s3ql.1 index e6c786d..d06ff1e 100644 --- a/doc/man/fsck.s3ql.1 +++ b/doc/man/fsck.s3ql.1 @@ -1,4 +1,4 @@ -.TH "FSCK.S3QL" "1" "May 20, 2011" "1.0.1" "S3QL" +.TH "FSCK.S3QL" "1" "September 28, 2011" "1.2" "S3QL" .SH NAME fsck.s3ql \- Check an S3QL file system for errors . @@ -45,30 +45,9 @@ which only briefly document the available userspace commands). .sp The \fBmkfs.s3ql\fP command checks the new file system in the location specified by \fIstorage url\fP for errors and attempts to repair any -problems. -.sp -The form of the storage url depends on the backend that is used. The -following backends are supported: -.SS Amazon S3 -.sp -To store your file system in an Amazon S3 bucket, use a storage URL of -the form \fBs3://<bucketname>\fP. Bucket names must conform to the S3 Bucket -Name Restrictions. -.SS Local -.sp -The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -\fBlocal://<path>\fP. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. \fBlocal:///var/archive\fP. -.SS SFTP -.sp -The storage URL for SFTP connections has the form -.sp -.nf -.ft C -sftp://<host>[:port]/<path> -.ft P -.fi +problems. The storage url depends on the backend that is used. The +S3QL User\(aqs Guide should be consulted for a description of the +available backends. .SH OPTIONS .sp The \fBmkfs.s3ql\fP command accepts the following options. @@ -76,49 +55,38 @@ The \fBmkfs.s3ql\fP command accepts the following options. .INDENT 3.5 .INDENT 0.0 .TP -.BI \-\-homedir \ <path> -. -Directory for log files, cache and authentication info. -(default: \fB~/.s3ql)\fP +.BI \-\-log \ <target> +Write logging info into this file. File will be rotated +when it reaches 1 MB, and at most 5 old log files will be +kept. Specify \fBnone\fP to disable logging. Default: +\fB~/.s3ql/fsck.log\fP +.TP +.BI \-\-cachedir \ <path> +Store cached data in this directory (default: \fB~/.s3ql)\fP +.TP +.BI \-\-authfile \ <path> +Read authentication credentials from this file (default: +\fB~/.s3ql/authinfo2)\fP .TP .BI \-\-debug \ <module> -. activate debugging output from <module>. Use \fBall\fP to get debug messages from all modules. This option can be specified multiple times. .TP .B \-\-quiet -. be really quiet .TP .B \-\-version -. just print program version and exit .TP -.B \-\-ssl -. -Use SSL when connecting to remote servers. This option is -not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext even -for unencrypted file systems. -.TP .B \-\-batch -. If user input is required, exit without prompting. .TP .B \-\-force -. Force checking even if file system is marked clean. .UNINDENT .UNINDENT .UNINDENT -.SH FILES -.sp -Authentication data for backends and bucket encryption passphrases are -read from \fBauthinfo\fP in \fB~/.s3ql\fP or the directory -specified with \fB\-\-homedir\fP. Log files are placed in the same -directory. .SH EXIT STATUS .sp \fBmkfs.s3ql\fP returns exit code 0 if the operation succeeded and 1 if some diff --git a/doc/man/mkfs.s3ql.1 b/doc/man/mkfs.s3ql.1 index dc57add..9af4449 100644 --- a/doc/man/mkfs.s3ql.1 +++ b/doc/man/mkfs.s3ql.1 @@ -1,4 +1,4 @@ -.TH "MKFS.S3QL" "1" "May 20, 2011" "1.0.1" "S3QL" +.TH "MKFS.S3QL" "1" "September 28, 2011" "1.2" "S3QL" .SH NAME mkfs.s3ql \- Create an S3QL file system . @@ -44,30 +44,14 @@ sure to consult the full documentation (rather than just the man pages which only briefly document the available userspace commands). .sp The \fBmkfs.s3ql\fP command creates a new file system in the location -specified by \fIstorage url\fP. +specified by \fIstorage url\fP. The storage url depends on the backend +that is used. The S3QL User\(aqs Guide should be consulted for a +description of the available backends. .sp -The form of the storage url depends on the backend that is used. The -following backends are supported: -.SS Amazon S3 -.sp -To store your file system in an Amazon S3 bucket, use a storage URL of -the form \fBs3://<bucketname>\fP. Bucket names must conform to the S3 Bucket -Name Restrictions. -.SS Local -.sp -The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -\fBlocal://<path>\fP. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. \fBlocal:///var/archive\fP. -.SS SFTP -.sp -The storage URL for SFTP connections has the form -.sp -.nf -.ft C -sftp://<host>[:port]/<path> -.ft P -.fi +Unless you have specified the \fB\-\-plain\fP option, \fBmkfs.s3ql\fP will ask +you to enter an encryption password. This password will \fInot\fP be read +from an authentication file specified with the \fB\-\-authfile\fP +option to prevent accidental creation of an encrypted bucket. .SH OPTIONS .sp The \fBmkfs.s3ql\fP command accepts the following options. @@ -75,63 +59,38 @@ The \fBmkfs.s3ql\fP command accepts the following options. .INDENT 3.5 .INDENT 0.0 .TP -.BI \-\-homedir \ <path> -. -Directory for log files, cache and authentication -info. (default: \fB~/.s3ql)\fP +.BI \-\-cachedir \ <path> +Store cached data in this directory (default: \fB~/.s3ql)\fP +.TP +.BI \-\-authfile \ <path> +Read authentication credentials from this file (default: +\fB~/.s3ql/authinfo2)\fP .TP .BI \-\-debug \ <module> -. activate debugging output from <module>. Use \fBall\fP to -get debug messages from all modules. This option can -be specified multiple times. +get debug messages from all modules. This option can be +specified multiple times. .TP .B \-\-quiet -. be really quiet .TP .B \-\-version -. just print program version and exit .TP -.B \-\-ssl -. -Use SSL when connecting to remote servers. This option -is not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext -even for unencrypted file systems. -.TP -.BI \-\-s3\-location \ <name> -. -Storage location for new S3 buckets. Allowed values: -\fBEU\fP, \fBus\-west\-1\fP, \fBap\-southeast\-1\fP, or \fBus\-standard\fP. -(default: EU) -.TP .BI \-L \ <name> -. Filesystem label .TP .BI \-\-blocksize \ <size> -. Maximum block size in KB (default: 10240) .TP .B \-\-plain -. Create unencrypted file system. .TP .B \-\-force -. Overwrite any existing data. .UNINDENT .UNINDENT .UNINDENT -.SH FILES -.sp -Authentication data for backends and bucket encryption passphrases are -read from \fBauthinfo\fP in \fB~/.s3ql\fP or the directory -specified with \fB\-\-homedir\fP. Log files are placed in the same -directory. .SH EXIT STATUS .sp \fBmkfs.s3ql\fP returns exit code 0 if the operation succeeded and 1 if some diff --git a/doc/man/mount.s3ql.1 b/doc/man/mount.s3ql.1 index fa20a20..0c10701 100644 --- a/doc/man/mount.s3ql.1 +++ b/doc/man/mount.s3ql.1 @@ -1,4 +1,4 @@ -.TH "MOUNT.S3QL" "1" "May 20, 2011" "1.0.1" "S3QL" +.TH "MOUNT.S3QL" "1" "September 28, 2011" "1.2" "S3QL" .SH NAME mount.s3ql \- Mount an S3QL file system . @@ -44,30 +44,9 @@ sure to consult the full documentation (rather than just the man pages which only briefly document the available userspace commands). .sp The \fBmount.s3ql\fP command mounts the S3QL file system stored in \fIstorage -url\fP in the directory \fImount point\fP. -.sp -The form of the storage url depends on the backend that is used. The -following backends are supported: -.SS Amazon S3 -.sp -To store your file system in an Amazon S3 bucket, use a storage URL of -the form \fBs3://<bucketname>\fP. Bucket names must conform to the S3 Bucket -Name Restrictions. -.SS Local -.sp -The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -\fBlocal://<path>\fP. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. \fBlocal:///var/archive\fP. -.SS SFTP -.sp -The storage URL for SFTP connections has the form -.sp -.nf -.ft C -sftp://<host>[:port]/<path> -.ft P -.fi +url\fP in the directory \fImount point\fP. The storage url depends on the +backend that is used. The S3QL User\(aqs Guide should be consulted for a +description of the available backends. .SH OPTIONS .sp The \fBmount.s3ql\fP command accepts the following options. @@ -75,35 +54,32 @@ The \fBmount.s3ql\fP command accepts the following options. .INDENT 3.5 .INDENT 0.0 .TP -.BI \-\-homedir \ <path> -. -Directory for log files, cache and authentication -info. (default: \fB~/.s3ql)\fP +.BI \-\-log \ <target> +Write logging info into this file. File will be +rotated when it reaches 1 MB, and at most 5 old log +files will be kept. Specify \fBnone\fP to disable +logging. Default: \fB~/.s3ql/mount.log\fP +.TP +.BI \-\-cachedir \ <path> +Store cached data in this directory (default: +\fB~/.s3ql)\fP +.TP +.BI \-\-authfile \ <path> +Read authentication credentials from this file +(default: \fB~/.s3ql/authinfo2)\fP .TP .BI \-\-debug \ <module> -. activate debugging output from <module>. Use \fBall\fP to get debug messages from all modules. This option can be specified multiple times. .TP .B \-\-quiet -. be really quiet .TP .B \-\-version -. just print program version and exit .TP -.B \-\-ssl -. -Use SSL when connecting to remote servers. This option -is not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext -even for unencrypted file systems. -.TP .BI \-\-cachesize \ <size> -. Cache size in kb (default: 102400 (100 MB)). Should be at least 10 times the blocksize of the filesystem, otherwise an object may be retrieved and written @@ -111,7 +87,6 @@ several times during a single write() or read() operation. .TP .BI \-\-max\-cache\-entries \ <num> -. Maximum number of entries in cache (default: 768). Each cache entry requires one file descriptor, so if you increase this number you have to make sure that @@ -120,7 +95,6 @@ your process file descriptor limit (as set with cache entries + 100). .TP .B \-\-allow\-other -. Normally, only the user who called \fBmount.s3ql\fP can access the mount point. This user then also has full access to it, independent of individual file @@ -130,53 +104,43 @@ well and individual file permissions are taken into account for all users. .TP .B \-\-allow\-root -. Like \fB\-\-allow\-other\fP, but restrict access to the mounting user and the root user. .TP .B \-\-fg -. Do not daemonize, stay in foreground .TP .B \-\-single -. Run in single threaded mode. If you don\(aqt understand this, then you don\(aqt need it. .TP .B \-\-upstart -. Stay in foreground and raise SIGSTOP once mountpoint is up. .TP .B \-\-profile -. Create profiling information. If you don\(aqt understand this, then you don\(aqt need it. .TP .BI \-\-compress \ <name> -. Compression algorithm to use when storing new data. Allowed values: \fBlzma\fP, \fBbzip2\fP, \fBzlib\fP, none. (default: \fBlzma\fP) .TP .BI \-\-metadata\-upload\-interval \ <seconds> -. Interval in seconds between complete metadata uploads. Set to 0 to disable. Default: 24h. .TP -.BI \-\-compression\-threads \ <no> -. -Number of parallel compression and encryption threads -to use (default: 1). +.BI \-\-threads \ <no> +Number of parallel upload threads to use (default: +auto). +.TP +.B \-\-nfs +Support export of S3QL file systems over NFS (default: +False) .UNINDENT .UNINDENT .UNINDENT -.SH FILES -.sp -Authentication data for backends and bucket encryption passphrases are -read from \fBauthinfo\fP in \fB~/.s3ql\fP or the directory -specified with \fB\-\-homedir\fP. Log files are placed in the same -directory. .SH EXIT STATUS .sp \fBmount.s3ql\fP returns exit code 0 if the operation succeeded and 1 if some diff --git a/doc/man/s3qladm.1 b/doc/man/s3qladm.1 index 24cb81a..805f831 100644 --- a/doc/man/s3qladm.1 +++ b/doc/man/s3qladm.1 @@ -1,4 +1,4 @@ -.TH "S3QLADM" "1" "May 20, 2011" "1.0.1" "S3QL" +.TH "S3QLADM" "1" "September 28, 2011" "1.2" "S3QL" .SH NAME s3qladm \- Manage S3QL buckets . @@ -50,28 +50,8 @@ The \fBs3qladm\fP command performs various operations on S3QL buckets. The file system contained in the bucket \fImust not be mounted\fP when using \fBs3qladm\fP or things will go wrong badly. .sp -The form of the storage url depends on the backend that is used. The -following backends are supported: -.SS Amazon S3 -.sp -To store your file system in an Amazon S3 bucket, use a storage URL of -the form \fBs3://<bucketname>\fP. Bucket names must conform to the S3 Bucket -Name Restrictions. -.SS Local -.sp -The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -\fBlocal://<path>\fP. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. \fBlocal:///var/archive\fP. -.SS SFTP -.sp -The storage URL for SFTP connections has the form -.sp -.nf -.ft C -sftp://<host>[:port]/<path> -.ft P -.fi +The storage url depends on the backend that is used. The S3QL User\(aqs +Guide should be consulted for a description of the available backends. .SH OPTIONS .sp The \fBs3qladm\fP command accepts the following options. @@ -80,31 +60,28 @@ The \fBs3qladm\fP command accepts the following options. .INDENT 0.0 .TP .BI \-\-debug \ <module> -. activate debugging output from <module>. Use \fBall\fP to get debug messages from all modules. This option can be specified multiple times. .TP .B \-\-quiet -. be really quiet .TP -.BI \-\-homedir \ <path> -. -Directory for log files, cache and authentication info. -(default: \fB~/.s3ql)\fP +.BI \-\-log \ <target> +Write logging info into this file. File will be rotated +when it reaches 1 MB, and at most 5 old log files will be +kept. Specify \fBnone\fP to disable logging. Default: +\fBnone\fP +.TP +.BI \-\-authfile \ <path> +Read authentication credentials from this file (default: +\fB~/.s3ql/authinfo2)\fP +.TP +.BI \-\-cachedir \ <path> +Store cached data in this directory (default: \fB~/.s3ql)\fP .TP .B \-\-version -. just print program version and exit -.TP -.B \-\-ssl -. -Use SSL when connecting to remote servers. This option is -not enabled by default, because for encrypted file -systems, all data is already encrypted anyway, and -authentication data is never transmitted in plaintext -even for unencrypted file systems. .UNINDENT .UNINDENT .UNINDENT @@ -117,27 +94,17 @@ The following actions may be specified: .INDENT 0.0 .TP .B passphrase -. Changes the encryption passphrase of the bucket. .TP .B upgrade -. Upgrade the file system contained in the bucket to the newest revision. .TP .B delete -. Delete the bucket and all its contents. .TP .B download\-metadata -. Interactively download backups of the file system metadata. .UNINDENT -.SH FILES -.sp -Authentication data for backends and bucket encryption passphrases are -read from \fBauthinfo\fP in \fB~/.s3ql\fP or the directory -specified with \fB\-\-homedir\fP. Log files are placed in the same -directory. .SH EXIT STATUS .sp \fBs3qladm\fP returns exit code 0 if the operation succeeded and 1 if some diff --git a/doc/man/s3qlcp.1 b/doc/man/s3qlcp.1 index b2bc8b2..e73d9a9 100644 --- a/doc/man/s3qlcp.1 +++ b/doc/man/s3qlcp.1 @@ -1,4 +1,4 @@ -.TH "S3QLCP" "1" "May 20, 2011" "1.0.1" "S3QL" +.TH "S3QLCP" "1" "September 28, 2011" "1.2" "S3QL" .SH NAME s3qlcp \- Copy-on-write replication on S3QL file systems . @@ -58,30 +58,22 @@ the root user. This limitation might be removed in the future (see \fI\%issue 15 Note that: .INDENT 0.0 .IP \(bu 2 -. After the replication, both source and target directory will still be completely ordinary directories. You can regard \fB<src>\fP as a snapshot of \fB<target>\fP or vice versa. However, the most common usage of \fBs3qlcp\fP is to regularly duplicate the same source directory, say \fBdocuments\fP, to different target directories. For a e.g. monthly replication, the target directories would typically be -named something like \fBdocuments_Januray\fP for the replication in +named something like \fBdocuments_January\fP for the replication in January, \fBdocuments_February\fP for the replication in February etc. In this case it is clear that the target directories should be regarded as snapshots of the source directory. .IP \(bu 2 -. Exactly the same effect could be achieved by an ordinary copy program like \fBcp \-a\fP. However, this procedure would be orders of magnitude slower, because \fBcp\fP would have to read every file completely (so that S3QL had to fetch all the data over the network from the backend) before writing them into the destination folder. -.IP \(bu 2 -. -Before starting with the replication, S3QL has to flush the local -cache. So if you just copied lots of new data into the file system -that has not yet been uploaded, replication will take longer than -usual. .UNINDENT .SS Snapshotting vs Hardlinking .sp @@ -93,13 +85,11 @@ identical file already exists in the backup. However, using hardlinks has two large disadvantages: .INDENT 0.0 .IP \(bu 2 -. backups and restores always have to be made with a special program that takes care of the hardlinking. The backup must not be touched by any other programs (they may make changes that inadvertently affect other hardlinked files) .IP \(bu 2 -. special care needs to be taken to handle files which are already hardlinked (the restore program needs to know that the hardlink was not just introduced by the backup program to safe space) @@ -115,15 +105,12 @@ The \fBs3qlcp\fP command accepts the following options: .INDENT 0.0 .TP .B \-\-debug -. activate debugging output .TP .B \-\-quiet -. be really quiet .TP .B \-\-version -. just print program version and exit .UNINDENT .UNINDENT diff --git a/doc/man/s3qlctrl.1 b/doc/man/s3qlctrl.1 index f8c0d56..9da182d 100644 --- a/doc/man/s3qlctrl.1 +++ b/doc/man/s3qlctrl.1 @@ -1,4 +1,4 @@ -.TH "S3QLCTRL" "1" "May 20, 2011" "1.0.1" "S3QL" +.TH "S3QLCTRL" "1" "September 28, 2011" "1.2" "S3QL" .SH NAME s3qlctrl \- Control a mounted S3QL file system . @@ -50,20 +50,22 @@ which only briefly document the available userspace commands). The \fBs3qlctrl\fP command performs various actions on the S3QL file system mounted in \fBmountpoint\fP. .sp +\fBs3qlctrl\fP can only be called by the user that mounted the file system +and (if the file system was mounted with \fB\-\-allow\-other\fP or +\fB\-\-allow\-root\fP) the root user. This limitation might be +removed in the future (see \fI\%issue 155\fP). +.sp The following actions may be specified: .INDENT 0.0 .TP .B flushcache -. Uploads all changed file data to the backend. .TP .B upload\-meta -. Upload metadata to the backend. All file system operations will block while a snapshot of the metadata is prepared for upload. .TP .B cachesize -. Changes the cache size of the file system. This action requires an additional argument that specifies the new cache size in kB, so the complete command line is: @@ -75,7 +77,6 @@ s3qlctrl [options] cachesize <mountpoint> <new\-cache\-size> .fi .TP .B log -. Change the amount of information that is logged into \fB~/.s3ql/mount.log\fP file. The complete syntax is: .sp @@ -99,15 +100,12 @@ what specific action is being invoked: .INDENT 0.0 .TP .B \-\-debug -. activate debugging output .TP .B \-\-quiet -. be really quiet .TP .B \-\-version -. just print program version and exit .UNINDENT .UNINDENT diff --git a/doc/man/s3qllock.1 b/doc/man/s3qllock.1 index 5892a46..4f4f88d 100644 --- a/doc/man/s3qllock.1 +++ b/doc/man/s3qllock.1 @@ -1,4 +1,4 @@ -.TH "S3QLLOCK" "1" "May 20, 2011" "1.0.1" "S3QL" +.TH "S3QLLOCK" "1" "September 28, 2011" "1.2" "S3QL" .SH NAME s3qllock \- Make trees on an S3QL file system immutable . @@ -48,6 +48,11 @@ system immutable. Immutable trees can no longer be changed in any way whatsoever. You can not add new files or directories and you can not change or delete existing files and directories. The only way to get rid of an immutable tree is to use the \fBs3qlrm\fP command. +.sp +\fBs3qllock\fP can only be called by the user that mounted the file system +and (if the file system was mounted with \fB\-\-allow\-other\fP or +\fB\-\-allow\-root\fP) the root user. This limitation might be +removed in the future (see \fI\%issue 155\fP). .SH RATIONALE .sp Immutability is a feature designed for backups. Traditionally, backups @@ -87,15 +92,12 @@ The \fBs3qllock\fP command accepts the following options: .INDENT 0.0 .TP .B \-\-debug -. activate debugging output .TP .B \-\-quiet -. be really quiet .TP .B \-\-version -. just print program version and exit .UNINDENT .UNINDENT diff --git a/doc/man/s3qlrm.1 b/doc/man/s3qlrm.1 index 66af40f..1a4a09e 100644 --- a/doc/man/s3qlrm.1 +++ b/doc/man/s3qlrm.1 @@ -1,4 +1,4 @@ -.TH "S3QLRM" "1" "May 20, 2011" "1.0.1" "S3QL" +.TH "S3QLRM" "1" "September 28, 2011" "1.2" "S3QL" .SH NAME s3qlrm \- Fast tree removal on S3QL file systems . @@ -51,6 +51,11 @@ you to delete immutable trees (which can be created with .sp Be warned that there is no additional confirmation. The directory will be removed entirely and immediately. +.sp +\fBs3qlrm\fP can only be called by the user that mounted the file system +and (if the file system was mounted with \fB\-\-allow\-other\fP or +\fB\-\-allow\-root\fP) the root user. This limitation might be +removed in the future (see \fI\%issue 155\fP). .SH OPTIONS .sp The \fBs3qlrm\fP command accepts the following options: @@ -59,15 +64,12 @@ The \fBs3qlrm\fP command accepts the following options: .INDENT 0.0 .TP .B \-\-debug -. activate debugging output .TP .B \-\-quiet -. be really quiet .TP .B \-\-version -. just print program version and exit .UNINDENT .UNINDENT diff --git a/doc/man/s3qlstat.1 b/doc/man/s3qlstat.1 index b9ce4f6..b2849e2 100644 --- a/doc/man/s3qlstat.1 +++ b/doc/man/s3qlstat.1 @@ -1,4 +1,4 @@ -.TH "S3QLSTAT" "1" "May 20, 2011" "1.0.1" "S3QL" +.TH "S3QLSTAT" "1" "September 28, 2011" "1.2" "S3QL" .SH NAME s3qlstat \- Gather S3QL file system statistics . @@ -58,15 +58,12 @@ The \fBs3qlstat\fP command accepts the following options: .INDENT 0.0 .TP .B \-\-debug -. activate debugging output .TP .B \-\-quiet -. be really quiet .TP .B \-\-version -. just print program version and exit .UNINDENT .UNINDENT diff --git a/doc/man/umount.s3ql.1 b/doc/man/umount.s3ql.1 index b1ea3e8..cfc4bdd 100644 --- a/doc/man/umount.s3ql.1 +++ b/doc/man/umount.s3ql.1 @@ -1,4 +1,4 @@ -.TH "UMOUNT.S3QL" "1" "May 20, 2011" "1.0.1" "S3QL" +.TH "UMOUNT.S3QL" "1" "September 28, 2011" "1.2" "S3QL" .SH NAME umount.s3ql \- Unmount an S3QL file system . @@ -63,19 +63,15 @@ The \fBumount.s3ql\fP command accepts the following options. .INDENT 0.0 .TP .B \-\-debug -. activate debugging output .TP .B \-\-quiet -. be really quiet .TP .B \-\-version -. just print program version and exit .TP .B \-\-lazy, \-z -. Lazy umount. Detaches the file system immediately, even if there are still open files. The data will be uploaded in the background once all open files have been closed. diff --git a/doc/manual.pdf b/doc/manual.pdf Binary files differindex 0553294..c6558d7 100644 --- a/doc/manual.pdf +++ b/doc/manual.pdf diff --git a/rst/about.rst b/rst/about.rst index bef3684..d3fc8a5 100644 --- a/rst/about.rst +++ b/rst/about.rst @@ -4,13 +4,15 @@ About S3QL ============ -S3QL is a file system that stores all its data online. It supports -`Amazon S3 <http://aws.amazon.com/s3 Amazon S3>`_ as well as arbitrary -SFTP servers and effectively provides you with a hard disk of dynamic, -infinite capacity that can be accessed from any computer with internet -access. - -S3QL is providing a standard, full featured UNIX file system that is +S3QL is a file system that stores all its data online using storage +services like `Google Storage +<http://code.google.com/apis/storage/>`_, `Amazon S3 +<http://aws.amazon.com/s3 Amazon S3>`_ or `OpenStack +<http://openstack.org/projects/storage/>`_. S3QL effectively provides +a hard disk of dynamic, infinite capacity that can be accessed from +any computer with internet access. + +S3QL is a standard conforming, full featured UNIX file system that is conceptually indistinguishable from any local file system. Furthermore, S3QL has additional features like compression, encryption, data de-duplication, immutable trees and snapshotting diff --git a/rst/adm.rst b/rst/adm.rst index 3e50f64..34fc7b4 100644 --- a/rst/adm.rst +++ b/rst/adm.rst @@ -14,7 +14,7 @@ The syntax is :: s3qladm [options] <action> <storage-url> where :var:`action` may be either of :program:`passphrase`, -:program:`upgrade`, :program:`delete` or :program:`download-metadata`. +:program:`upgrade`, :program:`clear` or :program:`download-metadata`. The :program:`s3qladm` accepts the following general options, no matter what specific action is being invoked: @@ -55,7 +55,7 @@ Deleting a file system A file system can be deleted with:: - s3qladm delete <storage url> + s3qladm clear <storage url> This physically deletes all the data and file system structures. diff --git a/rst/backends.rst b/rst/backends.rst index 480ff90..5bc9521 100644 --- a/rst/backends.rst +++ b/rst/backends.rst @@ -1,292 +1,205 @@ .. -*- mode: rst -*- +.. _storage_backends: + ================== Storage Backends ================== -S3QL can use different protocols to store the file system data. -Independent of the backend that you use, the place where your file -system data is being stored is called a *bucket*. (This is mostly for -historical reasons, since initially S3QL supported only the Amazon S3 -backend). - - -On Backend Reliability -====================== - -S3QL has been designed for use with a storage backend where data loss -is so infrequent that it can be completely neglected (e.g. the Amazon -S3 backend). If you decide to use a less reliable backend, you should -keep the following warning in mind and read this section carefully. - -.. WARNING:: - - S3QL is not able to compensate for any failures of the backend. In - particular, it is not able reconstruct any data that has been lost - or corrupted by the backend. The persistence and durability of data - stored in an S3QL file system is limited and determined by the - backend alone. - - -On the plus side, if a backend looses or corrupts some of the stored -data, S3QL *will* detect the problem. Missing data will be detected -when running `fsck.s3ql` or when attempting to access the data in the -mounted file system. In the later case you will get an IO Error, and -on unmounting S3QL will warn you that the file system is damaged and -you need to run `fsck.s3ql`. - -`fsck.s3ql` will report all the affected files and move them into the -`/lost+found` directory of the file system. - -You should be aware that, because of S3QL's data de-duplication -feature, the consequences of a data loss in the backend can be -significantly more severe than you may expect. More concretely, a data -loss in the backend at time *x* may cause data that is written *after* -time *x* to be lost as well. What may happen is this: - -#. You store an important file in the S3QL file system. -#. The backend looses the data blocks of this file. As long as you - do not access the file or run `fsck.s3ql`, S3QL - is not aware that the data has been lost by the backend. -#. You save an additional copy of the important file in a different - location on the same S3QL file system. -#. S3QL detects that the contents of the new file are identical to the - data blocks that have been stored earlier. Since at this point S3QL - is not aware that these blocks have been lost by the backend, it - does not save another copy of the file contents in the backend but - relies on the (presumably) existing blocks instead. -#. Therefore, even though you saved another copy, you still do not - have a backup of the important file (since both copies refer to the - same data blocks that have been lost by the backend). - -As one can see, this effect becomes the less important the more often -one runs `fsck.s3ql`, since `fsck.s3ql` will make S3QL aware of any -blocks that the backend may have lost. Figuratively, this establishes -a "checkpoint": data loss in the backend that occurred before running -`fsck.s3ql` can not affect any file system operations performed after -running `fsck.s3ql`. - - -Nevertheless, (as said at the beginning) the recommended way to use -S3QL is in combination with a sufficiently reliable storage backend. -In that case none of the above will ever be a concern. - - -The `authinfo` file -=================== - -Most backends first try to read the file `~/.s3ql/authinfo` to determine -the username and password for connecting to the remote host. If this -fails, both username and password are read from the terminal. - -The `authinfo` file has to contain entries of the form :: - - backend <backend> machine <host> login <user> password <password> - -So to use the login `joe` with password `jibbadup` when using the FTP -backend to connect to the host `backups.joesdomain.com`, you would -specify :: - - backend ftp machine backups.joesdomain.com login joe password jibbadup +The following backends are currently available in S3QL: + +Google Storage +============== + +`Google Storage <http://code.google.com/apis/storage/>`_ is an online +storage service offered by Google. It is the most feature-rich service +supported by S3QL and S3QL offers the best performance when used with +the Google Storage backend. + +To use the Google Storage backend, you need to have (or sign up for) a +Google account, and then `activate Google Storage +<http://code.google.com/apis/storage/docs/signup.html>`_ for your +account. The account is free, you will pay only for the amount of +storage and traffic that you actually use. Once you have created the +account, make sure to `activate legacy access +<http://code.google.com/apis/storage/docs/reference/v1/apiversion1.html#enabling>`_. + +To create a Google Storage bucket, you can use e.g. the `Google +Storage Manager +<https://sandbox.google.com/storage/>`_. The +storage URL for accessing the bucket in S3QL is then :: + + gs://<bucketname>/<prefix> + +Here *bucketname* is the name of the bucket, and *prefix* can be +an arbitrary prefix that will be prepended to all object names used by +S3QL. This allows you to store several S3QL file systems in the same +Google Storage bucket. + +Note that the backend login and password for accessing your Google +Storage bucket are not your Google account name and password, but the +*Google Storage developer access key* and *Google Storage developer +secret* that you can manage with the `Google Storage key management +tool +<https://code.google.com/apis/console/#:storage:legacy>`_. + +If you would like S3QL to connect using HTTPS instead of standard +HTTP, start the storage url with ``gss://`` instead of ``gs://``. Note +that at this point S3QL does not perform any server certificate +validation (see `issue 267 +<http://code.google.com/p/s3ql/issues/detail?id=267>`_). + + +Amazon S3 +========= + +`Amazon S3 <http://aws.amazon.com/s3>`_ is the online storage service +offered by `Amazon Web Services (AWS) <http://aws.amazon.com/>`_. To +use the S3 backend, you first need to sign up for an AWS account. The +account is free, you will pay only for the amount of storage and +traffic that you actually use. After that, you need to create a bucket +that will hold the S3QL file system, e.g. using the `AWS Management +Console <https://console.aws.amazon.com/s3/home>`_. For best +performance, it is recommend to create the bucket in the +geographically closest storage region, but not the US Standard +region (see below). + +The storage URL for accessing S3 buckets in S3QL has the form :: + + s3://<bucketname>/<prefix> + +Here *bucketname* is the name of the bucket, and *prefix* can be +an arbitrary prefix that will be prepended to all object names used by +S3QL. This allows you to store several S3QL file systems in the same +S3 bucket. + +Note that the backend login and password for accessing S3 are not the +user id and password that you use to log into the Amazon Webpage, but +the *AWS access key id* and *AWS secret access key* shown under `My +Account/Access Identifiers +<https://aws-portal.amazon.com/gp/aws/developer/account/index.html?ie=UTF8&action=access-key>`_. - -Consistency Guarantees -====================== - -The different backends provide different types of *consistency -guarantees*. Informally, a consistency guarantee tells you how fast -the backend will apply changes to the stored data. +If you would like S3QL to connect using HTTPS instead of standard +HTTP, start the storage url with ``s3s://`` instead of ``s3://``. Note +that, as of May 2011, Amazon S3 is faster when accessed using a +standard HTTP connection, and that S3QL does not perform any server +certificate validation (see `issue 267 +<http://code.google.com/p/s3ql/issues/detail?id=267>`_). -S3QL defines the following three levels: -* **Read-after-Write Consistency.** This is the strongest consistency - guarantee. If a backend offers read-after-write consistency, it - guarantees that as soon as you have committed any changes to the - backend, subsequent requests will take into account these changes. +Reduced Redundancy Storage (RRS) +-------------------------------- -* **Read-after-Create Consistency.** If a backend provides only - read-after-create consistency, only the creation of a new object is - guaranteed to be taken into account for subsequent requests. This - means that, for example, if you overwrite data in an existing - object, subsequent requests may still return the old data for a - certain period of time. +S3QL does not allow the use of `reduced redundancy storage +<http://aws.amazon.com/s3/#protecting>`_. The reason for that is a +combination of three factors: -* **Eventual consistency.** This is the lowest consistency level. - Basically, any changes that you make to the backend may not be - visible for a certain amount of time after the change has been made. - However, you are guaranteed that no change will be lost. All changes - will *eventually* become visible. - - . +* RRS has a relatively low reliability, on average you loose one + out of every ten-thousand objects a year. So you can expect to + occasionally loose some data. +* When `fsck.s3ql` asks S3 for a list of the stored objects, this list + includes even those objects that have been lost. Therefore + `fsck.s3ql` *can not detect lost objects* and lost data will only + become apparent when you try to actually read from a file whose data + has been lost. This is a (very unfortunate) peculiarity of Amazon + S3. -As long as your backend provides read-after-write or read-after-create -consistency, you do not have to worry about consistency guarantees at -all. However, if you plan to use a backend with only eventual -consistency, you have to be a bit careful in some situations. - - -.. _eventual_consistency: - -Dealing with Eventual Consistency ---------------------------------- +* Due to the data de-duplication feature of S3QL, unnoticed lost + objects may cause subsequent data loss later in time (see + :ref:`backend_reliability` for details). -.. NOTE:: - The following applies only to storage backends that do not provide - read-after-create or read-after-write consistency. Currently, - this is only the Amazon S3 backend *if used with the US-Standard - storage region*. If you use a different storage backend, or the S3 - backend with a different storage region, this section does not apply - to you. +Potential issues when using the US Standard storage region +---------------------------------------------------------- -While the file system is mounted, S3QL is able to automatically handle -all issues related to the weak eventual consistency guarantee. -However, some issues may arise during the mount process and when the -file system is checked. +In the US Standard storage region, Amazon S3 does not guarantee read +after create consistency. This means that after a new object has been +stored, requests to read this object may still fail for a little +while. While the file system is mounted, S3QL is able to automatically +handle all issues related to this so-called eventual consistency. +However, problems may arise during the mount process and when the file +system is checked: Suppose that you mount the file system, store some new data, delete -some old data and unmount it again. Now remember that eventual -consistency means that there is no guarantee that these changes will -be visible immediately. At least in theory it is therefore possible -that if you mount the file system again, S3QL does not see any of the -changes that you have done and presents you an "old version" of the -file system without them. Even worse, if you notice the problem and -unmount the file system, S3QL will upload the old status (which S3QL -necessarily has to consider as current) and thereby permanently -override the newer version (even though this change may not become -immediately visible either). - -The same problem applies when checking the file system. If the backend +some old data and unmount it again. Now there is no guarantee that +these changes will be visible immediately. At least in theory it is +therefore possible that if you mount the file system again, S3QL +does not see any of the changes that you have done and presents you +an "old version" of the file system without them. Even worse, if you +notice the problem and unmount the file system, S3QL will upload the +old status (which S3QL necessarily has to consider as current) and +thereby permanently override the newer version (even though this +change may not become immediately visible either). + +The same problem applies when checking the file system. If S3 provides S3QL with only partially updated data, S3QL has no way to find out if this a real consistency problem that needs to be fixed or if it is only a temporary problem that will resolve itself automatically (because there are still changes that have not become visible yet). -While this may seem to be a rather big problem, the likelihood of it -to occur is rather low. In practice, most storage providers rarely -need more than a few seconds to apply incoming changes, so to trigger -this problem one would have to unmount and remount the file system in -a very short time window. Many people therefore make sure that they -wait a few minutes between successive mounts (or file system checks) -and decide that the remaining risk is negligible. - -Nevertheless, the eventual consistency guarantee does not impose an -upper limit on the time that it may take for change to become visible. -Therefore there is no "totally safe" waiting time that would totally -eliminate this problem; a theoretical possibility always remains. - - - -The Amazon S3 Backend -===================== - -To store your file system in an Amazon S3 bucket, use a storage URL of -the form `s3://<bucketname>`. Bucket names must conform to the `S3 -Bucket Name Restrictions`_. - -The S3 backend offers exceptionally strong reliability guarantees. As -of August 2010, Amazon guarantees a durability of 99.999999999% per -year. In other words, if you store a thousand million objects then on -average you would loose less than one object in a hundred years. - -The Amazon S3 backend provides read-after-create consistency for the -EU, Asia-Pacific and US-West storage regions. *For the US-Standard -storage region, Amazon S3 provides only eventual consistency* (please -refer to :ref:`eventual_consistency` for information about -what this entails). - -When connecting to Amazon S3, S3QL uses an unencrypted HTTP -connection, so if you want your data to stay confidential, you have -to create the S3QL file system with encryption (this is also the default). - -When reading the authentication information for the S3 backend from -the `authinfo` file, the `host` field is ignored, i.e. the first entry -with `s3` as a backend will be used. For example :: - - backend s3 machine any login myAWSaccessKeyId password myAwsSecretAccessKey - -Note that the bucket names come from a global pool, so chances are -that your favorite name has already been taken by another S3 user. -Usually a longer bucket name containing some random numbers, like -`19283712_yourname_s3ql`, will work better. - -If you do not already have one, you need to obtain an Amazon S3 -account from `Amazon AWS <http://aws.amazon.com/>`_. The account is -free, you will pay only for the amount of storage that you actually -use. - -Note that the login and password for accessing S3 are not the user id -and password that you use to log into the Amazon Webpage, but the "AWS -access key id" and "AWS secret access key" shown under `My -Account/Access Identifiers -<https://aws-portal.amazon.com/gp/aws/developer/account/index.html?ie=UTF8&action=access-key>`_. - -.. _`S3 Bucket Name Restrictions`: http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/BucketRestrictions.html - -.. NOTE:: - - S3QL also allows you to use `reduced redundancy storage - <http://aws.amazon.com/s3/#protecting>`_ by using ``s3rr://`` - instead of ``s3://`` in the storage url. However, this not - recommended. The reason is a combination of three factors: - - * RRS has a relatively low reliability, on average you loose one - out of every ten-thousand objects a year. So you can expect to - occasionally loose some data. - - * When `fsck.s3ql` asks Amazon S3 for a list of the stored objects, - this list includes even those objects that have been lost. - Therefore `fsck.s3ql` *can not detect lost objects* and lost data - will only become apparent when you try to actually read from a - file whose data has been lost. This is a (very unfortunate) - peculiarity of Amazon S3. +The likelihood of this to happen is rather low. In practice, most +objects are ready for retrieval just a few seconds after they have +been stored, so to trigger this problem one would have to unmount and +remount the file system in a very short time window. However, since S3 +does not place any upper limit on the length of this window, it is +recommended to not place S3QL buckets in the US Standard storage +region. As of May 2011, all other storage regions provide stronger +consistency guarantees that completely eliminate any of the described +problems. - * Due to the data de-duplication feature of S3QL, unnoticed lost - objects may cause subsequent data loss later in time (see `On - Backend Reliability`_ for details). - In other words, you should really only store an S3QL file system - using RRS if you know exactly what you are getting into. - +S3 compatible +============= +S3QL is also able to access other, S3 compatible storage services for +which no specific backend exists. Note that when accessing such +services, only the lowest common denominator of available features can +be used, so it is generally recommended to use a service specific +backend instead. -The Local Backend -================= +The storage URL for accessing an arbitrary S3 compatible storage +service is :: -The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -`local://<path>`. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. `local:///var/archive`. + s3c://<hostname>:<port>/<bucketname>/<prefix> -The local backend provides read-after-write consistency. +or :: -The SFTP Backend -================ + s3cs://<hostname>:<port>/<bucketname>/<prefix> -The SFTP backend uses the SFTP protocol, which is a file transfer -protocol similar to ftp, but uses an encrypted SSH connection. -It provides read-after-write consistency. +to use HTTPS connections. Note, however, that at this point S3QL does +not verify the server certificate (cf. `issue 267 +<http://code.google.com/p/s3ql/issues/detail?id=267>`_). -Note that the SFTP backend is rather slow and has not been tested -as extensively as the S3 and Local backends. -The storage URL for SFTP connections has the form :: +Local +===== - sftp://<host>[:port]/<path> +S3QL is also able to store its data on the local file system. This can +be used to backup data on external media, or to access external +services that S3QL can not talk to directly (e.g., it is possible to +store data over SSH by first mounting the remote system using +`sshfs`_, then using the local backend to store the data in the sshfs +mountpoint). -The SFTP backend will always ask you for a password if you haven't -defined one in `~/.s3ql/authinfo`. However, public key authentication -is tried first and the password will only be used if the public key -authentication fails. +The storage URL for local storage is :: -The public and private keys will be read from the standard files in -`~/.ssh/`. Note that S3QL will refuse to connect to a computer with -unknown host key; to add the key to your local keyring you have to -establish a connection to that computer with the standard SSH command -line programs first. + local://<path> + +Note that you have to write three consecutive slashes to specify an +absolute path, e.g. `local:///var/archive`. Also, relative paths will +automatically be converted to absolute paths before the authentication +file is read, i.e. if you are in the `/home/john` directory and try to +mount `local://bucket`, the corresponding section in the +authentication file must match the storage url +`local:///home/john/bucket`. +SSH/SFTP +======== +Previous versions of S3QL included an SSH/SFTP backend. With newer +S3QL versions, it is recommended to instead combine the local backend +with `sshfs <http://fuse.sourceforge.net/sshfs.html>`_ (cf. :ref:`ssh_tipp`). diff --git a/rst/contrib.rst b/rst/contrib.rst index 3ee2323..7cbe997 100644 --- a/rst/contrib.rst +++ b/rst/contrib.rst @@ -14,8 +14,10 @@ the `contrib` directory of the source distribution or in benchmark.py ============ -This program measures your uplink bandwidth and compression speed and -recommends a compression algorithm for optimal throughput. +This program measures S3QL write performance, uplink bandwidth and +compression speed to determine the limiting factor. It also gives +recommendation for compression algorithm and number of upload threads +to achieve maximum performance. s3_copy.py diff --git a/rst/general.rst b/rst/general.rst new file mode 100644 index 0000000..e95ab8e --- /dev/null +++ b/rst/general.rst @@ -0,0 +1,149 @@ +.. -*- mode: rst -*- + +=================== +General Information +=================== + +Terminology +=========== + +S3QL can store data at different service providers and using different +protocols. The term *backend* refers to both the part of S3QL that +implements communication with a specific storage service and the +storage service itself. Most backends can hold more than one S3QL file +system and thus require some additional information that specifies the +file system location within the backend. This location is called a +*bucket* (for historical reasons). + +Many S3QL commands expect a *storage url* as a parameter. A storage +url specifies both the backend and the bucket and thus uniquely +identifies an S3QL file system. The form of the storage url depends on +the backend and is described together with the +:ref:`storage_backends`. + +.. _bucket_pw: + +Storing Authentication Information +================================== + +Normally, S3QL reads username and password for the backend as well as +an encryption passphrase for the bucket from the terminal. Most +commands also accept an :cmdopt:`--authfile` parameter that can be +used to read this information from a file instead. + +The authentication file consists of sections, led by a ``[section]`` +header and followed by ``name: value`` entries. The section headers +themselves are not used by S3QL but have to be unique within the file. + +In each section, the following entries can be defined: + +:storage-url: + Specifies the storage url to which this section applies. If a + storage url starts with the value of this entry, the section is + considered applicable. + +:backend-login: + Specifies the username to use for authentication with the backend. + +:backend-password: + Specifies the password to use for authentication with the backend. + +:bucket-passphrase: + Specifies the passphrase to use to decrypt the bucket (if it is + encrypted). + + +When reading the authentication file, S3QL considers every applicable +section in order and uses the last value that it found for each entry. +For example, consider the following authentication file:: + + [s3] + storage-url: s3:// + backend-login: joe + backend-password: notquitesecret + + [bucket1] + storage-url: s3://joes-first-bucket + bucket-passphrase: neitheristhis + + [bucket2] + storage-url: s3://joes-second-bucket + bucket-passphrase: swordfish + + [bucket3] + storage-url: s3://joes-second-bucket/with-prefix + backend-login: bill + backend-password: bi23ll + bucket-passphrase: ll23bi + +With this authentication file, S3QL would try to log in as "joe" +whenever the s3 backend is used, except when accessing a storage url +that begins with "s3://joes-second-bucket/with-prefix". In that case, +the last section becomes active and S3QL would use the "bill" +credentials. Furthermore, bucket encryption passphrases will be used +for storage urls that start with "s3://joes-first-bucket" or +"s3://joes-second-bucket". + +The authentication file is parsed by the `Python ConfigParser +module <http://docs.python.org/library/configparser.html>`_. + +.. _backend_reliability: + +On Backend Reliability +====================== + +S3QL has been designed for use with a storage backend where data loss +is so infrequent that it can be completely neglected (e.g. the Amazon +S3 backend). If you decide to use a less reliable backend, you should +keep the following warning in mind and read this section carefully. + +.. WARNING:: + + S3QL is not able to compensate for any failures of the backend. In + particular, it is not able reconstruct any data that has been lost + or corrupted by the backend. The persistence and durability of data + stored in an S3QL file system is limited and determined by the + backend alone. + + +On the plus side, if a backend looses or corrupts some of the stored +data, S3QL *will* detect the problem. Missing data will be detected +when running `fsck.s3ql` or when attempting to access the data in the +mounted file system. In the later case you will get an IO Error, and +on unmounting S3QL will warn you that the file system is damaged and +you need to run `fsck.s3ql`. + +`fsck.s3ql` will report all the affected files and move them into the +`/lost+found` directory of the file system. + +You should be aware that, because of S3QL's data de-duplication +feature, the consequences of a data loss in the backend can be +significantly more severe than you may expect. More concretely, a data +loss in the backend at time *x* may cause data that is written *after* +time *x* to be lost as well. What may happen is this: + +#. You store an important file in the S3QL file system. +#. The backend looses the data blocks of this file. As long as you + do not access the file or run `fsck.s3ql`, S3QL + is not aware that the data has been lost by the backend. +#. You save an additional copy of the important file in a different + location on the same S3QL file system. +#. S3QL detects that the contents of the new file are identical to the + data blocks that have been stored earlier. Since at this point S3QL + is not aware that these blocks have been lost by the backend, it + does not save another copy of the file contents in the backend but + relies on the (presumably) existing blocks instead. +#. Therefore, even though you saved another copy, you still do not + have a backup of the important file (since both copies refer to the + same data blocks that have been lost by the backend). + +As one can see, this effect becomes the less important the more often +one runs `fsck.s3ql`, since `fsck.s3ql` will make S3QL aware of any +blocks that the backend may have lost. Figuratively, this establishes +a "checkpoint": data loss in the backend that occurred before running +`fsck.s3ql` can not affect any file system operations performed after +running `fsck.s3ql`. + +Nevertheless, the recommended way to use S3QL is in combination with a +sufficiently reliable storage backend. In that case none of the above +will ever be a concern. diff --git a/rst/include/backends.rst b/rst/include/backends.rst deleted file mode 100644 index 5892edd..0000000 --- a/rst/include/backends.rst +++ /dev/null @@ -1,28 +0,0 @@ -.. -*- mode: rst -*- - -The form of the storage url depends on the backend that is used. The -following backends are supported: - -Amazon S3 ---------- - -To store your file system in an Amazon S3 bucket, use a storage URL of -the form `s3://<bucketname>`. Bucket names must conform to the S3 Bucket -Name Restrictions. - - -Local ------- - -The local backend stores file system data in a directory on your -computer. The storage URL for the local backend has the form -`local://<path>`. Note that you have to write three consecutive -slashes to specify an absolute path, e.g. `local:///var/archive`. - -SFTP ----- - -The storage URL for SFTP connections has the form :: - - sftp://<host>[:port]/<path> - diff --git a/rst/index.rst b/rst/index.rst index f3b5b72..960a63a 100644 --- a/rst/index.rst +++ b/rst/index.rst @@ -9,6 +9,7 @@ about installation + general backends mkfs adm diff --git a/rst/installation.rst b/rst/installation.rst index b57325e..e153de0 100644 --- a/rst/installation.rst +++ b/rst/installation.rst @@ -27,13 +27,11 @@ running S3QL. Generally, you should first check if your distribution already provides a suitable packages and only install from source if that is not the case. -* Kernel version 2.6.9 or newer. Starting with kernel 2.6.26 - you will get significantly better write performance, so you should - actually use *2.6.26 or newer whenever possible*. - -* The `FUSE Library <http://fuse.sourceforge.net/>`_ should already be - installed on your system. However, you have to make sure that you - have at least version 2.8.0. +* Kernel: Linux 2.6.9 or newer or FreeBSD with `FUSE4BSD + <http://www.freshports.org/sysutils/fusefs-kmod/>`_. Starting with + kernel 2.6.26 you will get significantly better write performance, + so under Linux you should actually use *2.6.26 or newer whenever + possible*. * The `PyCrypto++ Python Module <http://pypi.python.org/pypi/pycryptopp>`_. To check if this module @@ -69,12 +67,6 @@ that is not the case. this module. If you are upgrading from such a version, make sure to completely remove the old S3QL version first. -* If you want to use the SFTP backend, then you also need the - `Paramiko Python Module <http://www.lag.net/paramiko/>`_. To check - if this module is installed, try to execute `python -c 'import - paramiko'`. - - .. _inst-s3ql: Installing S3QL diff --git a/rst/issues.rst b/rst/issues.rst index 29b76ce..ac2cb8c 100644 --- a/rst/issues.rst +++ b/rst/issues.rst @@ -4,6 +4,11 @@ Known Issues ============ +* S3QL does not verify TLS/SSL server certificates, so a + man-in-the-middle attack is principally possible. See `issue 267 + <http://code.google.com/p/s3ql/issues/detail?id=267>`_ for more + details. + * S3QL is rather slow when an application tries to write data in unreasonably small chunks. If a 1 MB file is copied in chunks of 1 KB, this will take more than 10 times as long as when it's copied @@ -74,7 +79,7 @@ Known Issues * S3QL relies on the backends not to run out of space. This is a given for big storage providers like Amazon S3, but you may stumble upon - this if you store buckets e.g. on a small sftp server. + this if you store buckets e.g. on smaller servers or servies. If there is no space left in the backend, attempts to write more data into the S3QL file system will fail and the file system will be diff --git a/rst/man/adm.rst b/rst/man/adm.rst index c23865e..9010cd8 100644 --- a/rst/man/adm.rst +++ b/rst/man/adm.rst @@ -23,8 +23,8 @@ The |command| command performs various operations on S3QL buckets. The file system contained in the bucket *must not be mounted* when using |command| or things will go wrong badly. -.. include:: ../include/backends.rst - +The storage url depends on the backend that is used. The S3QL User's +Guide should be consulted for a description of the available backends. Options ======= @@ -52,15 +52,6 @@ download-metadata Interactively download backups of the file system metadata. -Files -===== - -Authentication data for backends and bucket encryption passphrases are -read from :file:`authinfo` in :file:`~/.s3ql` or the directory -specified with :cmdopt:`--homedir`. Log files are placed in the same -directory. - - .. include:: ../include/postman.rst .. |command| replace:: :program:`s3qladm` diff --git a/rst/man/cp.rst b/rst/man/cp.rst index d0cbb41..a397fe9 100644 --- a/rst/man/cp.rst +++ b/rst/man/cp.rst @@ -40,7 +40,7 @@ Note that: usage of `s3qlcp` is to regularly duplicate the same source directory, say `documents`, to different target directories. For a e.g. monthly replication, the target directories would typically be - named something like `documents_Januray` for the replication in + named something like `documents_January` for the replication in January, `documents_February` for the replication in February etc. In this case it is clear that the target directories should be regarded as snapshots of the source directory. @@ -51,12 +51,6 @@ Note that: completely (so that S3QL had to fetch all the data over the network from the backend) before writing them into the destination folder. -* Before starting with the replication, S3QL has to flush the local - cache. So if you just copied lots of new data into the file system - that has not yet been uploaded, replication will take longer than - usual. - - Snapshotting vs Hardlinking --------------------------- diff --git a/rst/man/ctrl.rst b/rst/man/ctrl.rst index 4afa33b..8173162 100644 --- a/rst/man/ctrl.rst +++ b/rst/man/ctrl.rst @@ -24,6 +24,12 @@ Description The |command| command performs various actions on the S3QL file system mounted in :var:`mountpoint`. +|command| can only be called by the user that mounted the file system +and (if the file system was mounted with :cmdopt:`--allow-other` or +:cmdopt:`--allow-root`) the root user. This limitation might be +removed in the future (see `issue 155 +<http://code.google.com/p/s3ql/issues/detail?id=155>`_). + The following actions may be specified: flushcache diff --git a/rst/man/fsck.rst b/rst/man/fsck.rst index ef6ed2d..8f901c7 100644 --- a/rst/man/fsck.rst +++ b/rst/man/fsck.rst @@ -18,10 +18,9 @@ Description The |command| command checks the new file system in the location specified by *storage url* for errors and attempts to repair any -problems. - -.. include:: ../include/backends.rst - +problems. The storage url depends on the backend that is used. The +S3QL User's Guide should be consulted for a description of the +available backends. Options ======= @@ -31,14 +30,6 @@ The |command| command accepts the following options. .. pipeinclude:: ../../bin/fsck.s3ql --help :start-after: show this help message and exit -Files -===== - -Authentication data for backends and bucket encryption passphrases are -read from :file:`authinfo` in :file:`~/.s3ql` or the directory -specified with :cmdopt:`--homedir`. Log files are placed in the same -directory. - .. include:: ../include/postman.rst .. |command| replace:: :command:`mkfs.s3ql` diff --git a/rst/man/lock.rst b/rst/man/lock.rst index f17bf32..19c3e3a 100644 --- a/rst/man/lock.rst +++ b/rst/man/lock.rst @@ -23,6 +23,12 @@ whatsoever. You can not add new files or directories and you can not change or delete existing files and directories. The only way to get rid of an immutable tree is to use the :program:`s3qlrm` command. +|command| can only be called by the user that mounted the file system +and (if the file system was mounted with :cmdopt:`--allow-other` or +:cmdopt:`--allow-root`) the root user. This limitation might be +removed in the future (see `issue 155 +<http://code.google.com/p/s3ql/issues/detail?id=155>`_). + Rationale ========= diff --git a/rst/man/mkfs.rst b/rst/man/mkfs.rst index c61270a..3a61717 100644 --- a/rst/man/mkfs.rst +++ b/rst/man/mkfs.rst @@ -17,10 +17,15 @@ Description .. include:: ../include/about.rst The |command| command creates a new file system in the location -specified by *storage url*. +specified by *storage url*. The storage url depends on the backend +that is used. The S3QL User's Guide should be consulted for a +description of the available backends. + +Unless you have specified the `--plain` option, `mkfs.s3ql` will ask +you to enter an encryption password. This password will *not* be read +from an authentication file specified with the :cmdopt:`--authfile` +option to prevent accidental creation of an encrypted bucket. -.. include:: ../include/backends.rst - Options ======= @@ -30,13 +35,6 @@ The |command| command accepts the following options. .. pipeinclude:: ../../bin/mkfs.s3ql --help :start-after: show this help message and exit -Files -===== - -Authentication data for backends and bucket encryption passphrases are -read from :file:`authinfo` in :file:`~/.s3ql` or the directory -specified with :cmdopt:`--homedir`. Log files are placed in the same -directory. .. include:: ../include/postman.rst diff --git a/rst/man/mount.rst b/rst/man/mount.rst index 3905c03..fb10a11 100644 --- a/rst/man/mount.rst +++ b/rst/man/mount.rst @@ -19,10 +19,10 @@ Description .. include:: ../include/about.rst The |command| command mounts the S3QL file system stored in *storage -url* in the directory *mount point*. +url* in the directory *mount point*. The storage url depends on the +backend that is used. The S3QL User's Guide should be consulted for a +description of the available backends. -.. include:: ../include/backends.rst - Options ======= @@ -33,14 +33,6 @@ The |command| command accepts the following options. :start-after: show this help message and exit -Files -===== - -Authentication data for backends and bucket encryption passphrases are -read from :file:`authinfo` in :file:`~/.s3ql` or the directory -specified with :cmdopt:`--homedir`. Log files are placed in the same -directory. - .. include:: ../include/postman.rst diff --git a/rst/man/pcp.rst b/rst/man/pcp.rst index cd7a66c..c7b3bef 100644 --- a/rst/man/pcp.rst +++ b/rst/man/pcp.rst @@ -21,6 +21,10 @@ The |command| command is a is a wrapper that starts several allows much better copying performance on file system that have relatively high latency when retrieving individual files like S3QL. +**Note**: Using this program only improves performance when copying +*from* an S3QL file system. When copying *to* an S3QL file system, +using |command| is more likely to *decrease* performance. + Options ======= diff --git a/rst/man/rm.rst b/rst/man/rm.rst index 0832e27..dda1eda 100644 --- a/rst/man/rm.rst +++ b/rst/man/rm.rst @@ -25,6 +25,12 @@ you to delete immutable trees (which can be created with Be warned that there is no additional confirmation. The directory will be removed entirely and immediately. + +|command| can only be called by the user that mounted the file system +and (if the file system was mounted with :cmdopt:`--allow-other` or +:cmdopt:`--allow-root`) the root user. This limitation might be +removed in the future (see `issue 155 +<http://code.google.com/p/s3ql/issues/detail?id=155>`_). Options diff --git a/rst/mkfs.rst b/rst/mkfs.rst index 0b9fa97..3eceb0a 100644 --- a/rst/mkfs.rst +++ b/rst/mkfs.rst @@ -14,7 +14,8 @@ This command accepts the following options: .. pipeinclude:: ../bin/mkfs.s3ql --help :start-after: show this help message and exit -Unless you have specified the `--plain` option, `mkfs.s3ql` will ask you -to enter an encryption password. If you do not want to enter this -password every time that you mount the file system, you can store it -in the `~/.s3ql/authinfo` file, see :ref:`bucket_pw`. +Unless you have specified the `--plain` option, `mkfs.s3ql` will ask +you to enter an encryption password. This password will *not* be read +from an authentication file specified with the :cmdopt:`--authfile` +option to prevent accidental creation of an encrypted bucket. + diff --git a/rst/mount.rst b/rst/mount.rst index 609c4a4..4f220cd 100644 --- a/rst/mount.rst +++ b/rst/mount.rst @@ -22,33 +22,6 @@ This command accepts the following options: .. pipeinclude:: ../bin/mount.s3ql --help :start-after: show this help message and exit -.. _bucket_pw: - -Storing Encryption Passwords -============================ - -If you are trying to mount an encrypted bucket, `mount.s3ql` will first -try to read the password from the `.s3ql/authinfo` file (the same file -that is used to read the backend authentication data) and prompt the -user to enter the password only if this fails. - -The `authinfo` entries to specify bucket passwords are of the form :: - - storage-url <storage-url> password <password> - -So to always use the password `topsecret` when mounting `s3://joes_bucket`, -the entry would be :: - - storage-url s3://joes_bucket password topsecret - -.. NOTE:: - - If you are using the local backend, the storage url will - always be converted to an absolute path. So if you are in the - `/home/john` directory and try to mount `local://bucket`, the matching - `authinfo` entry has to have a storage url of - `local:///home/john/bucket`. - Compression Algorithms ====================== @@ -83,9 +56,8 @@ Parallel Compression ==================== If you are running S3QL on a system with multiple cores, you might -want to set ``--compression-threads`` to a value bigger than one. This -will instruct S3QL to compress and encrypt several blocks at the same -time. +want to set the ``--threads`` value larger than one. This will +instruct S3QL to compress and encrypt several blocks at the same time. If you want to do this in combination with using the LZMA compression algorithm, you should keep an eye on memory usage though. Every diff --git a/rst/special.rst b/rst/special.rst index c5acade..a217a93 100644 --- a/rst/special.rst +++ b/rst/special.rst @@ -11,8 +11,8 @@ Snapshotting and Copy-on-Write ============================== The command `s3qlcp` can be used to duplicate a directory tree without -physically copying the file contents. This is possible due to the data -de-duplication feature of S3QL. +physically copying the file contents. This is made possible by the +data de-duplication feature of S3QL. The syntax of `s3qlcp` is:: diff --git a/rst/tips.rst b/rst/tips.rst index b857f75..705bc20 100644 --- a/rst/tips.rst +++ b/rst/tips.rst @@ -4,8 +4,21 @@ Tips & Tricks ============= +.. _ssh_tipp: -.. _copy_performance: +SSH Backend +=========== + +By combining S3QL's local backend with `sshfs +<http://fuse.sourceforge.net/sshfs.html>`_, it is possible to store an +S3QL file system on arbitrary SSH servers: first mount the remote +target directory into the local filesystem, :: + + sshfs user@my.server.com:/mnt/s3ql /mnt/sshfs + +and then give the mountpoint to S3QL as a local destination:: + + mount.s3ql local:///mnt/sshfs/mybucket /mnt/s3ql Permanently mounted backup file system @@ -28,11 +41,16 @@ If you decide to do so, you should make sure to :cmdopt:`--metadata-upload-interval` option of :program:`mount.s3ql` to zero). - +.. _copy_performance: Improving copy performance ========================== +.. NOTE:: + + The following applies only when copying data **from** an S3QL file + system, **not** when copying data **to** an S3QL file system. + If you want to copy a lot of smaller files *from* an S3QL file system (e.g. for a system restore) you will probably notice that the performance is rather bad. @@ -44,6 +62,7 @@ network latency), no matter how big or small the file is. So when you copy lots of small files, 99% of the time is actually spend waiting for network data. + Theoretically, this problem is easy to solve: you just have to copy several files at the same time. In practice, however, almost all unix utilities (``cp``, ``rsync``, ``tar`` and friends) insist on copying @@ -4,7 +4,7 @@ setup.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function @@ -12,7 +12,7 @@ from __future__ import division, print_function import sys import os import subprocess -import logging +import logging.handlers from glob import glob # Work around setuptools bug @@ -112,11 +112,11 @@ def main(): author_email='Nikolaus@rath.org', url='http://code.google.com/p/s3ql/', download_url='http://code.google.com/p/s3ql/downloads/list', - license='LGPL', + license='GPLv3', classifiers=['Development Status :: 4 - Beta', 'Environment :: No Input/Output (Daemon)', 'Environment :: Console', - 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)', + 'License :: OSI Approved :: GNU Library or Lesser General Public License (GPLv3)', 'Topic :: Internet', 'Operating System :: POSIX', 'Topic :: System :: Archiving'], @@ -145,12 +145,12 @@ def main(): }, install_requires=['apsw >= 3.7.0', 'pycryptopp', - 'llfuse >= 0.31', +# 'llfuse >= 0.35', 'argparse >= 1.1', 'pyliblzma >= 0.5.3' ], tests_require=['apsw >= 3.7.0', 'unittest2', 'pycryptopp', - 'llfuse >= 0.29', +# 'llfuse >= 0.35', 'argparse >= 1.1', 'pyliblzma >= 0.5.3' ], test_suite='tests', @@ -167,16 +167,12 @@ class test(setuptools_test.test): description = "Run self-tests" user_options = (setuptools_test.test.user_options + [('debug=', None, 'Activate debugging for specified modules ' - '(separated by commas, specify "all" for all modules)'), - ('awskey=', None, 'Specify AWS access key to use, secret key will be asked for. ' - 'If this option is not specified, tests requiring access ' - 'to Amazon Web Services will be skipped.')]) + '(separated by commas, specify "all" for all modules)')]) def initialize_options(self): setuptools_test.test.initialize_options(self) self.debug = None - self.awskey = None def finalize_options(self): setuptools_test.test.finalize_options(self) @@ -184,22 +180,24 @@ class test(setuptools_test.test): if self.debug: self.debug = [ x.strip() for x in self.debug.split(',') ] - def run_tests(self): # Add test modules sys.path.insert(0, os.path.join(basedir, 'tests')) import unittest2 as unittest import _common - from s3ql.common import (setup_excepthook, add_file_logging, add_stdout_logging, - LoggerFilter) - from getpass import getpass + from s3ql.common import (setup_excepthook, add_stdout_logging, LoggerFilter) # Initialize logging if not yet initialized root_logger = logging.getLogger() if not root_logger.handlers: add_stdout_logging(quiet=True) - add_file_logging(os.path.join(basedir, 'setup.log')) + handler = logging.handlers.RotatingFileHandler("setup.log", + maxBytes=10*1024**2, backupCount=0) + formatter = logging.Formatter('%(asctime)s.%(msecs)03d [%(process)s] %(threadName)s: ' + '[%(name)s] %(message)s', datefmt="%Y-%m-%d %H:%M:%S") + handler.setFormatter(formatter) + root_logger.addHandler(handler) setup_excepthook() if self.debug: root_logger.setLevel(logging.DEBUG) @@ -209,14 +207,6 @@ class test(setuptools_test.test): root_logger.setLevel(logging.INFO) else: root_logger.debug("Logging already initialized.") - - # Init AWS - if self.awskey: - if sys.stdin.isatty(): - pw = getpass("Enter AWS password: ") - else: - pw = sys.stdin.readline().rstrip() - _common.aws_credentials = (self.awskey, pw) # Define our own test loader to order modules alphabetically from pkg_resources import resource_listdir, resource_exists diff --git a/src/s3ql.egg-info/PKG-INFO b/src/s3ql.egg-info/PKG-INFO index 015c3a4..f6ed0a0 100644 --- a/src/s3ql.egg-info/PKG-INFO +++ b/src/s3ql.egg-info/PKG-INFO @@ -1,11 +1,11 @@ Metadata-Version: 1.1 Name: s3ql -Version: 1.0.1 +Version: 1.2 Summary: a full-featured file system for online data storage Home-page: http://code.google.com/p/s3ql/ Author: Nikolaus Rath Author-email: Nikolaus@rath.org -License: LGPL +License: GPLv3 Download-URL: http://code.google.com/p/s3ql/downloads/list Description: .. -*- mode: rst -*- @@ -13,13 +13,15 @@ Description: .. -*- mode: rst -*- About S3QL ============ - S3QL is a file system that stores all its data online. It supports - `Amazon S3 <http://aws.amazon.com/s3 Amazon S3>`_ as well as arbitrary - SFTP servers and effectively provides you with a hard disk of dynamic, - infinite capacity that can be accessed from any computer with internet - access. + S3QL is a file system that stores all its data online using storage + services like `Google Storage + <http://code.google.com/apis/storage/>`_, `Amazon S3 + <http://aws.amazon.com/s3 Amazon S3>`_ or `OpenStack + <http://openstack.org/projects/storage/>`_. S3QL effectively provides + a hard disk of dynamic, infinite capacity that can be accessed from + any computer with internet access. - S3QL is providing a standard, full featured UNIX file system that is + S3QL is a standard conforming, full featured UNIX file system that is conceptually indistinguishable from any local file system. Furthermore, S3QL has additional features like compression, encryption, data de-duplication, immutable trees and snapshotting @@ -106,7 +108,7 @@ Platform: Linux Classifier: Development Status :: 4 - Beta Classifier: Environment :: No Input/Output (Daemon) Classifier: Environment :: Console -Classifier: License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL) +Classifier: License :: OSI Approved :: GNU Library or Lesser General Public License (GPLv3) Classifier: Topic :: Internet Classifier: Operating System :: POSIX Classifier: Topic :: System :: Archiving diff --git a/src/s3ql.egg-info/SOURCES.txt b/src/s3ql.egg-info/SOURCES.txt index 828f8ca..4c762fa 100644 --- a/src/s3ql.egg-info/SOURCES.txt +++ b/src/s3ql.egg-info/SOURCES.txt @@ -1,5 +1,6 @@ Changes.txt INSTALL.txt +LICENSE setup.py bin/fsck.s3ql bin/mkfs.s3ql @@ -26,6 +27,7 @@ doc/html/adm.html doc/html/backends.html doc/html/contrib.html doc/html/fsck.html +doc/html/general.html doc/html/index.html doc/html/installation.html doc/html/issues.html @@ -43,6 +45,7 @@ doc/html/_sources/adm.txt doc/html/_sources/backends.txt doc/html/_sources/contrib.txt doc/html/_sources/fsck.txt +doc/html/_sources/general.txt doc/html/_sources/index.txt doc/html/_sources/installation.txt doc/html/_sources/issues.txt @@ -128,6 +131,7 @@ rst/backends.rst rst/conf.py rst/contrib.rst rst/fsck.rst +rst/general.rst rst/index.rst rst/installation.rst rst/issues.rst @@ -140,7 +144,6 @@ rst/umount.rst rst/_static/sphinxdoc.css rst/_templates/layout.html rst/include/about.rst -rst/include/backends.rst rst/include/postman.rst rst/man/adm.rst rst/man/cp.rst @@ -163,11 +166,8 @@ src/s3ql/database.py src/s3ql/fs.py src/s3ql/fsck.py src/s3ql/inode_cache.py -src/s3ql/multi_lock.py src/s3ql/ordered_dict.py src/s3ql/parse_args.py -src/s3ql/thread_group.py -src/s3ql/upload_manager.py src/s3ql.egg-info/PKG-INFO src/s3ql.egg-info/SOURCES.txt src/s3ql.egg-info/dependency_links.txt @@ -177,28 +177,13 @@ src/s3ql.egg-info/top_level.txt src/s3ql.egg-info/zip-safe src/s3ql/backends/__init__.py src/s3ql/backends/common.py -src/s3ql/backends/ftp.py -src/s3ql/backends/ftplib.py +src/s3ql/backends/gs.py +src/s3ql/backends/gss.py src/s3ql/backends/local.py src/s3ql/backends/s3.py -src/s3ql/backends/sftp.py -src/s3ql/backends/boto/__init__.py -src/s3ql/backends/boto/connection.py -src/s3ql/backends/boto/exception.py -src/s3ql/backends/boto/handler.py -src/s3ql/backends/boto/resultset.py -src/s3ql/backends/boto/storage_uri.py -src/s3ql/backends/boto/utils.py -src/s3ql/backends/boto/pyami/__init__.py -src/s3ql/backends/boto/pyami/config.py -src/s3ql/backends/boto/s3/__init__.py -src/s3ql/backends/boto/s3/acl.py -src/s3ql/backends/boto/s3/bucket.py -src/s3ql/backends/boto/s3/bucketlistresultset.py -src/s3ql/backends/boto/s3/connection.py -src/s3ql/backends/boto/s3/key.py -src/s3ql/backends/boto/s3/prefix.py -src/s3ql/backends/boto/s3/user.py +src/s3ql/backends/s3c.py +src/s3ql/backends/s3cs.py +src/s3ql/backends/s3s.py src/s3ql/cli/__init__.py src/s3ql/cli/adm.py src/s3ql/cli/cp.py @@ -214,7 +199,6 @@ tests/__init__.py tests/_common.py tests/data.tar.bz2 tests/t1_backends.py -tests/t1_multi_lock.py tests/t1_ordered_dict.py tests/t2_block_cache.py tests/t3_fs_api.py diff --git a/src/s3ql.egg-info/requires.txt b/src/s3ql.egg-info/requires.txt index be48a76..30c88d6 100644 --- a/src/s3ql.egg-info/requires.txt +++ b/src/s3ql.egg-info/requires.txt @@ -1,5 +1,4 @@ apsw >= 3.7.0 pycryptopp -llfuse >= 0.31 argparse >= 1.1 pyliblzma >= 0.5.3
\ No newline at end of file diff --git a/src/s3ql/__init__.py b/src/s3ql/__init__.py index 8efcd98..c7db3ce 100644 --- a/src/s3ql/__init__.py +++ b/src/s3ql/__init__.py @@ -3,14 +3,14 @@ __init__.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function __all__ = [ 'backends', 'cli', 'parse_args', 'block_cache', "common", 'daemonize', - 'database', 'fs', 'fsck', 'multi_lock', 'ordered_dict', 'thread_group', - 'upload_manager', 'VERSION', 'CURRENT_FS_REV' ] + 'database', 'fs', 'fsck', 'ordered_dict', + 'VERSION', 'CURRENT_FS_REV' ] -VERSION = '1.0.1' -CURRENT_FS_REV = 11 +VERSION = '1.2' +CURRENT_FS_REV = 12 diff --git a/src/s3ql/backends/__init__.py b/src/s3ql/backends/__init__.py index eb8a8da..a42e518 100644 --- a/src/s3ql/backends/__init__.py +++ b/src/s3ql/backends/__init__.py @@ -1,11 +1,11 @@ ''' -__init__.py - this file is part of S3QL (http://s3ql.googlecode.com) +backends/__init__.py - this file is part of S3QL (http://s3ql.googlecode.com) -Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> +Copyright (C) Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import -__all__ = [ 'common', 'ftp', 'ftplib', 'local', 's3', 'sftp', 'boto' ] +__all__ = [ 'common', 'local', 's3', 's3s', 'gs', 'gss', 's3c', 's3cs' ] diff --git a/src/s3ql/backends/boto/__init__.py b/src/s3ql/backends/boto/__init__.py deleted file mode 100644 index 244450b..0000000 --- a/src/s3ql/backends/boto/__init__.py +++ /dev/null @@ -1,358 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -from .pyami.config import Config, BotoConfigLocations -from .storage_uri import BucketStorageUri, FileStorageUri -import os, re, sys -import logging -import logging.config -from .exception import InvalidUriError - -Version = '1.9b' -UserAgent = 'Boto/%s (%s)' % (Version, sys.platform) -config = Config() - -def init_logging(): - pass - -class NullHandler(logging.Handler): - def emit(self, record): - pass - -log = logging.getLogger('boto') -log.addHandler(NullHandler()) - -# convenience function to set logging to a particular file -def set_file_logger(name, filepath, level=logging.INFO, format_string=None): - global log - if not format_string: - format_string = "%(asctime)s %(name)s [%(levelname)s]:%(message)s" - logger = logging.getLogger(name) - logger.setLevel(level) - fh = logging.FileHandler(filepath) - fh.setLevel(level) - formatter = logging.Formatter(format_string) - fh.setFormatter(formatter) - logger.addHandler(fh) - log = logger - -def set_stream_logger(name, level=logging.DEBUG, format_string=None): - global log - if not format_string: - format_string = "%(asctime)s %(name)s [%(levelname)s]:%(message)s" - logger = logging.getLogger(name) - logger.setLevel(level) - fh = logging.StreamHandler() - fh.setLevel(level) - formatter = logging.Formatter(format_string) - fh.setFormatter(formatter) - logger.addHandler(fh) - log = logger - -def connect_sqs(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.sqs.connection.SQSConnection` - :return: A connection to Amazon's SQS - """ - from boto.sqs.connection import SQSConnection - return SQSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_s3(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.s3.connection.S3Connection` - :return: A connection to Amazon's S3 - """ - from boto.s3.connection import S3Connection - return S3Connection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_gs(gs_access_key_id=None, gs_secret_access_key=None, **kwargs): - """ - @type gs_access_key_id: string - @param gs_access_key_id: Your Google Storage Access Key ID - - @type gs_secret_access_key: string - @param gs_secret_access_key: Your Google Storage Secret Access Key - - @rtype: L{GSConnection<boto.gs.connection.GSConnection>} - @return: A connection to Google's Storage service - """ - from boto.gs.connection import GSConnection - return GSConnection(gs_access_key_id, gs_secret_access_key, **kwargs) - -def connect_ec2(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.ec2.connection.EC2Connection` - :return: A connection to Amazon's EC2 - """ - from boto.ec2.connection import EC2Connection - return EC2Connection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_elb(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.ec2.elb.ELBConnection` - :return: A connection to Amazon's Load Balancing Service - """ - from boto.ec2.elb import ELBConnection - return ELBConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_autoscale(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.ec2.autoscale.AutoScaleConnection` - :return: A connection to Amazon's Auto Scaling Service - """ - from boto.ec2.autoscale import AutoScaleConnection - return AutoScaleConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_cloudwatch(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.ec2.cloudwatch.CloudWatchConnection` - :return: A connection to Amazon's EC2 Monitoring service - """ - from boto.ec2.cloudwatch import CloudWatchConnection - return CloudWatchConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_sdb(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.sdb.connection.SDBConnection` - :return: A connection to Amazon's SDB - """ - from boto.sdb.connection import SDBConnection - return SDBConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_fps(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.fps.connection.FPSConnection` - :return: A connection to FPS - """ - from boto.fps.connection import FPSConnection - return FPSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_cloudfront(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.fps.connection.FPSConnection` - :return: A connection to FPS - """ - from boto.cloudfront import CloudFrontConnection - return CloudFrontConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_vpc(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.vpc.VPCConnection` - :return: A connection to VPC - """ - from boto.vpc import VPCConnection - return VPCConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_rds(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.rds.RDSConnection` - :return: A connection to RDS - """ - from boto.rds import RDSConnection - return RDSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_emr(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.emr.EmrConnection` - :return: A connection to Elastic mapreduce - """ - from boto.emr import EmrConnection - return EmrConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_sns(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.sns.SNSConnection` - :return: A connection to Amazon's SNS - """ - from boto.sns import SNSConnection - return SNSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - - -def check_extensions(module_name, module_path): - """ - This function checks for extensions to boto modules. It should be called in the - __init__.py file of all boto modules. See: - http://code.google.com/p/boto/wiki/ExtendModules - - for details. - """ - option_name = '%s_extend' % module_name - version = config.get('Boto', option_name, None) - if version: - dirname = module_path[0] - path = os.path.join(dirname, version) - if os.path.isdir(path): - log.info('extending module %s with: %s' % (module_name, path)) - module_path.insert(0, path) - -_aws_cache = {} - -def _get_aws_conn(service): - global _aws_cache - conn = _aws_cache.get(service) - if not conn: - meth = getattr(sys.modules[__name__], 'connect_'+service) - conn = meth() - _aws_cache[service] = conn - return conn - -def lookup(service, name): - global _aws_cache - conn = _get_aws_conn(service) - obj = _aws_cache.get('.'.join((service,name)), None) - if not obj: - obj = conn.lookup(name) - _aws_cache['.'.join((service,name))] = obj - return obj - -def storage_uri(uri_str, default_provider='file', debug=False): - """Instantiate a StorageUri from a URI string. - - :type uri_str: string - :param uri_str: URI naming bucket + optional object. - :type default_provider: string - :param default_provider: default provider for provider-less URIs. - - :rtype: :class:`boto.StorageUri` subclass - :return: StorageUri subclass for given URI. - - uri_str must be one of the following formats: - gs://bucket/name - s3://bucket/name - gs://bucket - s3://bucket - filename - The last example uses the default provider ('file', unless overridden) - """ - - # Manually parse URI components instead of using urlparse.urlparse because - # what we're calling URIs don't really fit the standard syntax for URIs - # (the latter includes an optional host/net location part). - end_provider_idx = uri_str.find('://') - if end_provider_idx == -1: - provider = default_provider.lower() - path = uri_str - else: - provider = uri_str[0:end_provider_idx].lower() - path = uri_str[end_provider_idx + 3:] - - if provider not in ['file', 's3', 'gs']: - raise InvalidUriError('Unrecognized provider "%s"' % provider) - if provider == 'file': - # For file URIs we have no bucket name, and use the complete path - # (minus 'file://') as the object name. - return FileStorageUri(path, debug) - else: - path_parts = path.split('/', 1) - bucket_name = path_parts[0] - # Ensure the bucket name is valid, to avoid possibly confusing other - # parts of the code. (For example if we didn't catch bucket names - # containing ':', when a user tried to connect to the server with that - # name they might get a confusing error about non-integer port numbers.) - if (bucket_name and - not re.match('^[a-z0-9][a-z0-9\._-]{1,253}[a-z0-9]$', bucket_name)): - raise InvalidUriError('Invalid bucket name in URI "%s"' % uri_str) - object_name = '' - if len(path_parts) > 1: - object_name = path_parts[1] - return BucketStorageUri(provider, bucket_name, object_name, debug) diff --git a/src/s3ql/backends/boto/connection.py b/src/s3ql/backends/boto/connection.py deleted file mode 100644 index 7c225c7..0000000 --- a/src/s3ql/backends/boto/connection.py +++ /dev/null @@ -1,683 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2008 rPath, Inc. -# Copyright (c) 2009 The Echo Nest Corporation -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -# -# Parts of this code were copied or derived from sample code supplied by AWS. -# The following notice applies to that code. -# -# This software code is made available "AS IS" without warranties of any -# kind. You may copy, display, modify and redistribute the software -# code either by itself or as incorporated into your code; provided that -# you do not remove any proprietary notices. Your use of this software -# code is at your own risk and you waive any claim against Amazon -# Digital Services, Inc. or its affiliates with respect to your use of -# this software code. (c) 2006 Amazon Digital Services, Inc. or its -# affiliates. - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -""" -Handles basic connections to AWS -""" - -import base64 -import hmac -import httplib -import socket, errno -import re -import sys -import time -import urllib, urlparse -import os -import xml.sax -import Queue -from .. import boto -from .exception import BotoClientError, BotoServerError -from .resultset import ResultSet -from . import utils -from . import config, UserAgent, handler - -# -# the following is necessary because of the incompatibilities -# between Python 2.4, 2.5, and 2.6 as well as the fact that some -# people running 2.4 have installed hashlib as a separate module -# this fix was provided by boto user mccormix. -# see: http://code.google.com/p/boto/issues/detail?id=172 -# for more details. -# -try: - from hashlib import sha1 as sha - from hashlib import sha256 as sha256 - - if sys.version[:3] == "2.4": - # we are using an hmac that expects a .new() method. - class Faker: - def __init__(self, which): - self.which = which - self.digest_size = self.which().digest_size - - def new(self, *args, **kwargs): - return self.which(*args, **kwargs) - - sha = Faker(sha) - sha256 = Faker(sha256) - -except ImportError: - import sha - sha256 = None - -PORTS_BY_SECURITY = { True: 443, False: 80 } - -class ConnectionPool: - def __init__(self, hosts, connections_per_host): - self._hosts = boto.utils.LRUCache(hosts) - self.connections_per_host = connections_per_host - - def __getitem__(self, key): - if key not in self._hosts: - self._hosts[key] = Queue.Queue(self.connections_per_host) - return self._hosts[key] - - def __repr__(self): - return 'ConnectionPool:%s' % ','.join(self._hosts._dict.keys()) - -class ProviderCredentials(object): - - ProviderCredentialMap = { - 'aws' : ('aws_access_key_id', 'aws_secret_access_key'), - 'google' : ('gs_access_key_id', 'gs_secret_access_key'), - } - - def __init__(self, provider, access_key=None, secret_key=None): - self.provider = provider - self.access_key = None - self.secret_key = None - provider_map = self.ProviderCredentialMap[self.provider] - access_key_name, secret_key_name = self.ProviderCredentialMap[provider] - if access_key: - self.access_key = access_key - elif os.environ.has_key(access_key_name.upper()): - self.access_key = os.environ[access_key_name.upper()] - elif config.has_option('Credentials', access_key_name): - self.access_key = config.get('Credentials', access_key_name) - - if secret_key: - self.secret_key = secret_key - elif os.environ.has_key(secret_key_name.upper()): - self.secret_key = os.environ[secret_key_name.upper()] - elif config.has_option('Credentials', secret_key_name): - self.secret_key = config.get('Credentials', secret_key_name) - -class AWSAuthConnection(object): - - def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, debug=0, - https_connection_factory=None, path='/', provider='aws'): - """ - :type host: string - :param host: The host to make the connection to - - :type aws_access_key_id: string - :param aws_access_key_id: AWS Access Key ID (provided by Amazon) - - :type aws_secret_access_key: string - :param aws_secret_access_key: Secret Access Key (provided by Amazon) - - :type is_secure: boolean - :param is_secure: Whether the connection is over SSL - - :type https_connection_factory: list or tuple - :param https_connection_factory: A pair of an HTTP connection - factory and the exceptions to catch. - The factory should have a similar - interface to L{httplib.HTTPSConnection}. - - :type proxy: - :param proxy: - - :type proxy_port: int - :param proxy_port: The port to use when connecting over a proxy - - :type proxy_user: string - :param proxy_user: The username to connect with on the proxy - - :type proxy_pass: string - :param proxy_pass: The password to use when connection over a proxy. - - :type port: integer - :param port: The port to use to connect - """ - - self.num_retries = 5 - self.is_secure = is_secure - self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) - # define exceptions from httplib that we want to catch and retry - self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror) - # define values in socket exceptions we don't want to catch - self.socket_exception_values = (errno.EINTR,) - if https_connection_factory is not None: - self.https_connection_factory = https_connection_factory[0] - self.http_exceptions += https_connection_factory[1] - else: - self.https_connection_factory = None - if (is_secure): - self.protocol = 'https' - else: - self.protocol = 'http' - self.host = host - self.path = path - if debug: - self.debug = debug - else: - self.debug = config.getint('Boto', 'debug', debug) - if port: - self.port = port - else: - self.port = PORTS_BY_SECURITY[is_secure] - - self.provider_credentials = ProviderCredentials(provider, - aws_access_key_id, - aws_secret_access_key) - - # initialize an HMAC for signatures, make copies with each request - self.hmac = hmac.new(self.aws_secret_access_key, digestmod=sha) - if sha256: - self.hmac_256 = hmac.new(self.aws_secret_access_key, digestmod=sha256) - else: - self.hmac_256 = None - - # cache up to 20 connections per host, up to 20 hosts - self._pool = ConnectionPool(20, 20) - self._connection = (self.server_name(), self.is_secure) - self._last_rs = None - - def __repr__(self): - return '%s:%s' % (self.__class__.__name__, self.host) - - def _cached_name(self, host, is_secure): - if host is None: - host = self.server_name() - cached_name = is_secure and 'https://' or 'http://' - cached_name += host - return cached_name - - def connection(self): - return self.get_http_connection(*self._connection) - connection = property(connection) - - def aws_access_key_id(self): - return self.provider_credentials.access_key - aws_access_key_id = property(aws_access_key_id) - gs_access_key_id = aws_access_key_id - access_key = aws_access_key_id - - def aws_secret_access_key(self): - return self.provider_credentials.secret_key - aws_secret_access_key = property(aws_secret_access_key) - gs_secret_access_key = aws_secret_access_key - secret_key = aws_secret_access_key - - def get_path(self, path='/'): - pos = path.find('?') - if pos >= 0: - params = path[pos:] - path = path[:pos] - else: - params = None - if path[-1] == '/': - need_trailing = True - else: - need_trailing = False - path_elements = self.path.split('/') - path_elements.extend(path.split('/')) - path_elements = [p for p in path_elements if p] - path = '/' + '/'.join(path_elements) - if path[-1] != '/' and need_trailing: - path += '/' - if params: - path = path + params - return path - - def server_name(self, port=None): - if not port: - port = self.port - if port == 80: - signature_host = self.host - else: - # This unfortunate little hack can be attributed to - # a difference in the 2.6 version of httplib. In old - # versions, it would append ":443" to the hostname sent - # in the Host header and so we needed to make sure we - # did the same when calculating the V2 signature. In 2.6 - # it no longer does that. Hence, this kludge. - if sys.version[:3] == "2.6" and port == 443: - signature_host = self.host - else: - signature_host = '%s:%d' % (self.host, port) - return signature_host - - def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass): - self.proxy = proxy - self.proxy_port = proxy_port - self.proxy_user = proxy_user - self.proxy_pass = proxy_pass - if os.environ.has_key('http_proxy') and not self.proxy: - pattern = re.compile( - '(?:http://)?' \ - '(?:(?P<user>\w+):(?P<pass>.*)@)?' \ - '(?P<host>[\w\-\.]+)' \ - '(?::(?P<port>\d+))?' - ) - match = pattern.match(os.environ['http_proxy']) - if match: - self.proxy = match.group('host') - self.proxy_port = match.group('port') - self.proxy_user = match.group('user') - self.proxy_pass = match.group('pass') - else: - if not self.proxy: - self.proxy = config.get_value('Boto', 'proxy', None) - if not self.proxy_port: - self.proxy_port = config.get_value('Boto', 'proxy_port', None) - if not self.proxy_user: - self.proxy_user = config.get_value('Boto', 'proxy_user', None) - if not self.proxy_pass: - self.proxy_pass = config.get_value('Boto', 'proxy_pass', None) - - if not self.proxy_port and self.proxy: - print "http_proxy environment variable does not specify " \ - "a port, using default" - self.proxy_port = self.port - self.use_proxy = (self.proxy != None) - - def get_http_connection(self, host, is_secure): - queue = self._pool[self._cached_name(host, is_secure)] - try: - return queue.get_nowait() - except Queue.Empty: - return self.new_http_connection(host, is_secure) - - def new_http_connection(self, host, is_secure): - if self.use_proxy: - host = '%s:%d' % (self.proxy, int(self.proxy_port)) - if host is None: - host = self.server_name() - boto.log.debug('establishing HTTP connection') - if is_secure: - if self.use_proxy: - connection = self.proxy_ssl() - elif self.https_connection_factory: - connection = self.https_connection_factory(host) - else: - connection = httplib.HTTPSConnection(host) - else: - connection = httplib.HTTPConnection(host) - if self.debug > 1: - connection.set_debuglevel(self.debug) - # self.connection must be maintained for backwards-compatibility - # however, it must be dynamically pulled from the connection pool - # set a private variable which will enable that - if host.split(':')[0] == self.host and is_secure == self.is_secure: - self._connection = (host, is_secure) - return connection - - def put_http_connection(self, host, is_secure, connection): - try: - self._pool[self._cached_name(host, is_secure)].put_nowait(connection) - except Queue.Full: - # gracefully fail in case of pool overflow - connection.close() - - def proxy_ssl(self): - host = '%s:%d' % (self.host, self.port) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - sock.connect((self.proxy, int(self.proxy_port))) - except: - raise - sock.sendall("CONNECT %s HTTP/1.0\r\n" % host) - sock.sendall("User-Agent: %s\r\n" % UserAgent) - if self.proxy_user and self.proxy_pass: - for k, v in self.get_proxy_auth_header().items(): - sock.sendall("%s: %s\r\n" % (k, v)) - sock.sendall("\r\n") - resp = httplib.HTTPResponse(sock, strict=True) - resp.begin() - - if resp.status != 200: - # Fake a socket error, use a code that make it obvious it hasn't - # been generated by the socket library - raise socket.error(-71, - "Error talking to HTTP proxy %s:%s: %s (%s)" % - (self.proxy, self.proxy_port, resp.status, resp.reason)) - - # We can safely close the response, it duped the original socket - resp.close() - - h = httplib.HTTPConnection(host) - - # Wrap the socket in an SSL socket - if hasattr(httplib, 'ssl'): - sslSock = httplib.ssl.SSLSocket(sock) - else: # Old Python, no ssl module - sslSock = socket.ssl(sock, None, None) - sslSock = httplib.FakeSocket(sock, sslSock) - # This is a bit unclean - h.sock = sslSock - return h - - def prefix_proxy_to_path(self, path, host=None): - path = self.protocol + '://' + (host or self.server_name()) + path - return path - - def get_proxy_auth_header(self): - auth = base64.encodestring(self.proxy_user + ':' + self.proxy_pass) - return {'Proxy-Authorization': 'Basic %s' % auth} - - def _mexe(self, method, path, data, headers, host=None, sender=None): - """ - mexe - Multi-execute inside a loop, retrying multiple times to handle - transient Internet errors by simply trying again. - Also handles redirects. - - This code was inspired by the S3Utils classes posted to the boto-users - Google group by Larry Bates. Thanks! - """ - boto.log.debug('Method: %s' % method) - boto.log.debug('Path: %s' % path) - boto.log.debug('Data: %s' % data) - boto.log.debug('Headers: %s' % headers) - boto.log.debug('Host: %s' % host) - response = None - body = None - e = None - num_retries = config.getint('Boto', 'num_retries', self.num_retries) - i = 0 - connection = self.get_http_connection(host, self.is_secure) - while i <= num_retries: - try: - if callable(sender): - response = sender(connection, method, path, data, headers) - else: - connection.request(method, path, data, headers) - response = connection.getresponse() - location = response.getheader('location') - # -- gross hack -- - # httplib gets confused with chunked responses to HEAD requests - # so I have to fake it out - if method == 'HEAD' and getattr(response, 'chunked', False): - response.chunked = 0 - if response.status == 500 or response.status == 503: - boto.log.debug('received %d response, retrying in %d seconds' % (response.status, 2 ** i)) - body = response.read() - elif response.status == 408: - body = response.read() - print '-------------------------' - print ' 4 0 8 ' - print 'path=%s' % path - print body - print '-------------------------' - elif response.status < 300 or response.status >= 400 or \ - not location: - self.put_http_connection(host, self.is_secure, connection) - return response - else: - scheme, host, path, params, query, fragment = \ - urlparse.urlparse(location) - if query: - path += '?' + query - boto.log.debug('Redirecting: %s' % scheme + '://' + host + path) - connection = self.get_http_connection(host, - scheme == 'https') - continue - except KeyboardInterrupt: - sys.exit('Keyboard Interrupt') - except httplib.BadStatusLine as e: - boto.log.warn('Bad status line: %r, retrying..', e.line) - connection = self.new_http_connection(host, self.is_secure) - except self.http_exceptions, e: - boto.log.warn('encountered http exception, reconnecting', - exc_info=True) - connection = self.new_http_connection(host, self.is_secure) - time.sleep(2 ** i) - i += 1 - # If we made it here, it's because we have exhausted our retries and stil haven't - # succeeded. So, if we have a response object, use it to raise an exception. - # Otherwise, raise the exception that must have already happened. - if response: - raise BotoServerError(response.status, response.reason, body) - elif e: - raise e - else: - raise BotoClientError('Please report this exception as a Boto Issue!') - - def make_request(self, method, path, headers=None, data='', host=None, - auth_path=None, sender=None): - path = self.get_path(path) - if headers == None: - headers = {} - else: - headers = headers.copy() - headers['User-Agent'] = UserAgent - if not headers.has_key('Content-Length'): - headers['Content-Length'] = str(len(data)) - if self.use_proxy: - path = self.prefix_proxy_to_path(path, host) - if self.proxy_user and self.proxy_pass and not self.is_secure: - # If is_secure, we don't have to set the proxy authentication - # header here, we did that in the CONNECT to the proxy. - headers.update(self.get_proxy_auth_header()) - request_string = auth_path or path - for key in headers: - val = headers[key] - if isinstance(val, unicode): - headers[key] = urllib.quote_plus(val.encode('utf-8')) - self.add_aws_auth_header(headers, method, request_string) - return self._mexe(method, path, data, headers, host, sender) - - def add_aws_auth_header(self, headers, method, path): - path = self.get_path(path) - if not headers.has_key('Date'): - headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", - time.gmtime()) - - c_string = boto.utils.canonical_string(method, path, headers) - boto.log.debug('Canonical: %s' % c_string) - hmac = self.hmac.copy() - hmac.update(c_string) - b64_hmac = base64.encodestring(hmac.digest()).strip() - headers['Authorization'] = "AWS %s:%s" % (self.aws_access_key_id, b64_hmac) - - def close(self): - """(Optional) Close any open HTTP connections. This is non-destructive, - and making a new request will open a connection again.""" - - boto.log.debug('closing all HTTP connections') - self.connection = None # compat field - - -class AWSQueryConnection(AWSAuthConnection): - - APIVersion = '' - SignatureVersion = '1' - ResponseError = BotoServerError - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, host=None, debug=0, - https_connection_factory=None, path='/'): - AWSAuthConnection.__init__(self, host, aws_access_key_id, aws_secret_access_key, - is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, - debug, https_connection_factory, path) - - def get_utf8_value(self, value): - if not isinstance(value, str) and not isinstance(value, unicode): - value = str(value) - if isinstance(value, unicode): - return value.encode('utf-8') - else: - return value - - def calc_signature_0(self, params): - boto.log.debug('using calc_signature_0') - hmac = self.hmac.copy() - s = params['Action'] + params['Timestamp'] - hmac.update(s) - keys = params.keys() - keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower())) - pairs = [] - for key in keys: - val = self.get_utf8_value(params[key]) - pairs.append(key + '=' + urllib.quote(val)) - qs = '&'.join(pairs) - return (qs, base64.b64encode(hmac.digest())) - - def calc_signature_1(self, params): - boto.log.debug('using calc_signature_1') - hmac = self.hmac.copy() - keys = params.keys() - keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower())) - pairs = [] - for key in keys: - hmac.update(key) - val = self.get_utf8_value(params[key]) - hmac.update(val) - pairs.append(key + '=' + urllib.quote(val)) - qs = '&'.join(pairs) - return (qs, base64.b64encode(hmac.digest())) - - def calc_signature_2(self, params, verb, path): - boto.log.debug('using calc_signature_2') - string_to_sign = '%s\n%s\n%s\n' % (verb, self.server_name().lower(), path) - if self.hmac_256: - hmac = self.hmac_256.copy() - params['SignatureMethod'] = 'HmacSHA256' - else: - hmac = self.hmac.copy() - params['SignatureMethod'] = 'HmacSHA1' - keys = params.keys() - keys.sort() - pairs = [] - for key in keys: - val = self.get_utf8_value(params[key]) - pairs.append(urllib.quote(key, safe='') + '=' + urllib.quote(val, safe='-_~')) - qs = '&'.join(pairs) - boto.log.debug('query string: %s' % qs) - string_to_sign += qs - boto.log.debug('string_to_sign: %s' % string_to_sign) - hmac.update(string_to_sign) - b64 = base64.b64encode(hmac.digest()) - boto.log.debug('len(b64)=%d' % len(b64)) - boto.log.debug('base64 encoded digest: %s' % b64) - return (qs, b64) - - def get_signature(self, params, verb, path): - if self.SignatureVersion == '0': - t = self.calc_signature_0(params) - elif self.SignatureVersion == '1': - t = self.calc_signature_1(params) - elif self.SignatureVersion == '2': - t = self.calc_signature_2(params, verb, path) - else: - raise BotoClientError('Unknown Signature Version: %s' % self.SignatureVersion) - return t - - def make_request(self, action, params=None, path='/', verb='GET'): - headers = {} - if params == None: - params = {} - params['Action'] = action - params['Version'] = self.APIVersion - params['AWSAccessKeyId'] = self.aws_access_key_id - params['SignatureVersion'] = self.SignatureVersion - params['Timestamp'] = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) - qs, signature = self.get_signature(params, verb, self.get_path(path)) - if verb == 'POST': - headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8' - request_body = qs + '&Signature=' + urllib.quote(signature) - qs = path - else: - request_body = '' - qs = path + '?' + qs + '&Signature=' + urllib.quote(signature) - return AWSAuthConnection.make_request(self, verb, qs, - data=request_body, - headers=headers) - - def build_list_params(self, params, items, label): - if isinstance(items, str): - items = [items] - for i in range(1, len(items) + 1): - params['%s.%d' % (label, i)] = items[i - 1] - - # generics - - def get_list(self, action, params, markers, path='/', parent=None, verb='GET'): - if not parent: - parent = self - response = self.make_request(action, params, path, verb) - body = response.read() - boto.log.debug(body) - if response.status == 200: - rs = ResultSet(markers) - h = handler.XmlHandler(rs, parent) - xml.sax.parseString(body, h) - return rs - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def get_object(self, action, params, cls, path='/', parent=None, verb='GET'): - if not parent: - parent = self - response = self.make_request(action, params, path, verb) - body = response.read() - boto.log.debug(body) - if response.status == 200: - obj = cls(parent) - h = handler.XmlHandler(obj, parent) - xml.sax.parseString(body, h) - return obj - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def get_status(self, action, params, path='/', parent=None, verb='GET'): - if not parent: - parent = self - response = self.make_request(action, params, path, verb) - body = response.read() - boto.log.debug(body) - if response.status == 200: - rs = ResultSet() - h = handler.XmlHandler(rs, parent) - xml.sax.parseString(body, h) - return rs.status - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - diff --git a/src/s3ql/backends/boto/exception.py b/src/s3ql/backends/boto/exception.py deleted file mode 100644 index 609998f..0000000 --- a/src/s3ql/backends/boto/exception.py +++ /dev/null @@ -1,305 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -""" -Exception classes - Subclassing allows you to check for specific errors -""" -import base64 -import xml.sax -from . import handler -from .resultset import ResultSet -import base64 - - -class BotoClientError(StandardError): - """ - General Boto Client error (error accessing AWS) - """ - - def __init__(self, reason): - StandardError.__init__(self) - self.reason = reason - - def __repr__(self): - return 'S3Error: %s' % self.reason - - def __str__(self): - return 'S3Error: %s' % self.reason - -class SDBPersistenceError(StandardError): - - pass - -class S3PermissionsError(BotoClientError): - """ - Permissions error when accessing a bucket or key on S3. - """ - pass - -class BotoServerError(StandardError): - - def __init__(self, status, reason, body=None): - StandardError.__init__(self) - self.status = status - self.reason = reason - self.body = body or '' - self.request_id = None - self.error_code = None - self.error_message = None - self.box_usage = None - - # Attempt to parse the error response. If body isn't present, - # then just ignore the error response. - if self.body: - try: - h = handler.XmlHandler(self, self) - xml.sax.parseString(self.body, h) - except xml.sax.SAXParseException, pe: - # Go ahead and clean up anything that may have - # managed to get into the error data so we - # don't get partial garbage. - print "Warning: failed to parse error message from AWS: %s" % pe - self._cleanupParsedProperties() - - def __getattr__(self, name): - if name == 'message': - return self.error_message - if name == 'code': - return self.error_code - raise AttributeError - - def __repr__(self): - return '%s: %s %s\n%s' % (self.__class__.__name__, - self.status, self.reason, self.body) - - def __str__(self): - return '%s: %s %s\n%s' % (self.__class__.__name__, - self.status, self.reason, self.body) - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name in ('RequestId', 'RequestID'): - self.request_id = value - elif name == 'Code': - self.error_code = value - elif name == 'Message': - self.error_message = value - elif name == 'BoxUsage': - self.box_usage = value - return None - - def _cleanupParsedProperties(self): - self.request_id = None - self.error_code = None - self.error_message = None - self.box_usage = None - -class ConsoleOutput: - - def __init__(self, parent=None): - self.parent = parent - self.instance_id = None - self.timestamp = None - self.comment = None - self.output = None - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'instanceId': - self.instance_id = value - elif name == 'output': - self.output = base64.b64decode(value) - else: - setattr(self, name, value) - -class S3CreateError(BotoServerError): - """ - Error creating a bucket or key on S3. - """ - def __init__(self, status, reason, body=None): - self.bucket = None - BotoServerError.__init__(self, status, reason, body) - - def endElement(self, name, value, connection): - if name == 'BucketName': - self.bucket = value - else: - return BotoServerError.endElement(self, name, value, connection) - -class S3CopyError(BotoServerError): - """ - Error copying a key on S3. - """ - pass - -class SQSError(BotoServerError): - """ - General Error on Simple Queue Service. - """ - def __init__(self, status, reason, body=None): - self.detail = None - self.type = None - BotoServerError.__init__(self, status, reason, body) - - def startElement(self, name, attrs, connection): - return BotoServerError.startElement(self, name, attrs, connection) - - def endElement(self, name, value, connection): - if name == 'Detail': - self.detail = value - elif name == 'Type': - self.type = value - else: - return BotoServerError.endElement(self, name, value, connection) - - def _cleanupParsedProperties(self): - BotoServerError._cleanupParsedProperties(self) - for p in ('detail', 'type'): - setattr(self, p, None) - -class SQSDecodeError(BotoClientError): - """ - Error when decoding an SQS message. - """ - def __init__(self, reason, message): - BotoClientError.__init__(self, reason) - self.message = message - - def __repr__(self): - return 'SQSDecodeError: %s' % self.reason - - def __str__(self): - return 'SQSDecodeError: %s' % self.reason - -class S3ResponseError(BotoServerError): - """ - Error in response from S3. - """ - def __init__(self, status, reason, body=None): - self.resource = None - BotoServerError.__init__(self, status, reason, body) - - def startElement(self, name, attrs, connection): - return BotoServerError.startElement(self, name, attrs, connection) - - def endElement(self, name, value, connection): - if name == 'Resource': - self.resource = value - else: - return BotoServerError.endElement(self, name, value, connection) - - def _cleanupParsedProperties(self): - BotoServerError._cleanupParsedProperties(self) - for p in ('resource'): - setattr(self, p, None) - -class EC2ResponseError(BotoServerError): - """ - Error in response from EC2. - """ - - def __init__(self, status, reason, body=None): - self.errors = None - self._errorResultSet = [] - BotoServerError.__init__(self, status, reason, body) - self.errors = [ (e.error_code, e.error_message) \ - for e in self._errorResultSet ] - if len(self.errors): - self.error_code, self.error_message = self.errors[0] - - def startElement(self, name, attrs, connection): - if name == 'Errors': - self._errorResultSet = ResultSet([('Error', _EC2Error)]) - return self._errorResultSet - else: - return None - - def endElement(self, name, value, connection): - if name == 'RequestID': - self.request_id = value - else: - return None # don't call subclass here - - def _cleanupParsedProperties(self): - BotoServerError._cleanupParsedProperties(self) - self._errorResultSet = [] - for p in ('errors'): - setattr(self, p, None) - -class EmrResponseError(BotoServerError): - """ - Error in response from EMR - """ - pass - -class _EC2Error: - - def __init__(self, connection=None): - self.connection = connection - self.error_code = None - self.error_message = None - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Code': - self.error_code = value - elif name == 'Message': - self.error_message = value - else: - return None - -class SDBResponseError(BotoServerError): - """ - Error in respones from SDB. - """ - pass - -class AWSConnectionError(BotoClientError): - """ - General error connecting to Amazon Web Services. - """ - pass - -class S3DataError(BotoClientError): - """ - Error receiving data from S3. - """ - pass - -class FPSResponseError(BotoServerError): - pass - - -class InvalidUriError(Exception): - """Exception raised when URI is invalid.""" - - def __init__(self, message): - Exception.__init__(self) - self.message = message diff --git a/src/s3ql/backends/boto/handler.py b/src/s3ql/backends/boto/handler.py deleted file mode 100644 index 528fbae..0000000 --- a/src/s3ql/backends/boto/handler.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -import xml.sax - -class XmlHandler(xml.sax.ContentHandler): - - def __init__(self, root_node, connection): - self.connection = connection - self.nodes = [('root', root_node)] - self.current_text = '' - - def startElement(self, name, attrs): - self.current_text = '' - new_node = self.nodes[-1][1].startElement(name, attrs, self.connection) - if new_node != None: - self.nodes.append((name, new_node)) - - def endElement(self, name): - self.nodes[-1][1].endElement(name, self.current_text, self.connection) - if self.nodes[-1][0] == name: - self.nodes.pop() - self.current_text = '' - - def characters(self, content): - self.current_text += content - - diff --git a/src/s3ql/backends/boto/pyami/__init__.py b/src/s3ql/backends/boto/pyami/__init__.py deleted file mode 100644 index 403f980..0000000 --- a/src/s3ql/backends/boto/pyami/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Dummy diff --git a/src/s3ql/backends/boto/pyami/config.py b/src/s3ql/backends/boto/pyami/config.py deleted file mode 100644 index aa8d72c..0000000 --- a/src/s3ql/backends/boto/pyami/config.py +++ /dev/null @@ -1,206 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -import StringIO, os, re -import ConfigParser - -BotoConfigPath = '/etc/boto.cfg' -BotoConfigLocations = [BotoConfigPath] -if 'HOME' in os.environ: - UserConfigPath = os.path.expanduser('~/.boto') - BotoConfigLocations.append(UserConfigPath) -else: - UserConfigPath = None -if 'BOTO_CONFIG' in os.environ: - BotoConfigLocations.append(os.path.expanduser(os.environ['BOTO_CONFIG'])) - -class Config(ConfigParser.SafeConfigParser): - - def __init__(self, path=None, fp=None, do_load=True): - ConfigParser.SafeConfigParser.__init__(self, {'working_dir' : '/mnt/pyami', - 'debug' : '0'}) - if do_load: - if path: - self.load_from_path(path) - elif fp: - self.readfp(fp) - else: - self.read(BotoConfigLocations) - if "AWS_CREDENTIAL_FILE" in os.environ: - self.load_credential_file(os.path.expanduser(os.environ['AWS_CREDENTIAL_FILE'])) - - def load_credential_file(self, path): - """Load a credential file as is setup like the Java utilities""" - c_data = StringIO.StringIO() - c_data.write("[Credentials]\n") - for line in open(path, "r").readlines(): - c_data.write(line.replace("AWSAccessKeyId", "aws_access_key_id").replace("AWSSecretKey", "aws_secret_access_key")) - c_data.seek(0) - self.readfp(c_data) - - def load_from_path(self, path): - file = open(path) - for line in file.readlines(): - match = re.match("^#import[\s\t]*([^\s^\t]*)[\s\t]*$", line) - if match: - extended_file = match.group(1) - (dir, file) = os.path.split(path) - self.load_from_path(os.path.join(dir, extended_file)) - self.read(path) - - def save_option(self, path, section, option, value): - """ - Write the specified Section.Option to the config file specified by path. - Replace any previous value. If the path doesn't exist, create it. - Also add the option the the in-memory config. - """ - config = ConfigParser.SafeConfigParser() - config.read(path) - if not config.has_section(section): - config.add_section(section) - config.set(section, option, value) - fp = open(path, 'w') - config.write(fp) - fp.close() - if not self.has_section(section): - self.add_section(section) - self.set(section, option, value) - - def save_user_option(self, section, option, value): - self.save_option(UserConfigPath, section, option, value) - - def save_system_option(self, section, option, value): - self.save_option(BotoConfigPath, section, option, value) - - def get_instance(self, name, default=None): - try: - val = self.get('Instance', name) - except: - val = default - return val - - def get_user(self, name, default=None): - try: - val = self.get('User', name) - except: - val = default - return val - - def getint_user(self, name, default=0): - try: - val = self.getint('User', name) - except: - val = default - return val - - def get_value(self, section, name, default=None): - return self.get(section, name, default) - - def get(self, section, name, default=None): - try: - val = ConfigParser.SafeConfigParser.get(self, section, name) - except: - val = default - return val - - def getint(self, section, name, default=0): - try: - val = ConfigParser.SafeConfigParser.getint(self, section, name) - except: - val = int(default) - return val - - def getfloat(self, section, name, default=0.0): - try: - val = ConfigParser.SafeConfigParser.getfloat(self, section, name) - except: - val = float(default) - return val - - def getbool(self, section, name, default=False): - if self.has_option(section, name): - val = self.get(section, name) - if val.lower() == 'true': - val = True - else: - val = False - else: - val = default - return val - - def setbool(self, section, name, value): - if value: - self.set(section, name, 'true') - else: - self.set(section, name, 'false') - - def dump(self): - s = StringIO.StringIO() - self.write(s) - print s.getvalue() - - def dump_safe(self, fp=None): - if not fp: - fp = StringIO.StringIO() - for section in self.sections(): - fp.write('[%s]\n' % section) - for option in self.options(section): - if option == 'aws_secret_access_key': - fp.write('%s = xxxxxxxxxxxxxxxxxx\n' % option) - else: - fp.write('%s = %s\n' % (option, self.get(section, option))) - - def dump_to_sdb(self, domain_name, item_name): - import simplejson - sdb = boto.connect_sdb() - domain = sdb.lookup(domain_name) - if not domain: - domain = sdb.create_domain(domain_name) - item = domain.new_item(item_name) - item.active = False - for section in self.sections(): - d = {} - for option in self.options(section): - d[option] = self.get(section, option) - item[section] = simplejson.dumps(d) - item.save() - - def load_from_sdb(self, domain_name, item_name): - import simplejson - sdb = boto.connect_sdb() - domain = sdb.lookup(domain_name) - item = domain.get_item(item_name) - for section in item.keys(): - if not self.has_section(section): - self.add_section(section) - d = simplejson.loads(item[section]) - for attr_name in d.keys(): - attr_value = d[attr_name] - if attr_value == None: - attr_value = 'None' - if isinstance(attr_value, bool): - self.setbool(section, attr_name, attr_value) - else: - self.set(section, attr_name, attr_value) diff --git a/src/s3ql/backends/boto/resultset.py b/src/s3ql/backends/boto/resultset.py deleted file mode 100644 index 234b140..0000000 --- a/src/s3ql/backends/boto/resultset.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -class ResultSet(list): - """ - The ResultSet is used to pass results back from the Amazon services - to the client. It has an ugly but workable mechanism for parsing - the XML results from AWS. Because I don't really want any dependencies - on external libraries, I'm using the standard SAX parser that comes - with Python. The good news is that it's quite fast and efficient but - it makes some things rather difficult. - - You can pass in, as the marker_elem parameter, a list of tuples. - Each tuple contains a string as the first element which represents - the XML element that the resultset needs to be on the lookout for - and a Python class as the second element of the tuple. Each time the - specified element is found in the XML, a new instance of the class - will be created and popped onto the stack. - - """ - - def __init__(self, marker_elem=None): - list.__init__(self) - if isinstance(marker_elem, list): - self.markers = marker_elem - else: - self.markers = [] - self.marker = None - self.key_marker = None - self.next_key_marker = None - self.next_version_id_marker = None - self.version_id_marker = None - self.is_truncated = False - self.next_token = None - self.status = True - - def startElement(self, name, attrs, connection): - for t in self.markers: - if name == t[0]: - obj = t[1](connection) - self.append(obj) - return obj - return None - - def to_boolean(self, value, true_value='true'): - if value == true_value: - return True - else: - return False - - def endElement(self, name, value, connection): - if name == 'IsTruncated': - self.is_truncated = self.to_boolean(value) - elif name == 'Marker': - self.marker = value - elif name == 'KeyMarker': - self.key_marker = value - elif name == 'VersionIdMarker': - self.version_id_marker = value - elif name == 'NextKeyMarker': - self.next_key_marker = value - elif name == 'NextVersionIdMarker': - self.next_version_id_marker = value - elif name == 'Prefix': - self.prefix = value - elif name == 'return': - self.status = self.to_boolean(value) - elif name == 'StatusCode': - self.status = self.to_boolean(value, 'Success') - elif name == 'ItemName': - self.append(value) - elif name == 'NextToken': - self.next_token = value - elif name == 'BoxUsage': - try: - connection.box_usage += float(value) - except: - pass - elif name == 'IsValid': - self.status = self.to_boolean(value, 'True') - else: - setattr(self, name, value) - -class BooleanResult(object): - - def __init__(self, marker_elem=None): - self.status = True - self.request_id = None - self.box_usage = None - - def __repr__(self): - if self.status: - return 'True' - else: - return 'False' - - def __nonzero__(self): - return self.status - - def startElement(self, name, attrs, connection): - return None - - def to_boolean(self, value, true_value='true'): - if value == true_value: - return True - else: - return False - - def endElement(self, name, value, connection): - if name == 'return': - self.status = self.to_boolean(value) - elif name == 'StatusCode': - self.status = self.to_boolean(value, 'Success') - elif name == 'IsValid': - self.status = self.to_boolean(value, 'True') - elif name == 'RequestId': - self.request_id = value - elif name == 'requestId': - self.request_id = value - elif name == 'BoxUsage': - self.request_id = value - else: - setattr(self, name, value) - diff --git a/src/s3ql/backends/boto/s3/__init__.py b/src/s3ql/backends/boto/s3/__init__.py deleted file mode 100644 index d13fba5..0000000 --- a/src/s3ql/backends/boto/s3/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -from ... import boto - -from .connection import S3Connection as Connection -from .key import Key -from .bucket import Bucket - -__all__ = ['Connection', 'Key', 'Bucket'] diff --git a/src/s3ql/backends/boto/s3/acl.py b/src/s3ql/backends/boto/s3/acl.py deleted file mode 100644 index 27df47e..0000000 --- a/src/s3ql/backends/boto/s3/acl.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -from .user import User -import StringIO - -CannedACLStrings = ['private', 'public-read', - 'public-read-write', 'authenticated-read'] - - -class Policy: - - def __init__(self, parent=None): - self.parent = parent - self.acl = None - - def __repr__(self): - grants = [] - for g in self.acl.grants: - if g.id == self.owner.id: - grants.append("%s (owner) = %s" % (g.display_name, g.permission)) - else: - if g.type == 'CanonicalUser': - u = g.display_name - elif g.type == 'Group': - u = g.uri - else: - u = g.email - grants.append("%s = %s" % (u, g.permission)) - return "<Policy: %s>" % ", ".join(grants) - - def startElement(self, name, attrs, connection): - if name == 'Owner': - self.owner = User(self) - return self.owner - elif name == 'AccessControlList': - self.acl = ACL(self) - return self.acl - else: - return None - - def endElement(self, name, value, connection): - if name == 'Owner': - pass - elif name == 'AccessControlList': - pass - else: - setattr(self, name, value) - - def to_xml(self): - s = '<AccessControlPolicy>' - s += self.owner.to_xml() - s += self.acl.to_xml() - s += '</AccessControlPolicy>' - return s - -class ACL: - - def __init__(self, policy=None): - self.policy = policy - self.grants = [] - - def add_grant(self, grant): - self.grants.append(grant) - - def add_email_grant(self, permission, email_address): - grant = Grant(permission=permission, type='AmazonCustomerByEmail', - email_address=email_address) - self.grants.append(grant) - - def add_user_grant(self, permission, user_id): - grant = Grant(permission=permission, type='CanonicalUser', id=user_id) - self.grants.append(grant) - - def startElement(self, name, attrs, connection): - if name == 'Grant': - self.grants.append(Grant(self)) - return self.grants[-1] - else: - return None - - def endElement(self, name, value, connection): - if name == 'Grant': - pass - else: - setattr(self, name, value) - - def to_xml(self): - s = '<AccessControlList>' - for grant in self.grants: - s += grant.to_xml() - s += '</AccessControlList>' - return s - -class Grant: - - NameSpace = 'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' - - def __init__(self, permission=None, type=None, id=None, - display_name=None, uri=None, email_address=None): - self.permission = permission - self.id = id - self.display_name = display_name - self.uri = uri - self.email_address = email_address - self.type = type - - def startElement(self, name, attrs, connection): - if name == 'Grantee': - self.type = attrs['xsi:type'] - return None - - def endElement(self, name, value, connection): - if name == 'ID': - self.id = value - elif name == 'DisplayName': - self.display_name = value - elif name == 'URI': - self.uri = value - elif name == 'EmailAddress': - self.email_address = value - elif name == 'Grantee': - pass - elif name == 'Permission': - self.permission = value - else: - setattr(self, name, value) - - def to_xml(self): - s = '<Grant>' - s += '<Grantee %s xsi:type="%s">' % (self.NameSpace, self.type) - if self.type == 'CanonicalUser': - s += '<ID>%s</ID>' % self.id - s += '<DisplayName>%s</DisplayName>' % self.display_name - elif self.type == 'Group': - s += '<URI>%s</URI>' % self.uri - else: - s += '<EmailAddress>%s</EmailAddress>' % self.email_address - s += '</Grantee>' - s += '<Permission>%s</Permission>' % self.permission - s += '</Grant>' - return s - - diff --git a/src/s3ql/backends/boto/s3/bucket.py b/src/s3ql/backends/boto/s3/bucket.py deleted file mode 100644 index cb3872b..0000000 --- a/src/s3ql/backends/boto/s3/bucket.py +++ /dev/null @@ -1,749 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -from ... import boto -from .. import handler -from ..resultset import ResultSet -from .acl import Policy, CannedACLStrings, ACL, Grant -from .user import User -from .key import Key -from .prefix import Prefix -from ..exception import S3ResponseError, S3PermissionsError, S3CopyError -from .bucketlistresultset import BucketListResultSet -from .. import utils -import xml.sax -import urllib -import re - -S3Permissions = ['READ', 'WRITE', 'READ_ACP', 'WRITE_ACP', 'FULL_CONTROL'] - -class Bucket: - - BucketLoggingBody = """<?xml version="1.0" encoding="UTF-8"?> - <BucketLoggingStatus xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> - <LoggingEnabled> - <TargetBucket>%s</TargetBucket> - <TargetPrefix>%s</TargetPrefix> - </LoggingEnabled> - </BucketLoggingStatus>""" - - EmptyBucketLoggingBody = """<?xml version="1.0" encoding="UTF-8"?> - <BucketLoggingStatus xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> - </BucketLoggingStatus>""" - - LoggingGroup = 'http://acs.amazonaws.com/groups/s3/LogDelivery' - - BucketPaymentBody = """<?xml version="1.0" encoding="UTF-8"?> - <RequestPaymentConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> - <Payer>%s</Payer> - </RequestPaymentConfiguration>""" - - VersioningBody = """<?xml version="1.0" encoding="UTF-8"?> - <VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> - <Status>%s</Status> - <MfaDelete>%s</MfaDelete> - </VersioningConfiguration>""" - - VersionRE = '<Status>([A-Za-z]+)</Status>' - MFADeleteRE = '<MfaDelete>([A-Za-z]+)</MfaDelete>' - - def __init__(self, connection=None, name=None, key_class=Key): - self.name = name - self.connection = connection - self.key_class = key_class - - def __repr__(self): - return '<Bucket: %s>' % self.name - - def __iter__(self): - return iter(BucketListResultSet(self)) - - def __contains__(self, key_name): - return not (self.get_key(key_name) is None) - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Name': - self.name = value - elif name == 'CreationDate': - self.creation_date = value - else: - setattr(self, name, value) - - def set_key_class(self, key_class): - """ - Set the Key class associated with this bucket. By default, this - would be the boto.s3.key.Key class but if you want to subclass that - for some reason this allows you to associate your new class with a - bucket so that when you call bucket.new_key() or when you get a listing - of keys in the bucket you will get an instances of your key class - rather than the default. - - :type key_class: class - :param key_class: A subclass of Key that can be more specific - """ - self.key_class = key_class - - def lookup(self, key_name, headers=None): - """ - Deprecated: Please use get_key method. - - :type key_name: string - :param key_name: The name of the key to retrieve - - :rtype: :class:`boto.s3.key.Key` - :returns: A Key object from this bucket. - """ - return self.get_key(key_name, headers=headers) - - def get_key(self, key_name, headers=None, version_id=None): - """ - Check to see if a particular key exists within the bucket. This - method uses a HEAD request to check for the existance of the key. - Returns: An instance of a Key object or None - - :type key_name: string - :param key_name: The name of the key to retrieve - - :rtype: :class:`boto.s3.key.Key` - :returns: A Key object from this bucket. - """ - if version_id: - query_args = 'versionId=%s' % version_id - else: - query_args = None - response = self.connection.make_request('HEAD', self.name, key_name, - headers=headers, - query_args=query_args) - if response.status == 200: - response.read() - k = self.key_class(self) - k.metadata = boto.utils.get_aws_metadata(response.msg) - k.etag = response.getheader('etag') - k.content_type = response.getheader('content-type') - k.content_encoding = response.getheader('content-encoding') - k.last_modified = response.getheader('last-modified') - k.size = int(response.getheader('content-length')) - k.name = key_name - k.handle_version_headers(response) - return k - else: - if response.status == 404: - response.read() - return None - else: - raise S3ResponseError(response.status, response.reason, '') - - def list(self, prefix='', delimiter='', marker='', headers=None): - """ - List key objects within a bucket. This returns an instance of an - BucketListResultSet that automatically handles all of the result - paging, etc. from S3. You just need to keep iterating until - there are no more results. - Called with no arguments, this will return an iterator object across - all keys within the bucket. - - :type prefix: string - :param prefix: allows you to limit the listing to a particular - prefix. For example, if you call the method with - prefix='/foo/' then the iterator will only cycle - through the keys that begin with the string '/foo/'. - - :type delimiter: string - :param delimiter: can be used in conjunction with the prefix - to allow you to organize and browse your keys - hierarchically. See: - http://docs.amazonwebservices.com/AmazonS3/2006-03-01/ - for more details. - - :type marker: string - :param marker: The "marker" of where you are in the result set - - :rtype: :class:`boto.s3.bucketlistresultset.BucketListResultSet` - :return: an instance of a BucketListResultSet that handles paging, etc - """ - return BucketListResultSet(self, prefix, delimiter, marker, headers) - - def list_versions(self, prefix='', delimiter='', key_marker='', - version_id_marker='', headers=None): - """ - List key objects within a bucket. This returns an instance of an - BucketListResultSet that automatically handles all of the result - paging, etc. from S3. You just need to keep iterating until - there are no more results. - Called with no arguments, this will return an iterator object across - all keys within the bucket. - - :type prefix: string - :param prefix: allows you to limit the listing to a particular - prefix. For example, if you call the method with - prefix='/foo/' then the iterator will only cycle - through the keys that begin with the string '/foo/'. - - :type delimiter: string - :param delimiter: can be used in conjunction with the prefix - to allow you to organize and browse your keys - hierarchically. See: - http://docs.amazonwebservices.com/AmazonS3/2006-03-01/ - for more details. - - :type marker: string - :param marker: The "marker" of where you are in the result set - - :rtype: :class:`boto.s3.bucketlistresultset.BucketListResultSet` - :return: an instance of a BucketListResultSet that handles paging, etc - """ - return VersionedBucketListResultSet(self, prefix, delimiter, key_marker, - version_id_marker, headers) - - def _get_all(self, element_map, initial_query_string='', - headers=None, **params): - l = [] - for k,v in params.items(): - k = k.replace('_', '-') - if k == 'maxkeys': - k = 'max-keys' - if isinstance(v, unicode): - v = v.encode('utf-8') - if v is not None and v != '': - l.append('%s=%s' % (urllib.quote(k), urllib.quote(str(v)))) - if len(l): - s = initial_query_string + '&' + '&'.join(l) - else: - s = initial_query_string - response = self.connection.make_request('GET', self.name, - headers=headers, query_args=s) - body = response.read() - boto.log.debug(body) - if response.status == 200: - rs = ResultSet(element_map) - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - else: - raise S3ResponseError(response.status, response.reason, body) - - def get_all_keys(self, headers=None, **params): - """ - A lower-level method for listing contents of a bucket. - This closely models the actual S3 API and requires you to manually - handle the paging of results. For a higher-level method - that handles the details of paging for you, you can use the list method. - - :type max_keys: int - :param max_keys: The maximum number of keys to retrieve - - :type prefix: string - :param prefix: The prefix of the keys you want to retrieve - - :type marker: string - :param marker: The "marker" of where you are in the result set - - :type delimiter: string - :param delimiter: If this optional, Unicode string parameter - is included with your request, then keys that - contain the same string between the prefix and - the first occurrence of the delimiter will be - rolled up into a single result element in the - CommonPrefixes collection. These rolled-up keys - are not returned elsewhere in the response. - - :rtype: ResultSet - :return: The result from S3 listing the keys requested - - """ - return self._get_all([('Contents', self.key_class), - ('CommonPrefixes', Prefix)], - '', headers, **params) - - def get_all_versions(self, headers=None, **params): - """ - A lower-level, version-aware method for listing contents of a bucket. - This closely models the actual S3 API and requires you to manually - handle the paging of results. For a higher-level method - that handles the details of paging for you, you can use the list method. - - :type max_keys: int - :param max_keys: The maximum number of keys to retrieve - - :type prefix: string - :param prefix: The prefix of the keys you want to retrieve - - :type key_marker: string - :param key_marker: The "marker" of where you are in the result set - with respect to keys. - - :type version_id_marker: string - :param version_id_marker: The "marker" of where you are in the result - set with respect to version-id's. - - :type delimiter: string - :param delimiter: If this optional, Unicode string parameter - is included with your request, then keys that - contain the same string between the prefix and - the first occurrence of the delimiter will be - rolled up into a single result element in the - CommonPrefixes collection. These rolled-up keys - are not returned elsewhere in the response. - - :rtype: ResultSet - :return: The result from S3 listing the keys requested - - """ - return self._get_all([('Version', self.key_class), - ('CommonPrefixes', Prefix), - ('DeleteMarker', DeleteMarker)], - 'versions', headers, **params) - - def new_key(self, key_name=None): - """ - Creates a new key - - :type key_name: string - :param key_name: The name of the key to create - - :rtype: :class:`boto.s3.key.Key` or subclass - :returns: An instance of the newly created key object - """ - return self.key_class(self, key_name) - - def generate_url(self, expires_in, method='GET', - headers=None, force_http=False): - return self.connection.generate_url(expires_in, method, self.name, - headers=headers, - force_http=force_http) - - def delete_key(self, key_name, headers=None, - version_id=None, mfa_token=None): - """ - Deletes a key from the bucket. If a version_id is provided, - only that version of the key will be deleted. - - :type key_name: string - :param key_name: The key name to delete - - :type version_id: string - :param version_id: The version ID (optional) - - :type mfa_token: tuple or list of strings - :param mfa_token: A tuple or list consisting of the serial number - from the MFA device and the current value of - the six-digit token associated with the device. - This value is required anytime you are - deleting versioned objects from a bucket - that has the MFADelete option on the bucket. - """ - if version_id: - query_args = 'versionId=%s' % version_id - else: - query_args = None - if mfa_token: - if not headers: - headers = {} - headers['x-amz-mfa'] = ' '.join(mfa_token) - response = self.connection.make_request('DELETE', self.name, key_name, - headers=headers, - query_args=query_args) - body = response.read() - if response.status != 204: - raise S3ResponseError(response.status, response.reason, body) - - def copy_key(self, new_key_name, src_bucket_name, - src_key_name, metadata=None, src_version_id=None, - storage_class='STANDARD', preserve_acl=False): - """ - Create a new key in the bucket by copying another existing key. - - :type new_key_name: string - :param new_key_name: The name of the new key - - :type src_bucket_name: string - :param src_bucket_name: The name of the source bucket - - :type src_key_name: string - :param src_key_name: The name of the source key - - :type src_version_id: string - :param src_version_id: The version id for the key. This param - is optional. If not specified, the newest - version of the key will be copied. - - :type metadata: dict - :param metadata: Metadata to be associated with new key. - If metadata is supplied, it will replace the - metadata of the source key being copied. - If no metadata is supplied, the source key's - metadata will be copied to the new key. - - :type storage_class: string - :param storage_class: The storage class of the new key. - By default, the new key will use the - standard storage class. Possible values are: - STANDARD | REDUCED_REDUNDANCY - - :type preserve_acl: bool - :param preserve_acl: If True, the ACL from the source key - will be copied to the destination - key. If False, the destination key - will have the default ACL. - Note that preserving the ACL in the - new key object will require two - additional API calls to S3, one to - retrieve the current ACL and one to - set that ACL on the new object. If - you don't care about the ACL, a value - of False will be significantly more - efficient. - - :rtype: :class:`boto.s3.key.Key` or subclass - :returns: An instance of the newly created key object - """ - if preserve_acl: - acl = self.get_xml_acl(src_key_name) - src = '%s/%s' % (src_bucket_name, urllib.quote(src_key_name)) - if src_version_id: - src += '?version_id=%s' % src_version_id - headers = {'x-amz-copy-source' : src} - if storage_class != 'STANDARD': - headers['x-amz-storage-class'] = storage_class - if metadata: - headers['x-amz-metadata-directive'] = 'REPLACE' - headers = boto.utils.merge_meta(headers, metadata) - else: - headers['x-amz-metadata-directive'] = 'COPY' - response = self.connection.make_request('PUT', self.name, new_key_name, - headers=headers) - body = response.read() - if response.status == 200: - key = self.new_key(new_key_name) - h = handler.XmlHandler(key, self) - xml.sax.parseString(body, h) - if hasattr(key, 'Error'): - raise S3CopyError(key.Code, key.Message, body) - key.handle_version_headers(response) - if preserve_acl: - self.set_xml_acl(acl, new_key_name) - return key - else: - raise S3ResponseError(response.status, response.reason, body) - - def set_canned_acl(self, acl_str, key_name='', headers=None, - version_id=None): - assert acl_str in CannedACLStrings - - if headers: - headers['x-amz-acl'] = acl_str - else: - headers={'x-amz-acl': acl_str} - - query_args='acl' - if version_id: - query_args += '&versionId=%s' % version_id - response = self.connection.make_request('PUT', self.name, key_name, - headers=headers, query_args=query_args) - body = response.read() - if response.status != 200: - raise S3ResponseError(response.status, response.reason, body) - - def get_xml_acl(self, key_name='', headers=None, version_id=None): - query_args = 'acl' - if version_id: - query_args += '&versionId=%s' % version_id - response = self.connection.make_request('GET', self.name, key_name, - query_args=query_args, - headers=headers) - body = response.read() - if response.status != 200: - raise S3ResponseError(response.status, response.reason, body) - return body - - def set_xml_acl(self, acl_str, key_name='', headers=None, version_id=None): - query_args = 'acl' - if version_id: - query_args += '&versionId=%s' % version_id - response = self.connection.make_request('PUT', self.name, key_name, - data=acl_str, - query_args=query_args, - headers=headers) - body = response.read() - if response.status != 200: - raise S3ResponseError(response.status, response.reason, body) - - def set_acl(self, acl_or_str, key_name='', headers=None, version_id=None): - if isinstance(acl_or_str, Policy): - self.set_xml_acl(acl_or_str.to_xml(), key_name, - headers, version_id) - else: - self.set_canned_acl(acl_or_str, key_name, - headers, version_id) - - def get_acl(self, key_name='', headers=None, version_id=None): - query_args = 'acl' - if version_id: - query_args += '&versionId=%s' % version_id - response = self.connection.make_request('GET', self.name, key_name, - query_args=query_args, - headers=headers) - body = response.read() - if response.status == 200: - policy = Policy(self) - h = handler.XmlHandler(policy, self) - xml.sax.parseString(body, h) - return policy - else: - raise S3ResponseError(response.status, response.reason, body) - - def make_public(self, recursive=False, headers=None): - self.set_canned_acl('public-read', headers=headers) - if recursive: - for key in self: - self.set_canned_acl('public-read', key.name, headers=headers) - - def add_email_grant(self, permission, email_address, - recursive=False, headers=None): - """ - Convenience method that provides a quick way to add an email grant - to a bucket. This method retrieves the current ACL, creates a new - grant based on the parameters passed in, adds that grant to the ACL - and then PUT's the new ACL back to S3. - - :type permission: string - :param permission: The permission being granted. Should be one of: - (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). - - :type email_address: string - :param email_address: The email address associated with the AWS - account your are granting the permission to. - - :type recursive: boolean - :param recursive: A boolean value to controls whether the command - will apply the grant to all keys within the bucket - or not. The default value is False. By passing a - True value, the call will iterate through all keys - in the bucket and apply the same grant to each key. - CAUTION: If you have a lot of keys, this could take - a long time! - """ - if permission not in S3Permissions: - raise S3PermissionsError('Unknown Permission: %s' % permission) - policy = self.get_acl(headers=headers) - policy.acl.add_email_grant(permission, email_address) - self.set_acl(policy, headers=headers) - if recursive: - for key in self: - key.add_email_grant(permission, email_address, headers=headers) - - def add_user_grant(self, permission, user_id, recursive=False, headers=None): - """ - Convenience method that provides a quick way to add a canonical user grant to a bucket. - This method retrieves the current ACL, creates a new grant based on the parameters - passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. - - :type permission: string - :param permission: The permission being granted. Should be one of: - (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). - - :type user_id: string - :param user_id: The canonical user id associated with the AWS account your are granting - the permission to. - - :type recursive: boolean - :param recursive: A boolean value to controls whether the command - will apply the grant to all keys within the bucket - or not. The default value is False. By passing a - True value, the call will iterate through all keys - in the bucket and apply the same grant to each key. - CAUTION: If you have a lot of keys, this could take - a long time! - """ - if permission not in S3Permissions: - raise S3PermissionsError('Unknown Permission: %s' % permission) - policy = self.get_acl(headers=headers) - policy.acl.add_user_grant(permission, user_id) - self.set_acl(policy, headers=headers) - if recursive: - for key in self: - key.add_user_grant(permission, user_id, headers=headers) - - def list_grants(self, headers=None): - policy = self.get_acl(headers=headers) - return policy.acl.grants - - def get_location(self): - """ - Returns the LocationConstraint for the bucket. - - :rtype: str - :return: The LocationConstraint for the bucket or the empty string if - no constraint was specified when bucket was created. - """ - response = self.connection.make_request('GET', self.name, - query_args='location') - body = response.read() - if response.status == 200: - rs = ResultSet(self) - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs.LocationConstraint - else: - raise S3ResponseError(response.status, response.reason, body) - - def enable_logging(self, target_bucket, target_prefix='', headers=None): - if isinstance(target_bucket, Bucket): - target_bucket = target_bucket.name - body = self.BucketLoggingBody % (target_bucket, target_prefix) - response = self.connection.make_request('PUT', self.name, data=body, - query_args='logging', headers=headers) - body = response.read() - if response.status == 200: - return True - else: - raise S3ResponseError(response.status, response.reason, body) - - def disable_logging(self, headers=None): - body = self.EmptyBucketLoggingBody - response = self.connection.make_request('PUT', self.name, data=body, - query_args='logging', headers=headers) - body = response.read() - if response.status == 200: - return True - else: - raise S3ResponseError(response.status, response.reason, body) - - def get_logging_status(self, headers=None): - response = self.connection.make_request('GET', self.name, - query_args='logging', headers=headers) - body = response.read() - if response.status == 200: - return body - else: - raise S3ResponseError(response.status, response.reason, body) - - def set_as_logging_target(self, headers=None): - policy = self.get_acl(headers=headers) - g1 = Grant(permission='WRITE', type='Group', uri=self.LoggingGroup) - g2 = Grant(permission='READ_ACP', type='Group', uri=self.LoggingGroup) - policy.acl.add_grant(g1) - policy.acl.add_grant(g2) - self.set_acl(policy, headers=headers) - - def get_request_payment(self, headers=None): - response = self.connection.make_request('GET', self.name, - query_args='requestPayment', headers=headers) - body = response.read() - if response.status == 200: - return body - else: - raise S3ResponseError(response.status, response.reason, body) - - def set_request_payment(self, payer='BucketOwner', headers=None): - body = self.BucketPaymentBody % payer - response = self.connection.make_request('PUT', self.name, data=body, - query_args='requestPayment', headers=headers) - body = response.read() - if response.status == 200: - return True - else: - raise S3ResponseError(response.status, response.reason, body) - - def configure_versioning(self, versioning, mfa_delete=False, - mfa_token=None, headers=None): - """ - Configure versioning for this bucket. - Note: This feature is currently in beta release and is available - only in the Northern California region. - - :type versioning: bool - :param versioning: A boolean indicating whether version is - enabled (True) or disabled (False). - - :type mfa_delete: bool - :param mfa_delete: A boolean indicating whether the Multi-Factor - Authentication Delete feature is enabled (True) - or disabled (False). If mfa_delete is enabled - then all Delete operations will require the - token from your MFA device to be passed in - the request. - - :type mfa_token: tuple or list of strings - :param mfa_token: A tuple or list consisting of the serial number - from the MFA device and the current value of - the six-digit token associated with the device. - This value is required when you are changing - the status of the MfaDelete property of - the bucket. - """ - if versioning: - ver = 'Enabled' - else: - ver = 'Suspended' - if mfa_delete: - mfa = 'Enabled' - else: - mfa = 'Disabled' - body = self.VersioningBody % (ver, mfa) - if mfa_token: - if not headers: - headers = {} - headers['x-amz-mfa'] = ' '.join(mfa_token) - response = self.connection.make_request('PUT', self.name, data=body, - query_args='versioning', headers=headers) - body = response.read() - if response.status == 200: - return True - else: - raise S3ResponseError(response.status, response.reason, body) - - def get_versioning_status(self, headers=None): - """ - Returns the current status of versioning on the bucket. - - :rtype: dict - :returns: A dictionary containing a key named 'Versioning' - that can have a value of either Enabled, Disabled, - or Suspended. Also, if MFADelete has ever been enabled - on the bucket, the dictionary will contain a key - named 'MFADelete' which will have a value of either - Enabled or Suspended. - """ - response = self.connection.make_request('GET', self.name, - query_args='versioning', headers=headers) - body = response.read() - boto.log.debug(body) - if response.status == 200: - d = {} - ver = re.search(self.VersionRE, body) - if ver: - d['Versioning'] = ver.group(1) - mfa = re.search(self.MFADeleteRE, body) - if mfa: - d['MfaDelete'] = mfa.group(1) - return d - else: - raise S3ResponseError(response.status, response.reason, body) - - def delete(self, headers=None): - return self.connection.delete_bucket(self.name, headers=headers) diff --git a/src/s3ql/backends/boto/s3/bucketlistresultset.py b/src/s3ql/backends/boto/s3/bucketlistresultset.py deleted file mode 100644 index b3ce4b6..0000000 --- a/src/s3ql/backends/boto/s3/bucketlistresultset.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -def bucket_lister(bucket, prefix='', delimiter='', marker='', headers=None): - """ - A generator function for listing keys in a bucket. - """ - more_results = True - k = None - while more_results: - rs = bucket.get_all_keys(prefix=prefix, marker=marker, - delimiter=delimiter, headers=headers) - for k in rs: - yield k - if k: - marker = k.name - more_results= rs.is_truncated - -class BucketListResultSet: - """ - A resultset for listing keys within a bucket. Uses the bucket_lister - generator function and implements the iterator interface. This - transparently handles the results paging from S3 so even if you have - many thousands of keys within the bucket you can iterate over all - keys in a reasonably efficient manner. - """ - - def __init__(self, bucket=None, prefix='', delimiter='', marker='', headers=None): - self.bucket = bucket - self.prefix = prefix - self.delimiter = delimiter - self.marker = marker - self.headers = headers - - def __iter__(self): - return bucket_lister(self.bucket, prefix=self.prefix, - delimiter=self.delimiter, marker=self.marker, headers=self.headers) - -def versioned_bucket_lister(bucket, prefix='', delimiter='', - key_marker='', version_id_marker='', headers=None): - """ - A generator function for listing versions in a bucket. - """ - more_results = True - k = None - while more_results: - rs = bucket.get_all_versions(prefix=prefix, key_marker=key_marker, - version_id_marker=version_id_marker, - delimiter=delimiter, headers=headers) - for k in rs: - yield k - key_marker = rs.next_key_marker - version_id_marker = rs.next_version_id_marker - more_results= rs.is_truncated - -class VersionedBucketListResultSet: - """ - A resultset for listing versions within a bucket. Uses the bucket_lister - generator function and implements the iterator interface. This - transparently handles the results paging from S3 so even if you have - many thousands of keys within the bucket you can iterate over all - keys in a reasonably efficient manner. - """ - - def __init__(self, bucket=None, prefix='', delimiter='', key_marker='', - version_id_marker='', headers=None): - self.bucket = bucket - self.prefix = prefix - self.delimiter = delimiter - self.key_marker = key_marker - self.version_id_marker = version_id_marker - self.headers = headers - - def __iter__(self): - return versioned_bucket_lister(self.bucket, prefix=self.prefix, - delimiter=self.delimiter, - key_marker=self.key_marker, - version_id_marker=self.version_id_marker, - headers=self.headers) - - diff --git a/src/s3ql/backends/boto/s3/connection.py b/src/s3ql/backends/boto/s3/connection.py deleted file mode 100644 index 2770a4d..0000000 --- a/src/s3ql/backends/boto/s3/connection.py +++ /dev/null @@ -1,360 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -import xml.sax -import urllib, base64 -import time -from .. import utils -import types -from ..connection import AWSAuthConnection -from .. import handler -from .bucket import Bucket -from .key import Key -from ..resultset import ResultSet -from ..exception import S3ResponseError, S3CreateError, BotoClientError - -def assert_case_insensitive(f): - def wrapper(*args, **kwargs): - if len(args) == 3 and not (args[2].islower() or args[2].isalnum()): - raise BotoClientError("Bucket names cannot contain upper-case " \ - "characters when using either the sub-domain or virtual " \ - "hosting calling format.") - return f(*args, **kwargs) - return wrapper - -class _CallingFormat: - - def build_url_base(self, connection, protocol, server, bucket, key=''): - url_base = '%s://' % protocol - url_base += self.build_host(server, bucket) - url_base += connection.get_path(self.build_path_base(bucket, key)) - return url_base - - def build_host(self, server, bucket): - if bucket == '': - return server - else: - return self.get_bucket_server(server, bucket) - - def build_auth_path(self, bucket, key=''): - path = '' - if bucket != '': - path = '/' + bucket - return path + '/%s' % urllib.quote(key) - - def build_path_base(self, bucket, key=''): - return '/%s' % urllib.quote(key) - -class SubdomainCallingFormat(_CallingFormat): - - @assert_case_insensitive - def get_bucket_server(self, server, bucket): - return '%s.%s' % (bucket, server) - -class VHostCallingFormat(_CallingFormat): - - @assert_case_insensitive - def get_bucket_server(self, server, bucket): - return bucket - -class OrdinaryCallingFormat(_CallingFormat): - - def get_bucket_server(self, server, bucket): - return server - - def build_path_base(self, bucket, key=''): - path_base = '/' - if bucket: - path_base += "%s/" % bucket - return path_base + urllib.quote(key) - -class Location: - DEFAULT = '' - EU = 'EU' - USWest = 'us-west-1' - -#boto.set_stream_logger('s3') - -class S3Connection(AWSAuthConnection): - - DefaultHost = 's3.amazonaws.com' - QueryString = 'Signature=%s&Expires=%d&AWSAccessKeyId=%s' - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, - host=DefaultHost, debug=0, https_connection_factory=None, - calling_format=SubdomainCallingFormat(), path='/', provider='aws'): - self.calling_format = calling_format - AWSAuthConnection.__init__(self, host, - aws_access_key_id, aws_secret_access_key, - is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, - debug=debug, https_connection_factory=https_connection_factory, - path=path, provider=provider) - - def __iter__(self): - for bucket in self.get_all_buckets(): - yield bucket - - def __contains__(self, bucket_name): - return not (self.lookup(bucket_name) is None) - - def build_post_policy(self, expiration_time, conditions): - """ - Taken from the AWS book Python examples and modified for use with boto - """ - if type(expiration_time) != time.struct_time: - raise 'Policy document must include a valid expiration Time object' - - # Convert conditions object mappings to condition statements - - return '{"expiration": "%s",\n"conditions": [%s]}' % \ - (time.strftime(boto.utils.ISO8601, expiration_time), ",".join(conditions)) - - - def build_post_form_args(self, bucket_name, key, expires_in = 6000, - acl = None, success_action_redirect = None, max_content_length = None, - http_method = "http", fields=None, conditions=None): - """ - Taken from the AWS book Python examples and modified for use with boto - This only returns the arguments required for the post form, not the actual form - This does not return the file input field which also needs to be added - - :param bucket_name: Bucket to submit to - :type bucket_name: string - - :param key: Key name, optionally add ${filename} to the end to attach the submitted filename - :type key: string - - :param expires_in: Time (in seconds) before this expires, defaults to 6000 - :type expires_in: integer - - :param acl: ACL rule to use, if any - :type acl: :class:`boto.s3.acl.ACL` - - :param success_action_redirect: URL to redirect to on success - :type success_action_redirect: string - - :param max_content_length: Maximum size for this file - :type max_content_length: integer - - :type http_method: string - :param http_method: HTTP Method to use, "http" or "https" - - - :rtype: dict - :return: A dictionary containing field names/values as well as a url to POST to - - .. code-block:: python - - { - "action": action_url_to_post_to, - "fields": [ - { - "name": field_name, - "value": field_value - }, - { - "name": field_name2, - "value": field_value2 - } - ] - } - - """ - if fields == None: - fields = [] - if conditions == None: - conditions = [] - expiration = time.gmtime(int(time.time() + expires_in)) - - # Generate policy document - conditions.append('{"bucket": "%s"}' % bucket_name) - if key.endswith("${filename}"): - conditions.append('["starts-with", "$key", "%s"]' % key[:-len("${filename}")]) - else: - conditions.append('{"key": "%s"}' % key) - if acl: - conditions.append('{"acl": "%s"}' % acl) - fields.append({ "name": "acl", "value": acl}) - if success_action_redirect: - conditions.append('{"success_action_redirect": "%s"}' % success_action_redirect) - fields.append({ "name": "success_action_redirect", "value": success_action_redirect}) - if max_content_length: - conditions.append('["content-length-range", 0, %i]' % max_content_length) - fields.append({"name":'content-length-range', "value": "0,%i" % max_content_length}) - - policy = self.build_post_policy(expiration, conditions) - - # Add the base64-encoded policy document as the 'policy' field - policy_b64 = base64.b64encode(policy) - fields.append({"name": "policy", "value": policy_b64}) - - # Add the AWS access key as the 'AWSAccessKeyId' field - fields.append({"name": "AWSAccessKeyId", "value": self.aws_access_key_id}) - - # Add signature for encoded policy document as the 'AWSAccessKeyId' field - hmac_copy = self.hmac.copy() - hmac_copy.update(policy_b64) - signature = base64.encodestring(hmac_copy.digest()).strip() - fields.append({"name": "signature", "value": signature}) - fields.append({"name": "key", "value": key}) - - # HTTPS protocol will be used if the secure HTTP option is enabled. - url = '%s://%s.s3.amazonaws.com/' % (http_method, bucket_name) - - return {"action": url, "fields": fields} - - - def generate_url(self, expires_in, method, bucket='', key='', - headers=None, query_auth=True, force_http=False): - if not headers: - headers = {} - expires = int(time.time() + expires_in) - auth_path = self.calling_format.build_auth_path(bucket, key) - auth_path = self.get_path(auth_path) - canonical_str = boto.utils.canonical_string(method, auth_path, - headers, expires) - hmac_copy = self.hmac.copy() - hmac_copy.update(canonical_str) - b64_hmac = base64.encodestring(hmac_copy.digest()).strip() - encoded_canonical = urllib.quote_plus(b64_hmac) - self.calling_format.build_path_base(bucket, key) - if query_auth: - query_part = '?' + self.QueryString % (encoded_canonical, expires, - self.aws_access_key_id) - if 'x-amz-security-token' in headers: - query_part += '&x-amz-security-token=%s' % urllib.quote(headers['x-amz-security-token']); - else: - query_part = '' - if force_http: - protocol = 'http' - port = 80 - else: - protocol = self.protocol - port = self.port - return self.calling_format.build_url_base(self, protocol, self.server_name(port), - bucket, key) + query_part - - def get_all_buckets(self, headers=None): - response = self.make_request('GET') - body = response.read() - if response.status > 300: - raise S3ResponseError(response.status, response.reason, body) - rs = ResultSet([('Bucket', Bucket)]) - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - - def get_canonical_user_id(self, headers=None): - """ - Convenience method that returns the "CanonicalUserID" of the user who's credentials - are associated with the connection. The only way to get this value is to do a GET - request on the service which returns all buckets associated with the account. As part - of that response, the canonical userid is returned. This method simply does all of - that and then returns just the user id. - - :rtype: string - :return: A string containing the canonical user id. - """ - rs = self.get_all_buckets(headers=headers) - return rs.ID - - def get_bucket(self, bucket_name, validate=True, headers=None): - bucket = Bucket(self, bucket_name) - if validate: - bucket.get_all_keys(headers, maxkeys=0) - return bucket - - def lookup(self, bucket_name, validate=True, headers=None): - try: - bucket = self.get_bucket(bucket_name, validate, headers=headers) - except: - bucket = None - return bucket - - def create_bucket(self, bucket_name, headers=None, - location=Location.DEFAULT, policy=None): - """ - Creates a new located bucket. By default it's in the USA. You can pass - Location.EU to create an European bucket. - - :type bucket_name: string - :param bucket_name: The name of the new bucket - - :type headers: dict - :param headers: Additional headers to pass along with the request to AWS. - - :type location: :class:`boto.s3.connection.Location` - :param location: The location of the new bucket - - :type policy: :class:`boto.s3.acl.CannedACLStrings` - :param policy: A canned ACL policy that will be applied to the new key in S3. - - """ - # Not sure what Exception Type from boto.exception to use. - if not bucket_name.islower(): - raise Exception("Bucket names must be lower case.") - - if policy: - if headers: - headers['x-amz-acl'] = policy - else: - headers = {'x-amz-acl' : policy} - if location == Location.DEFAULT: - data = '' - else: - data = '<CreateBucketConstraint><LocationConstraint>' + \ - location + '</LocationConstraint></CreateBucketConstraint>' - response = self.make_request('PUT', bucket_name, headers=headers, - data=data) - body = response.read() - if response.status == 409: - raise S3CreateError(response.status, response.reason, body) - if response.status == 200: - return Bucket(self, bucket_name) - else: - raise S3ResponseError(response.status, response.reason, body) - - def delete_bucket(self, bucket, headers=None): - response = self.make_request('DELETE', bucket, headers=headers) - body = response.read() - if response.status != 204: - raise S3ResponseError(response.status, response.reason, body) - - def make_request(self, method, bucket='', key='', headers=None, data='', - query_args=None, sender=None): - if isinstance(bucket, Bucket): - bucket = bucket.name - if isinstance(key, Key): - key = key.name - path = self.calling_format.build_path_base(bucket, key) - auth_path = self.calling_format.build_auth_path(bucket, key) - host = self.calling_format.build_host(self.server_name(), bucket) - if query_args: - path += '?' + query_args - auth_path += '?' + query_args - return AWSAuthConnection.make_request(self, method, path, headers, - data, host, auth_path, sender) - diff --git a/src/s3ql/backends/boto/s3/key.py b/src/s3ql/backends/boto/s3/key.py deleted file mode 100644 index 5047e68..0000000 --- a/src/s3ql/backends/boto/s3/key.py +++ /dev/null @@ -1,901 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -import mimetypes -import os -import rfc822 -import StringIO -import base64 -from .. import utils -from ... import boto -from ..exception import S3ResponseError, S3DataError, BotoClientError -from .user import User -from ...boto import UserAgent, config -try: - from hashlib import md5 -except ImportError: - from md5 import md5 - - -class Key(object): - - DefaultContentType = 'application/octet-stream' - - BufferSize = 8192 - - def __init__(self, bucket=None, name=None): - self.bucket = bucket - self.name = name - self.metadata = {} - self.content_type = self.DefaultContentType - self.content_encoding = None - self.filename = None - self.etag = None - self.last_modified = None - self.owner = None - self.storage_class = 'STANDARD' - self.md5 = None - self.base64md5 = None - self.path = None - self.resp = None - self.mode = None - self.size = None - self.version_id = None - self.source_version_id = None - self.delete_marker = False - - def __repr__(self): - if self.bucket: - return '<Key: %s,%s>' % (self.bucket.name, self.name) - else: - return '<Key: None,%s>' % self.name - - def __getattr__(self, name): - if name == 'key': - return self.name - else: - raise AttributeError - - def __setattr__(self, name, value): - if name == 'key': - self.__dict__['name'] = value - else: - self.__dict__[name] = value - - def __iter__(self): - return self - - def handle_version_headers(self, resp): - self.version_id = resp.getheader('x-amz-version-id', None) - self.source_version_id = resp.getheader('x-amz-copy-source-version-id', None) - if resp.getheader('x-amz-delete-marker', 'false') == 'true': - self.delete_marker = True - else: - self.delete_marker = False - - def open_read(self, headers=None, query_args=None): - """ - Open this key for reading - - :type headers: dict - :param headers: Headers to pass in the web request - - :type query_args: string - :param query_args: Arguments to pass in the query string (ie, 'torrent') - """ - if self.resp == None: - self.mode = 'r' - - self.resp = self.bucket.connection.make_request('GET', - self.bucket.name, - self.name, headers, - query_args=query_args) - if self.resp.status < 199 or self.resp.status > 299: - body = self.resp.read() - raise S3ResponseError(self.resp.status, self.resp.reason, body) - response_headers = self.resp.msg - self.metadata = boto.utils.get_aws_metadata(response_headers) - for name, value in response_headers.items(): - if name.lower() == 'content-length': - self.size = int(value) - elif name.lower() == 'etag': - self.etag = value - elif name.lower() == 'content-type': - self.content_type = value - elif name.lower() == 'content-encoding': - self.content_encoding = value - elif name.lower() == 'last-modified': - self.last_modified = value - self.handle_version_headers(self.resp) - - def open_write(self, headers=None): - """ - Open this key for writing. - Not yet implemented - - :type headers: dict - :param headers: Headers to pass in the write request - """ - raise BotoClientError('Not Implemented') - - def open(self, mode='r', headers=None, query_args=None): - if mode == 'r': - self.mode = 'r' - self.open_read(headers=headers, query_args=query_args) - elif mode == 'w': - self.mode = 'w' - self.open_write(headers=headers) - else: - raise BotoClientError('Invalid mode: %s' % mode) - - closed = False - def close(self): - if self.resp: - self.resp.read() - self.resp = None - self.mode = None - self.closed = True - - def next(self): - """ - By providing a next method, the key object supports use as an iterator. - For example, you can now say: - - for bytes in key: - write bytes to a file or whatever - - All of the HTTP connection stuff is handled for you. - """ - self.open_read() - data = self.resp.read(self.BufferSize) - if not data: - self.close() - raise StopIteration - return data - - def read(self, size=0): - if size == 0: - size = self.BufferSize - self.open_read() - data = self.resp.read(size) - if not data: - self.close() - return data - - def change_storage_class(self, new_storage_class, dst_bucket=None): - """ - Change the storage class of an existing key. - Depending on whether a different destination bucket is supplied - or not, this will either move the item within the bucket, preserving - all metadata and ACL info bucket changing the storage class or it - will copy the item to the provided destination bucket, also - preserving metadata and ACL info. - - :type new_storage_class: string - :param new_storage_class: The new storage class for the Key. - Possible values are: - * STANDARD - * REDUCED_REDUNDANCY - - :type dst_bucket: string - :param dst_bucket: The name of a destination bucket. If not - provided the current bucket of the key - will be used. - - """ - self.storage_class = new_storage_class - return self.copy(self.bucket.name, self.name, - reduced_redundancy=True, preserve_acl=True) - - def copy(self, dst_bucket, dst_key, metadata=None, - reduced_redundancy=False, preserve_acl=False): - """ - Copy this Key to another bucket. - - :type dst_bucket: string - :param dst_bucket: The name of the destination bucket - - :type dst_key: string - :param dst_key: The name of the destination key - - :type metadata: dict - :param metadata: Metadata to be associated with new key. - If metadata is supplied, it will replace the - metadata of the source key being copied. - If no metadata is supplied, the source key's - metadata will be copied to the new key. - - :type reduced_redundancy: bool - :param reduced_redundancy: If True, this will force the storage - class of the new Key to be - REDUCED_REDUNDANCY regardless of the - storage class of the key being copied. - The Reduced Redundancy Storage (RRS) - feature of S3, provides lower - redundancy at lower storage cost. - - :type preserve_acl: bool - :param preserve_acl: If True, the ACL from the source key - will be copied to the destination - key. If False, the destination key - will have the default ACL. - Note that preserving the ACL in the - new key object will require two - additional API calls to S3, one to - retrieve the current ACL and one to - set that ACL on the new object. If - you don't care about the ACL, a value - of False will be significantly more - efficient. - - :rtype: :class:`boto.s3.key.Key` or subclass - :returns: An instance of the newly created key object - """ - dst_bucket = self.bucket.connection.lookup(dst_bucket) - if reduced_redundancy: - storage_class = 'REDUCED_REDUNDANCY' - else: - storage_class = self.storage_class - return dst_bucket.copy_key(dst_key, self.bucket.name, - self.name, metadata, - storage_class=storage_class, - preserve_acl=preserve_acl) - - def startElement(self, name, attrs, connection): - if name == 'Owner': - self.owner = User(self) - return self.owner - else: - return None - - def endElement(self, name, value, connection): - if name == 'Key': - self.name = value.encode('utf-8') - elif name == 'ETag': - self.etag = value - elif name == 'LastModified': - self.last_modified = value - elif name == 'Size': - self.size = int(value) - elif name == 'StorageClass': - self.storage_class = value - elif name == 'Owner': - pass - elif name == 'VersionId': - self.version_id = value - else: - setattr(self, name, value) - - def exists(self): - """ - Returns True if the key exists - - :rtype: bool - :return: Whether the key exists on S3 - """ - return bool(self.bucket.lookup(self.name)) - - def delete(self): - """ - Delete this key from S3 - """ - return self.bucket.delete_key(self.name) - - def get_metadata(self, name): - return self.metadata.get(name) - - def set_metadata(self, name, value): - self.metadata[name] = value - - def update_metadata(self, d): - self.metadata.update(d) - - # convenience methods for setting/getting ACL - def set_acl(self, acl_str, headers=None): - if self.bucket != None: - self.bucket.set_acl(acl_str, self.name, headers=headers) - - def get_acl(self, headers=None): - if self.bucket != None: - return self.bucket.get_acl(self.name, headers=headers) - - def get_xml_acl(self, headers=None): - if self.bucket != None: - return self.bucket.get_xml_acl(self.name, headers=headers) - - def set_xml_acl(self, acl_str, headers=None): - if self.bucket != None: - return self.bucket.set_xml_acl(acl_str, self.name, headers=headers) - - def set_canned_acl(self, acl_str, headers=None): - return self.bucket.set_canned_acl(acl_str, self.name, headers) - - def make_public(self, headers=None): - return self.bucket.set_canned_acl('public-read', self.name, headers) - - def generate_url(self, expires_in, method='GET', headers=None, - query_auth=True, force_http=False): - """ - Generate a URL to access this key. - - :type expires_in: int - :param expires_in: How long the url is valid for, in seconds - - :type method: string - :param method: The method to use for retrieving the file (default is GET) - - :type headers: dict - :param headers: Any headers to pass along in the request - - :type query_auth: bool - :param query_auth: - - :rtype: string - :return: The URL to access the key - """ - return self.bucket.connection.generate_url(expires_in, method, - self.bucket.name, self.name, - headers, query_auth, force_http) - - def send_file(self, fp, headers=None, cb=None, num_cb=10): - """ - Upload a file to a key into a bucket on S3. - - :type fp: file - :param fp: The file pointer to upload - - :type headers: dict - :param headers: The headers to pass along with the PUT request - - :type cb: function - :param cb: a callback function that will be called to report - progress on the upload. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted to S3 and the second representing - the total number of bytes that need to be transmitted. - - :type num_cb: int - :param num_cb: (optional) If a callback is specified with the cb - parameter this parameter determines the granularity - of the callback by defining the maximum number of - times the callback will be called during the file - transfer. Providing a negative integer will cause - your callback to be called with each buffer read. - - """ - def sender(http_conn, method, path, data, headers): - http_conn.putrequest(method, path) - for key in headers: - http_conn.putheader(key, headers[key]) - http_conn.endheaders() - fp.seek(0) - save_debug = self.bucket.connection.debug - self.bucket.connection.debug = 0 - if cb: - if num_cb > 2: - cb_count = self.size / self.BufferSize / (num_cb - 2) - elif num_cb < 0: - cb_count = -1 - else: - cb_count = 0 - i = total_bytes = 0 - cb(total_bytes, self.size) - l = fp.read(self.BufferSize) - while len(l) > 0: - http_conn.send(l) - if cb: - total_bytes += len(l) - i += 1 - if i == cb_count or cb_count == -1: - cb(total_bytes, self.size) - i = 0 - l = fp.read(self.BufferSize) - if cb: - cb(total_bytes, self.size) - response = http_conn.getresponse() - body = response.read() - fp.seek(0) - self.bucket.connection.debug = save_debug - if response.status == 500 or response.status == 503 or \ - response.getheader('location'): - # we'll try again - return response - elif response.status >= 200 and response.status <= 299: - self.etag = response.getheader('etag') - if self.etag != '"%s"' % self.md5: - raise S3DataError('ETag from S3 did not match computed MD5') - return response - else: - raise S3ResponseError(response.status, response.reason, body) - - if not headers: - headers = {} - else: - headers = headers.copy() - headers['User-Agent'] = UserAgent - headers['Content-MD5'] = self.base64md5 - if self.storage_class != 'STANDARD': - headers['x-amz-storage-class'] = self.storage_class - if headers.has_key('Content-Type'): - self.content_type = headers['Content-Type'] - elif self.path: - self.content_type = mimetypes.guess_type(self.path)[0] - if self.content_type == None: - self.content_type = self.DefaultContentType - headers['Content-Type'] = self.content_type - else: - headers['Content-Type'] = self.content_type - headers['Content-Length'] = str(self.size) - headers['Expect'] = '100-Continue' - headers = boto.utils.merge_meta(headers, self.metadata) - resp = self.bucket.connection.make_request('PUT', self.bucket.name, - self.name, headers, - sender=sender) - self.handle_version_headers(resp) - - def compute_md5(self, fp): - """ - :type fp: file - :param fp: File pointer to the file to MD5 hash. The file pointer will be - reset to the beginning of the file before the method returns. - - :rtype: tuple - :return: A tuple containing the hex digest version of the MD5 hash - as the first element and the base64 encoded version of the - plain digest as the second element. - """ - m = md5() - fp.seek(0) - s = fp.read(self.BufferSize) - while s: - m.update(s) - s = fp.read(self.BufferSize) - hex_md5 = m.hexdigest() - base64md5 = base64.encodestring(m.digest()) - if base64md5[-1] == '\n': - base64md5 = base64md5[0:-1] - self.size = fp.tell() - fp.seek(0) - return (hex_md5, base64md5) - - def set_contents_from_file(self, fp, headers=None, replace=True, - cb=None, num_cb=10, policy=None, md5=None, - reduced_redundancy=False): - """ - Store an object in S3 using the name of the Key object as the - key in S3 and the contents of the file pointed to by 'fp' as the - contents. - - :type fp: file - :param fp: the file whose contents to upload - - :type headers: dict - :param headers: additional HTTP headers that will be sent with the PUT request. - - :type replace: bool - :param replace: If this parameter is False, the method - will first check to see if an object exists in the - bucket with the same key. If it does, it won't - overwrite it. The default value is True which will - overwrite the object. - - :type cb: function - :param cb: a callback function that will be called to report - progress on the upload. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted to S3 and the second representing - the total number of bytes that need to be transmitted. - - :type cb: int - :param num_cb: (optional) If a callback is specified with the cb parameter - this parameter determines the granularity of the callback by defining - the maximum number of times the callback will be called during the file transfer. - - :type policy: :class:`boto.s3.acl.CannedACLStrings` - :param policy: A canned ACL policy that will be applied to the new key in S3. - - :type md5: A tuple containing the hexdigest version of the MD5 checksum of the - file as the first element and the Base64-encoded version of the plain - checksum as the second element. This is the same format returned by - the compute_md5 method. - :param md5: If you need to compute the MD5 for any reason prior to upload, - it's silly to have to do it twice so this param, if present, will be - used as the MD5 values of the file. Otherwise, the checksum will be computed. - - :type reduced_redundancy: bool - :param reduced_redundancy: If True, this will set the storage - class of the new Key to be - REDUCED_REDUNDANCY. The Reduced Redundancy - Storage (RRS) feature of S3, provides lower - redundancy at lower storage cost. - - """ - if headers is None: - headers = {} - if policy: - headers['x-amz-acl'] = policy - if reduced_redundancy: - self.storage_class = 'REDUCED_REDUNDANCY' - headers['x-amz-storage-class'] = self.storage_class - if hasattr(fp, 'name'): - self.path = fp.name - if self.bucket != None: - if not md5: - md5 = self.compute_md5(fp) - self.md5 = md5[0] - self.base64md5 = md5[1] - if self.name == None: - self.name = self.md5 - if not replace: - k = self.bucket.lookup(self.name) - if k: - return - self.send_file(fp, headers, cb, num_cb) - - def set_contents_from_filename(self, filename, headers=None, replace=True, - cb=None, num_cb=10, policy=None, md5=None, - reduced_redundancy=False): - """ - Store an object in S3 using the name of the Key object as the - key in S3 and the contents of the file named by 'filename'. - See set_contents_from_file method for details about the - parameters. - - :type filename: string - :param filename: The name of the file that you want to put onto S3 - - :type headers: dict - :param headers: Additional headers to pass along with the request to AWS. - - :type replace: bool - :param replace: If True, replaces the contents of the file if it already exists. - - :type cb: function - :param cb: (optional) a callback function that will be called to report - progress on the download. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted from S3 and the second representing - the total number of bytes that need to be transmitted. - - :type cb: int - :param num_cb: (optional) If a callback is specified with the cb parameter - this parameter determines the granularity of the callback by defining - the maximum number of times the callback will be called during the file transfer. - - :type policy: :class:`boto.s3.acl.CannedACLStrings` - :param policy: A canned ACL policy that will be applied to the new key in S3. - - :type md5: A tuple containing the hexdigest version of the MD5 checksum of the - file as the first element and the Base64-encoded version of the plain - checksum as the second element. This is the same format returned by - the compute_md5 method. - :param md5: If you need to compute the MD5 for any reason prior to upload, - it's silly to have to do it twice so this param, if present, will be - used as the MD5 values of the file. Otherwise, the checksum will be computed. - - :type reduced_redundancy: bool - :param reduced_redundancy: If True, this will set the storage - class of the new Key to be - REDUCED_REDUNDANCY. The Reduced Redundancy - Storage (RRS) feature of S3, provides lower - redundancy at lower storage cost. - - """ - fp = open(filename, 'rb') - self.set_contents_from_file(fp, headers, replace, cb, num_cb, - policy, md5, reduced_redundancy) - fp.close() - - def set_contents_from_string(self, s, headers=None, replace=True, - cb=None, num_cb=10, policy=None, md5=None, - reduced_redundancy=False): - """ - Store an object in S3 using the name of the Key object as the - key in S3 and the string 's' as the contents. - See set_contents_from_file method for details about the - parameters. - - :type headers: dict - :param headers: Additional headers to pass along with the request to AWS. - - :type replace: bool - :param replace: If True, replaces the contents of the file if it already exists. - - :type cb: function - :param cb: (optional) a callback function that will be called to report - progress on the download. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted from S3 and the second representing - the total number of bytes that need to be transmitted. - - :type cb: int - :param num_cb: (optional) If a callback is specified with the cb parameter - this parameter determines the granularity of the callback by defining - the maximum number of times the callback will be called during the file transfer. - - :type policy: :class:`boto.s3.acl.CannedACLStrings` - :param policy: A canned ACL policy that will be applied to the new key in S3. - - :type md5: A tuple containing the hexdigest version of the MD5 checksum of the - file as the first element and the Base64-encoded version of the plain - checksum as the second element. This is the same format returned by - the compute_md5 method. - :param md5: If you need to compute the MD5 for any reason prior to upload, - it's silly to have to do it twice so this param, if present, will be - used as the MD5 values of the file. Otherwise, the checksum will be computed. - - :type reduced_redundancy: bool - :param reduced_redundancy: If True, this will set the storage - class of the new Key to be - REDUCED_REDUNDANCY. The Reduced Redundancy - Storage (RRS) feature of S3, provides lower - redundancy at lower storage cost. - - """ - fp = StringIO.StringIO(s) - r = self.set_contents_from_file(fp, headers, replace, cb, num_cb, - policy, md5, reduced_redundancy) - fp.close() - return r - - def get_file(self, fp, headers=None, cb=None, num_cb=10, - torrent=False, version_id=None): - """ - Retrieves a file from an S3 Key - - :type fp: file - :param fp: File pointer to put the data into - - :type headers: string - :param: headers to send when retrieving the files - - :type cb: function - :param cb: (optional) a callback function that will be called to report - progress on the download. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted from S3 and the second representing - the total number of bytes that need to be transmitted. - - - :type cb: int - :param num_cb: (optional) If a callback is specified with the cb parameter - this parameter determines the granularity of the callback by defining - the maximum number of times the callback will be called during the file transfer. - - :type torrent: bool - :param torrent: Flag for whether to get a torrent for the file - """ - if cb: - if num_cb > 2: - cb_count = self.size / self.BufferSize / (num_cb - 2) - else: - cb_count = 0 - i = total_bytes = 0 - cb(total_bytes, self.size) - save_debug = self.bucket.connection.debug - if self.bucket.connection.debug == 1: - self.bucket.connection.debug = 0 - - query_args = '' - if torrent: - query_args = 'torrent' - elif version_id: - query_args = 'versionId=%s' % version_id - self.open('r', headers, query_args=query_args) - for bytes in self: - fp.write(bytes) - if cb: - total_bytes += len(bytes) - i += 1 - if i == cb_count: - cb(total_bytes, self.size) - i = 0 - if cb: - cb(total_bytes, self.size) - self.close() - self.bucket.connection.debug = save_debug - - def get_torrent_file(self, fp, headers=None, cb=None, num_cb=10): - """ - Get a torrent file (see to get_file) - - :type fp: file - :param fp: The file pointer of where to put the torrent - - :type headers: dict - :param headers: Headers to be passed - - :type cb: function - :param cb: Callback function to call on retrieved data - - :type cb: int - :param num_cb: (optional) If a callback is specified with the cb parameter - this parameter determines the granularity of the callback by defining - the maximum number of times the callback will be called during the file transfer. - - """ - return self.get_file(fp, headers, cb, num_cb, torrent=True) - - def get_contents_to_file(self, fp, headers=None, - cb=None, num_cb=10, - torrent=False, - version_id=None): - """ - Retrieve an object from S3 using the name of the Key object as the - key in S3. Write the contents of the object to the file pointed - to by 'fp'. - - :type fp: File -like object - :param fp: - - :type headers: dict - :param headers: additional HTTP headers that will be sent with the GET request. - - :type cb: function - :param cb: (optional) a callback function that will be called to report - progress on the download. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted from S3 and the second representing - the total number of bytes that need to be transmitted. - - - :type cb: int - :param num_cb: (optional) If a callback is specified with the cb parameter - this parameter determines the granularity of the callback by defining - the maximum number of times the callback will be called during the file transfer. - - :type torrent: bool - :param torrent: If True, returns the contents of a torrent file as a string. - - """ - if self.bucket != None: - self.get_file(fp, headers, cb, num_cb, torrent=torrent, - version_id=version_id) - - def get_contents_to_filename(self, filename, headers=None, - cb=None, num_cb=10, - torrent=False, - version_id=None): - """ - Retrieve an object from S3 using the name of the Key object as the - key in S3. Store contents of the object to a file named by 'filename'. - See get_contents_to_file method for details about the - parameters. - - :type filename: string - :param filename: The filename of where to put the file contents - - :type headers: dict - :param headers: Any additional headers to send in the request - - :type cb: function - :param cb: (optional) a callback function that will be called to report - progress on the download. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted from S3 and the second representing - the total number of bytes that need to be transmitted. - - - :type cb: int - :param num_cb: (optional) If a callback is specified with the cb parameter - this parameter determines the granularity of the callback by defining - the maximum number of times the callback will be called during the file transfer. - - :type torrent: bool - :param torrent: If True, returns the contents of a torrent file as a string. - - """ - fp = open(filename, 'wb') - self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent, - version_id=version_id) - fp.close() - # if last_modified date was sent from s3, try to set file's timestamp - if self.last_modified != None: - try: - modified_tuple = rfc822.parsedate_tz(self.last_modified) - modified_stamp = int(rfc822.mktime_tz(modified_tuple)) - os.utime(fp.name, (modified_stamp, modified_stamp)) - except Exception: pass - - def get_contents_as_string(self, headers=None, - cb=None, num_cb=10, - torrent=False, - version_id=None): - """ - Retrieve an object from S3 using the name of the Key object as the - key in S3. Return the contents of the object as a string. - See get_contents_to_file method for details about the - parameters. - - :type headers: dict - :param headers: Any additional headers to send in the request - - :type cb: function - :param cb: (optional) a callback function that will be called to report - progress on the download. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted from S3 and the second representing - the total number of bytes that need to be transmitted. - - :type cb: int - :param num_cb: (optional) If a callback is specified with the cb parameter - this parameter determines the granularity of the callback by defining - the maximum number of times the callback will be called during the file transfer. - - - :type cb: int - :param num_cb: (optional) If a callback is specified with the cb parameter - this parameter determines the granularity of the callback by defining - the maximum number of times the callback will be called during the file transfer. - - :type torrent: bool - :param torrent: If True, returns the contents of a torrent file as a string. - - :rtype: string - :returns: The contents of the file as a string - """ - fp = StringIO.StringIO() - self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent, - version_id=version_id) - return fp.getvalue() - - def add_email_grant(self, permission, email_address, headers=None): - """ - Convenience method that provides a quick way to add an email grant to a key. - This method retrieves the current ACL, creates a new grant based on the parameters - passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. - - :type permission: string - :param permission: The permission being granted. Should be one of: - READ|WRITE|READ_ACP|WRITE_ACP|FULL_CONTROL - See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html - for more details on permissions. - - :type email_address: string - :param email_address: The email address associated with the AWS account your are granting - the permission to. - """ - policy = self.get_acl(headers=headers) - policy.acl.add_email_grant(permission, email_address) - self.set_acl(policy, headers=headers) - - def add_user_grant(self, permission, user_id): - """ - Convenience method that provides a quick way to add a canonical user grant to a key. - This method retrieves the current ACL, creates a new grant based on the parameters - passed in, adds that grant to the ACL and then PUT's the new ACL back to S3. - - :type permission: string - :param permission: The permission being granted. Should be one of: - READ|WRITE|READ_ACP|WRITE_ACP|FULL_CONTROL - See http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingAuthAccess.html - for more details on permissions. - - :type user_id: string - :param user_id: The canonical user id associated with the AWS account your are granting - the permission to. - """ - policy = self.get_acl() - policy.acl.add_user_grant(permission, user_id) - self.set_acl(policy) diff --git a/src/s3ql/backends/boto/s3/prefix.py b/src/s3ql/backends/boto/s3/prefix.py deleted file mode 100644 index 25a234b..0000000 --- a/src/s3ql/backends/boto/s3/prefix.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -class Prefix: - def __init__(self, bucket=None, name=None): - self.bucket = bucket - self.name = name - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Prefix': - self.name = value - else: - setattr(self, name, value) - diff --git a/src/s3ql/backends/boto/s3/user.py b/src/s3ql/backends/boto/s3/user.py deleted file mode 100644 index 5d454e6..0000000 --- a/src/s3ql/backends/boto/s3/user.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -class User: - def __init__(self, parent=None, id='', display_name=''): - if parent: - parent.owner = self - self.type = None - self.id = id - self.display_name = display_name - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'DisplayName': - self.display_name = value - elif name == 'ID': - self.id = value - else: - setattr(self, name, value) - - def to_xml(self, element_name='Owner'): - if self.type: - s = '<%s xsi:type="%s">' % (element_name, self.type) - else: - s = '<%s>' % element_name - s += '<ID>%s</ID>' % self.id - s += '<DisplayName>%s</DisplayName>' % self.display_name - s += '</%s>' % element_name - return s diff --git a/src/s3ql/backends/boto/storage_uri.py b/src/s3ql/backends/boto/storage_uri.py deleted file mode 100644 index 44ed189..0000000 --- a/src/s3ql/backends/boto/storage_uri.py +++ /dev/null @@ -1,274 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -import os -from .exception import BotoClientError -from .exception import InvalidUriError - -class StorageUri(object): - """ - Base class for representing storage provider-independent bucket and - object name with a shorthand URI-like syntax. - - This is an abstract class: the constructor cannot be called (throws an - exception if you try). - """ - - connection = None - - def __init__(self): - """Uncallable constructor on abstract base StorageUri class. - """ - raise BotoClientError('Attempt to instantiate abstract StorageUri ' - 'class') - - def __str__(self): - """Returns string representation of URI.""" - return self.uri - - def equals(self, uri): - """Returns true if two URIs are equal.""" - return self.uri == uri.uri - - def connect(self, access_key_id=None, secret_access_key=None, **kwargs): - """ - Opens a connection to appropriate provider, depending on provider - portion of URI. Requires Credentials defined in boto config file (see - boto/pyami/config.py). - @type storage_uri: StorageUri - @param storage_uri: StorageUri specifying a bucket or a bucket+object - @rtype: L{AWSAuthConnection<boto.gs.connection.AWSAuthConnection>} - @return: A connection to storage service provider of the given URI. - """ - - if not self.connection: - if self.provider == 's3': - from boto.s3.connection import S3Connection - self.connection = S3Connection(access_key_id, - secret_access_key, **kwargs) - elif self.provider == 'gs': - from boto.gs.connection import GSConnection - self.connection = GSConnection(access_key_id, - secret_access_key, **kwargs) - elif self.provider == 'file': - from boto.file.connection import FileConnection - self.connection = FileConnection(self) - else: - raise InvalidUriError('Unrecognized provider "%s"' % - self.provider) - self.connection.debug = self.debug - return self.connection - - def delete_key(self, headers=None): - if not self.object_name: - raise InvalidUriError('delete_key on object-less URI (%s)' % - self.uri) - bucket = self.get_bucket() - return bucket.delete_key(self.object_name, headers) - - def get_all_keys(self, headers=None, **params): - bucket = self.get_bucket(headers) - return bucket.get_all_keys(headers, params) - - def get_bucket(self, validate=True, headers=None): - if self.bucket_name is None: - raise InvalidUriError('get_bucket on bucket-less URI (%s)' % - self.uri) - conn = self.connect() - return conn.get_bucket(self.bucket_name, validate, headers) - - def get_key(self): - if not self.object_name: - raise InvalidUriError('get_key on object-less URI (%s)' % self.uri) - bucket = self.get_bucket() - return bucket.get_key(self.object_name) - - def new_key(self): - if not self.object_name: - raise InvalidUriError('new_key on object-less URI (%s)' % self.uri) - bucket = self.get_bucket() - return bucket.new_key(self.object_name) - - def get_contents_as_string(self, headers=None, cb=None, num_cb=10, - torrent=False): - if not self.object_name: - raise InvalidUriError('get_contents_as_string on object-less URI ' - '(%s)' % self.uri) - return self.get_key().get_contents_as_string(headers, cb, num_cb, - torrent) - - -class BucketStorageUri(StorageUri): - """ - StorageUri subclass that handles bucket storage providers. - Callers should instantiate this class by calling boto.storage_uri(). - """ - - def __init__(self, provider, bucket_name=None, object_name=None, - debug=False): - """Instantiate a BucketStorageUri from provider,bucket,object tuple. - - @type provider: string - @param provider: provider name (gs, s3, etc.) - @type bucket_name: string - @param bucket_name: bucket name - @type object_name: string - @param object_name: object name - @type debug: bool - @param debug: whether to turn on debugging on calls to this class - - After instantiation the components are available in the following - fields: uri, provider, bucket_name, object_name. - """ - - self.provider = provider - self.bucket_name = bucket_name - self.object_name = object_name - if self.bucket_name and self.object_name: - self.uri = ('%s://%s/%s' % (self.provider, self.bucket_name, - self.object_name)) - elif self.bucket_name: - self.uri = ('%s://%s' % (self.provider, self.bucket_name)) - else: - self.uri = ('%s://' % self.provider) - self.debug = debug - - def clone_replace_name(self, new_name): - """Instantiate a BucketStorageUri from the current BucketStorageUri, - but replacing the object_name. - - @type new_name: string - @param new_name: new object name - """ - if not self.bucket_name: - raise InvalidUriError('clone_replace_name() on bucket-less URI %s' % - self.uri) - return BucketStorageUri(self.provider, self.bucket_name, new_name, - self.debug) - - def get_acl(self, headers=None): - if not self.bucket_name: - raise InvalidUriError('get_acl on bucket-less URI (%s)' % self.uri) - bucket = self.get_bucket() - # This works for both bucket- and object- level ACL (former passes - # key_name=None): - return bucket.get_acl(self.object_name, headers) - - def names_container(self): - """Returns True if this URI names a bucket (vs. an object). - """ - return self.object_name is None or self.object_name == '' - - def names_singleton(self): - """Returns True if this URI names an object (vs. a bucket). - """ - return self.object_name is not None and self.object_name != '' - - def is_file_uri(self): - return False - - def is_cloud_uri(self): - return True - - def create_bucket(self, headers=None, location='', policy=None): - if self.bucket_name is None: - raise InvalidUriError('create_bucket on bucket-less URI (%s)' % - self.uri) - conn = self.connect() - return conn.create_bucket(self.bucket_name, headers, location, policy) - - def delete_bucket(self, headers=None): - if self.bucket_name is None: - raise InvalidUriError('delete_bucket on bucket-less URI (%s)' % - self.uri) - conn = self.connect() - return conn.delete_bucket(self.bucket_name, headers) - - def get_all_buckets(self, headers=None): - conn = self.connect() - return conn.get_all_buckets(headers) - - def set_acl(self, acl_or_str, key_name='', headers=None): - if not self.bucket_name: - raise InvalidUriError('set_acl on bucket-less URI (%s)' % - self.uri) - self.get_bucket().set_acl(acl_or_str, key_name, headers) - - def set_canned_acl(self, acl_str, headers=None): - if not self.object_name: - raise InvalidUriError('set_canned_acl on object-less URI (%s)' % - self.uri) - key = self.get_key() - key.set_canned_acl(acl_str, headers) - - -class FileStorageUri(StorageUri): - """ - StorageUri subclass that handles files in the local file system. - Callers should instantiate this class by calling boto.storage_uri(). - - See file/README about how we map StorageUri operations onto a file system. - """ - - def __init__(self, object_name, debug): - """Instantiate a FileStorageUri from a path name. - - @type object_name: string - @param object_name: object name - - After instantiation the components are available in the following - fields: uri, provider, bucket_name (always blank for this "anonymous" - bucket), object_name. - """ - - self.provider = 'file' - self.bucket_name = '' - self.object_name = object_name - self.uri = 'file://' + object_name - self.debug = debug - - def clone_replace_name(self, new_name): - """Instantiate a FileStorageUri from the current FileStorageUri, - but replacing the object_name. - - @type new_name: string - @param new_name: new object name - """ - return FileStorageUri(new_name, self.debug) - - def names_container(self): - """Returns True if this URI names a directory. - """ - return os.path.isdir(self.object_name) - - def names_singleton(self): - """Returns True if this URI names a file. - """ - return os.path.isfile(self.object_name) - - def is_file_uri(self): - return True - - def is_cloud_uri(self): - return False diff --git a/src/s3ql/backends/boto/utils.py b/src/s3ql/backends/boto/utils.py deleted file mode 100644 index 93de734..0000000 --- a/src/s3ql/backends/boto/utils.py +++ /dev/null @@ -1,565 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -# -# Parts of this code were copied or derived from sample code supplied by AWS. -# The following notice applies to that code. -# -# This software code is made available "AS IS" without warranties of any -# kind. You may copy, display, modify and redistribute the software -# code either by itself or as incorporated into your code; provided that -# you do not remove any proprietary notices. Your use of this software -# code is at your own risk and you waive any claim against Amazon -# Digital Services, Inc. or its affiliates with respect to your use of -# this software code. (c) 2006 Amazon Digital Services, Inc. or its -# affiliates. - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - -""" -Some handy utility functions used by several classes. -""" - -import re -import urllib -import urllib2 -import subprocess -import StringIO -import time -import logging.handlers -from .. import boto -import tempfile -import smtplib -import datetime -from email.MIMEMultipart import MIMEMultipart -from email.MIMEBase import MIMEBase -from email.MIMEText import MIMEText -from email.Utils import formatdate -from email import Encoders - -try: - import hashlib - _hashfn = hashlib.sha512 -except ImportError: - import md5 - _hashfn = md5.md5 - -METADATA_PREFIX = 'x-amz-meta-' -AMAZON_HEADER_PREFIX = 'x-amz-' - -# generates the aws canonical string for the given parameters -def canonical_string(method, path, headers, expires=None): - interesting_headers = {} - for key in headers: - lk = key.lower() - if lk in ['content-md5', 'content-type', 'date'] or lk.startswith(AMAZON_HEADER_PREFIX): - interesting_headers[lk] = headers[key].strip() - - # these keys get empty strings if they don't exist - if not interesting_headers.has_key('content-type'): - interesting_headers['content-type'] = '' - if not interesting_headers.has_key('content-md5'): - interesting_headers['content-md5'] = '' - - # just in case someone used this. it's not necessary in this lib. - if interesting_headers.has_key('x-amz-date'): - interesting_headers['date'] = '' - - # if you're using expires for query string auth, then it trumps date - # (and x-amz-date) - if expires: - interesting_headers['date'] = str(expires) - - sorted_header_keys = interesting_headers.keys() - sorted_header_keys.sort() - - buf = "%s\n" % method - for key in sorted_header_keys: - val = interesting_headers[key] - if key.startswith(AMAZON_HEADER_PREFIX): - buf += "%s:%s\n" % (key, val) - else: - buf += "%s\n" % val - - # don't include anything after the first ? in the resource... - buf += "%s" % path.split('?')[0] - - # ...unless there is an acl or torrent parameter - if re.search("[&?]acl($|=|&)", path): - buf += "?acl" - elif re.search("[&?]logging($|=|&)", path): - buf += "?logging" - elif re.search("[&?]torrent($|=|&)", path): - buf += "?torrent" - elif re.search("[&?]location($|=|&)", path): - buf += "?location" - elif re.search("[&?]requestPayment($|=|&)", path): - buf += "?requestPayment" - elif re.search("[&?]versions($|=|&)", path): - buf += "?versions" - elif re.search("[&?]versioning($|=|&)", path): - buf += "?versioning" - else: - m = re.search("[&?]versionId=([^&]+)($|=|&)", path) - if m: - buf += '?versionId=' + m.group(1) - - return buf - -def merge_meta(headers, metadata): - final_headers = headers.copy() - for k in metadata.keys(): - if k.lower() in ['cache-control', 'content-md5', 'content-type', - 'content-encoding', 'content-disposition', - 'date', 'expires']: - final_headers[k] = metadata[k] - else: - final_headers[METADATA_PREFIX + k] = metadata[k] - - return final_headers - -def get_aws_metadata(headers): - metadata = {} - for hkey in headers.keys(): - if hkey.lower().startswith(METADATA_PREFIX): - #val = urllib.unquote_plus(headers[hkey]) - #metadata[hkey[len(METADATA_PREFIX):]] = unicode(val, 'utf-8') - metadata[hkey[len(METADATA_PREFIX):]] = headers[hkey] - del headers[hkey] - return metadata - -def retry_url(url, retry_on_404=True): - for i in range(0, 10): - try: - req = urllib2.Request(url) - resp = urllib2.urlopen(req) - return resp.read() - except urllib2.HTTPError, e: - # in 2.6 you use getcode(), in 2.5 and earlier you use code - if hasattr(e, 'getcode'): - code = e.getcode() - else: - code = e.code - if code == 404 and not retry_on_404: - return '' - except: - pass - boto.log.exception('Caught exception reading instance data') - time.sleep(2 ** i) - boto.log.error('Unable to read instance data, giving up') - return '' - -def _get_instance_metadata(url): - d = {} - data = retry_url(url) - if data: - fields = data.split('\n') - for field in fields: - if field.endswith('/'): - d[field[0:-1]] = _get_instance_metadata(url + field) - else: - p = field.find('=') - if p > 0: - key = field[p + 1:] - resource = field[0:p] + '/openssh-key' - else: - key = resource = field - val = retry_url(url + resource) - p = val.find('\n') - if p > 0: - val = val.split('\n') - d[key] = val - return d - -def get_instance_metadata(version='latest'): - """ - Returns the instance metadata as a nested Python dictionary. - Simple values (e.g. local_hostname, hostname, etc.) will be - stored as string values. Values such as ancestor-ami-ids will - be stored in the dict as a list of string values. More complex - fields such as public-keys and will be stored as nested dicts. - """ - url = 'http://169.254.169.254/%s/meta-data/' % version - return _get_instance_metadata(url) - -def get_instance_userdata(version='latest', sep=None): - url = 'http://169.254.169.254/%s/user-data' % version - user_data = retry_url(url, retry_on_404=False) - if user_data: - if sep: - l = user_data.split(sep) - user_data = {} - for nvpair in l: - t = nvpair.split('=') - user_data[t[0].strip()] = t[1].strip() - return user_data - -ISO8601 = '%Y-%m-%dT%H:%M:%SZ' - -def get_ts(ts=None): - if not ts: - ts = time.gmtime() - return time.strftime(ISO8601, ts) - -def parse_ts(ts): - return datetime.datetime.strptime(ts, ISO8601) - -def find_class(module_name, class_name=None): - if class_name: - module_name = "%s.%s" % (module_name, class_name) - modules = module_name.split('.') - c = None - - try: - for m in modules[1:]: - if c: - c = getattr(c, m) - else: - c = getattr(__import__(".".join(modules[0:-1])), m) - return c - except: - return None - -def update_dme(username, password, dme_id, ip_address): - """ - Update your Dynamic DNS record with DNSMadeEasy.com - """ - dme_url = 'https://www.dnsmadeeasy.com/servlet/updateip' - dme_url += '?username=%s&password=%s&id=%s&ip=%s' - s = urllib2.urlopen(dme_url % (username, password, dme_id, ip_address)) - return s.read() - -def fetch_file(uri, file=None, username=None, password=None): - """ - Fetch a file based on the URI provided. If you do not pass in a file pointer - a tempfile.NamedTemporaryFile, or None if the file could not be - retrieved is returned. - The URI can be either an HTTP url, or "s3://bucket_name/key_name" - """ - boto.log.info('Fetching %s' % uri) - if file == None: - file = tempfile.NamedTemporaryFile() - try: - if uri.startswith('s3://'): - bucket_name, key_name = uri[len('s3://'):].split('/', 1) - c = boto.connect_s3() - bucket = c.get_bucket(bucket_name) - key = bucket.get_key(key_name) - key.get_contents_to_file(file) - else: - if username and password: - passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - passman.add_password(None, uri, username, password) - authhandler = urllib2.HTTPBasicAuthHandler(passman) - opener = urllib2.build_opener(authhandler) - urllib2.install_opener(opener) - s = urllib2.urlopen(uri) - file.write(s.read()) - file.seek(0) - except: - raise - boto.log.exception('Problem Retrieving file: %s' % uri) - file = None - return file - -class ShellCommand(object): - - def __init__(self, command, wait=True): - self.exit_code = 0 - self.command = command - self.log_fp = StringIO.StringIO() - self.wait = wait - self.run() - - def run(self): - boto.log.info('running:%s' % self.command) - self.process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if(self.wait): - while self.process.poll() == None: - time.sleep(1) - t = self.process.communicate() - self.log_fp.write(t[0]) - self.log_fp.write(t[1]) - boto.log.info(self.log_fp.getvalue()) - self.exit_code = self.process.returncode - return self.exit_code - - def setReadOnly(self, value): - raise AttributeError - - def getStatus(self): - return self.exit_code - - status = property(getStatus, setReadOnly, None, 'The exit code for the command') - - def getOutput(self): - return self.log_fp.getvalue() - - output = property(getOutput, setReadOnly, None, 'The STDIN and STDERR output of the command') - -class AuthSMTPHandler(logging.handlers.SMTPHandler): - """ - This class extends the SMTPHandler in the standard Python logging module - to accept a username and password on the constructor and to then use those - credentials to authenticate with the SMTP server. To use this, you could - add something like this in your boto config file: - - [handler_hand07] - class=boto.utils.AuthSMTPHandler - level=WARN - formatter=form07 - args=('localhost', 'username', 'password', 'from@abc', ['user1@abc', 'user2@xyz'], 'Logger Subject') - """ - - def __init__(self, mailhost, username, password, fromaddr, toaddrs, subject): - """ - Initialize the handler. - - We have extended the constructor to accept a username/password - for SMTP authentication. - """ - logging.handlers.SMTPHandler.__init__(self, mailhost, fromaddr, toaddrs, subject) - self.username = username - self.password = password - - def emit(self, record): - """ - Emit a record. - - Format the record and send it to the specified addressees. - It would be really nice if I could add authorization to this class - without having to resort to cut and paste inheritance but, no. - """ - try: - port = self.mailport - if not port: - port = smtplib.SMTP_PORT - smtp = smtplib.SMTP(self.mailhost, port) - smtp.login(self.username, self.password) - msg = self.format(record) - msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\n\r\n%s" % ( - self.fromaddr, - ','.join(self.toaddrs), - self.getSubject(record), - formatdate(), msg) - smtp.sendmail(self.fromaddr, self.toaddrs, msg) - smtp.quit() - except (KeyboardInterrupt, SystemExit): - raise - except: - self.handleError(record) - -class LRUCache(dict): - """A dictionary-like object that stores only a certain number of items, and - discards its least recently used item when full. - - >>> cache = LRUCache(3) - >>> cache['A'] = 0 - >>> cache['B'] = 1 - >>> cache['C'] = 2 - >>> len(cache) - 3 - - >>> cache['A'] - 0 - - Adding new items to the cache does not increase its size. Instead, the least - recently used item is dropped: - - >>> cache['D'] = 3 - >>> len(cache) - 3 - >>> 'B' in cache - False - - Iterating over the cache returns the keys, starting with the most recently - used: - - >>> for key in cache: - ... print key - D - A - C - - This code is based on the LRUCache class from Genshi which is based on - Mighty's LRUCache from ``myghtyutils.util``, written - by Mike Bayer and released under the MIT license (Genshi uses the - BSD License). See: - - http://svn.myghty.org/myghtyutils/trunk/lib/myghtyutils/util.py - """ - - class _Item(object): - def __init__(self, key, value): - self.previous = self.next = None - self.key = key - self.value = value - def __repr__(self): - return repr(self.value) - - def __init__(self, capacity): - self._dict = dict() - self.capacity = capacity - self.head = None - self.tail = None - - def __contains__(self, key): - return key in self._dict - - def __iter__(self): - cur = self.head - while cur: - yield cur.key - cur = cur.next - - def __len__(self): - return len(self._dict) - - def __getitem__(self, key): - item = self._dict[key] - self._update_item(item) - return item.value - - def __setitem__(self, key, value): - item = self._dict.get(key) - if item is None: - item = self._Item(key, value) - self._dict[key] = item - self._insert_item(item) - else: - item.value = value - self._update_item(item) - self._manage_size() - - def __repr__(self): - return repr(self._dict) - - def _insert_item(self, item): - item.previous = None - item.next = self.head - if self.head is not None: - self.head.previous = item - else: - self.tail = item - self.head = item - self._manage_size() - - def _manage_size(self): - while len(self._dict) > self.capacity: - del self._dict[self.tail.key] - if self.tail != self.head: - self.tail = self.tail.previous - self.tail.next = None - else: - self.head = self.tail = None - - def _update_item(self, item): - if self.head == item: - return - - previous = item.previous - previous.next = item.next - if item.next is not None: - item.next.previous = previous - else: - self.tail = previous - - item.previous = None - item.next = self.head - self.head.previous = self.head = item - -class Password(object): - """ - Password object that stores itself as SHA512 hashed. - """ - def __init__(self, str=None): - """ - Load the string from an initial value, this should be the raw SHA512 hashed password - """ - self.str = str - - def set(self, value): - self.str = _hashfn(value).hexdigest() - - def __str__(self): - return str(self.str) - - def __eq__(self, other): - if other == None: - return False - return str(_hashfn(other).hexdigest()) == str(self.str) - - def __len__(self): - if self.str: - return len(self.str) - else: - return 0 - -def notify(subject, body=None, html_body=None, to_string=None, attachments=[], append_instance_id=True): - if append_instance_id: - subject = "[%s] %s" % (boto.config.get_value("Instance", "instance-id"), subject) - if not to_string: - to_string = boto.config.get_value('Notification', 'smtp_to', None) - if to_string: - try: - from_string = boto.config.get_value('Notification', 'smtp_from', 'boto') - msg = MIMEMultipart() - msg['From'] = from_string - msg['To'] = to_string - msg['Date'] = formatdate(localtime=True) - msg['Subject'] = subject - - if body: - msg.attach(MIMEText(body)) - - if html_body: - part = MIMEBase('text', 'html') - part.set_payload(html_body) - Encoders.encode_base64(part) - msg.attach(part) - - for part in attachments: - msg.attach(part) - - smtp_host = boto.config.get_value('Notification', 'smtp_host', 'localhost') - - # Alternate port support - if boto.config.get_value("Notification", "smtp_port"): - server = smtplib.SMTP(smtp_host, int(boto.config.get_value("Notification", "smtp_port"))) - else: - server = smtplib.SMTP(smtp_host) - - # TLS support - if boto.config.getbool("Notification", "smtp_tls"): - server.ehlo() - server.starttls() - server.ehlo() - smtp_user = boto.config.get_value('Notification', 'smtp_user', '') - smtp_pass = boto.config.get_value('Notification', 'smtp_pass', '') - if smtp_user: - server.login(smtp_user, smtp_pass) - server.sendmail(from_string, to_string, msg.as_string()) - server.quit() - except: - boto.log.exception('notify failed') - diff --git a/src/s3ql/backends/common.py b/src/s3ql/backends/common.py index 4b0bb16..7e22b9a 100644 --- a/src/s3ql/backends/common.py +++ b/src/s3ql/backends/common.py @@ -3,126 +3,110 @@ common.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import +from ..common import QuietError +from abc import ABCMeta, abstractmethod +from base64 import b64decode, b64encode from cStringIO import StringIO -import tempfile -import hmac -import logging -import pycryptopp +from contextlib import contextmanager +from getpass import getpass +from pycryptopp.cipher import aes +import ConfigParser +import bz2 import cPickle as pickle -import time import hashlib -import zlib -import os -import bz2 +import hmac +import logging import lzma -from base64 import b64decode, b64encode +import os +import re +import stat import struct -from abc import ABCMeta, abstractmethod +import sys +import threading +import time +import zlib + +# Not available in every pycryptopp version +if hasattr(aes, 'start_up_self_test'): + aes.start_up_self_test() log = logging.getLogger("backend") -__all__ = [ 'AbstractConnection', 'AbstractBucket', 'ChecksumError', 'UnsupportedError', - 'NoSuchObject', 'NoSuchBucket' ] +HMAC_SIZE = 32 def sha256(s): return hashlib.sha256(s).digest() -class AbstractConnection(object): - '''This class contains functionality shared between all backends. - - All derived classes are expected to be completely threadsafe - (except for internal methods starting with underscore) - ''' - __metaclass__ = ABCMeta - - def bucket_exists(self, name): - """Check if the bucket `name` exists""" - - try: - self.get_bucket(name) - except NoSuchBucket: - return False - else: - return True +class BucketPool(object): + '''A pool of buckets - def __contains__(self, name): - return self.bucket_exists(name) - - def close(self): - '''Close connection. + This class is threadsafe. All methods (except for internal methods + starting with underscore) may be called concurrently by different + threads. + ''' + + def __init__(self, factory): + '''Init pool - If this method is not called, the interpreter may be kept alive by - background threads initiated by the connection. + *factory* should be a callable that provides new + connections. ''' - pass - - def prepare_fork(self): - '''Prepare connection for forking + + self.factory = factory + self.pool = [] + self.lock = threading.Lock() - This method must be called before the process is forked, so that - the connection can properly terminate any threads that it uses. + def pop_conn(self): + '''Pop connection from pool''' - The connection (or any of its bucket objects) can not be used - between the calls to `prepare_fork()` and `finish_fork()`. - ''' - pass - - def finish_fork(self): - '''Re-initalize connection after forking + with self.lock: + if self.pool: + return self.pool.pop() + else: + return self.factory() + + def push_conn(self, conn): + '''Push connection back into pool''' - This method must be called after the process has forked, so that - the connection can properly restart any threads that it may - have stopped for the fork. + with self.lock: + self.pool.append(conn) - The connection (or any of its bucket objects) can not be used - between the calls to `prepare_fork()` and `finish_fork()`. - ''' - pass - - @abstractmethod - def create_bucket(self, name, passphrase=None, compression=None): - """Create bucket and return `Bucket` instance""" - pass - - @abstractmethod - def get_bucket(self, name, passphrase=None, compression=None): - """Get `Bucket` instance for bucket `name`""" - pass - - @abstractmethod - def delete_bucket(self, name, recursive=False): - """Delete bucket + @contextmanager + def __call__(self): + '''Provide connection from pool (context manager)''' - If `recursive` is False and the bucket still contains objects, the call - will fail. - """ - pass - + conn = self.pop_conn() + try: + yield conn + finally: + self.push_conn(conn) + class AbstractBucket(object): - '''This class contains functionality shared between all backends. + '''Functionality shared between all backends. Instances behave similarly to dicts. They can be iterated over and indexed into, but raise a separate set of exceptions. - All derived classes are expected to be completely threadsafe - (except for internal methods starting with underscore) + The bucket guarantees get after create consistency, i.e. a newly created + object will be immediately retrievable. Additional consistency guarantees + may or may not be available and can be queried for with instance methods. ''' __metaclass__ = ABCMeta - def __init__(self, passphrase, compression): - self.passphrase = passphrase - self.compression = compression + needs_login = True + + def __init__(self): super(AbstractBucket, self).__init__() def __getitem__(self, key): return self.fetch(key)[0] - + def __setitem__(self, key, value): self.store(key, value) @@ -139,19 +123,6 @@ class AbstractBucket(object): for key in self.list(): yield (key, self[key]) - def lookup(self, key): - """Return metadata for given key. - - If the key does not exist, `NoSuchObject` is raised. - """ - - if not isinstance(key, str): - raise TypeError('key must be of type str') - - meta_raw = self.raw_lookup(key) - return self._get_meta(meta_raw)[0] - - def fetch(self, key): """Return data stored under `key`. @@ -160,13 +131,10 @@ class AbstractBucket(object): ``bucket.fetch(key)[0]``. """ - if not isinstance(key, str): - raise TypeError('key must be of type str') - - fh = StringIO() - meta = self.fetch_fh(key, fh) - - return (fh.getvalue(), meta) + with self.open_read(key) as fh: + data = fh.read() + + return (data, fh.metadata) def store(self, key, val, metadata=None): """Store data under `key`. @@ -177,115 +145,225 @@ class AbstractBucket(object): If no metadata is required, one can simply assign to the subscripted bucket instead of using this function: ``bucket[key] = val`` is equivalent to ``bucket.store(key, val)``. + """ - Returns the size of the stored object (after compression). + with self.open_write(key, metadata) as fh: + fh.write(val) + + @abstractmethod + def lookup(self, key): + """Return metadata for given key. + + If the key does not exist, `NoSuchObject` is raised. """ - if isinstance(val, unicode): - val = val.encode('us-ascii') - if not isinstance(key, str): - raise TypeError('key must be of type str') + pass + + @abstractmethod + def open_read(self, key): + """Open object for reading - fh = StringIO(val) - return self.store_fh(key, fh, metadata) + Return a tuple of a file-like object. Bucket contents can be read from + the file-like object, metadata is stored in its *metadata* attribute and + can be modified by the caller at will. The object must be closed + explicitly. + """ + + pass + + @abstractmethod + def open_write(self, key, metadata=None): + """Open object for writing - def _get_meta(self, meta_raw, plain=False): - '''Get metadata & decompressor factory + `metadata` can be a dict of additional attributes to store with the + object. Returns a file-like object. The object must be closed + explicitly. + """ + + pass + + @abstractmethod + def is_get_consistent(self): + '''If True, objects retrievals are guaranteed to be up-to-date - If the bucket has a password set - but the object is not encrypted, `ObjectNotEncrypted` is raised - unless `plain` is true. + If this method returns True, then creating, deleting, or overwriting an + object is guaranteed to be immediately reflected in subsequent object + retrieval attempts. ''' + pass + + @abstractmethod + def is_list_create_consistent(self): + '''If True, new objects are guaranteed to show up in object listings + + If this method returns True, creation of objects will immediately be + reflected when retrieving the list of available objects. + ''' + pass + + @abstractmethod + def clear(self): + """Delete all objects in bucket""" + pass + + def contains(self, key): + '''Check if `key` is in bucket''' + + try: + self.lookup(key) + except NoSuchObject: + return False + else: + return True - convert_legacy_metadata(meta_raw) + @abstractmethod + def delete(self, key, force=False): + """Delete object stored under `key` - compr_alg = meta_raw['compression'] - encr_alg = meta_raw['encryption'] - encrypted = (encr_alg != 'None') + ``bucket.delete(key)`` can also be written as ``del bucket[key]``. + If `force` is true, do not return an error if the key does not exist. + """ + pass - if encrypted: - if not self.passphrase: - raise ChecksumError('Encrypted object and no passphrase supplied') + @abstractmethod + def list(self, prefix=''): + '''List keys in bucket - if encr_alg != 'AES': - raise RuntimeError('Unsupported encryption') - elif self.passphrase and not plain: - raise ObjectNotEncrypted() + Returns an iterator over all keys in the bucket. + ''' + pass - if compr_alg == 'BZIP2': - decomp = bz2.BZ2Decompressor - elif compr_alg == 'LZMA': - decomp = lzma.LZMADecompressor - elif compr_alg == 'ZLIB': - decomp = zlib.decompressobj - elif compr_alg == 'None': - decomp = DummyDecompressor - else: - raise RuntimeError('Unsupported compression: %s' % compr_alg) + @abstractmethod + def copy(self, src, dest): + """Copy data stored under key `src` to key `dest` + + If `dest` already exists, it will be overwritten. The copying + is done on the remote side. + """ + + pass - if 'meta' in meta_raw: - buf = b64decode(meta_raw['meta']) - if encrypted: - buf = decrypt(buf, self.passphrase) - metadata = pickle.loads(buf) - else: - metadata = dict() + def rename(self, src, dest): + """Rename key `src` to `dest` + + If `dest` already exists, it will be overwritten. The rename + is done on the remote side. + """ + + self.copy(src, dest) + self.delete(src) + +class BetterBucket(AbstractBucket): + ''' + This class adds encryption, compression and integrity protection to a plain + bucket. + ''' - return (metadata, decomp) + def __init__(self, passphrase, compression, bucket): + super(BetterBucket, self).__init__() + + self.passphrase = passphrase + self.compression = compression + self.bucket = bucket - def fetch_fh(self, key, fh, plain=False): - """Fetch data for `key` and write to `fh` + if compression not in ('bzip2', 'lzma', 'zlib', None): + raise ValueError('Unsupported compression: %s' % compression) + + def lookup(self, key): + """Return metadata for given key. - Return a dictionary with the metadata. If the bucket has a password set - but the object is not encrypted, `ObjectNotEncrypted` is raised - unless `plain` is true. + If the key does not exist, `NoSuchObject` is raised. """ - if not isinstance(key, str): - raise TypeError('key must be of type str') + metadata = self.bucket.lookup(key) + convert_legacy_metadata(metadata) + return self._unwrap_meta(metadata) - tmp = tempfile.TemporaryFile() - (fh, tmp) = (tmp, fh) + def _unwrap_meta(self, metadata): + '''Unwrap metadata + + If the bucket has a password set but the object is not encrypted, + `ObjectNotEncrypted` is raised. + ''' - meta_raw = self.raw_fetch(key, fh) - (metadata, decomp) = self._get_meta(meta_raw, plain) + encr_alg = metadata['encryption'] + encrypted = (encr_alg != 'None') - (fh, tmp) = (tmp, fh) - tmp.seek(0) - fh.seek(0) - if self.passphrase: - decrypt_uncompress_fh(tmp, fh, self.passphrase, decomp()) - else: - uncompress_fh(tmp, fh, decomp()) - tmp.close() + if encrypted and not self.passphrase: + raise ChecksumError('Encrypted object and no passphrase supplied') - return metadata + elif not encrypted and self.passphrase: + raise ObjectNotEncrypted() - def store_fh(self, key, fh, metadata=None): - """Store data in `fh` under `key` + buf = b64decode(metadata['meta']) + if encrypted: + buf = decrypt(buf, self.passphrase) + + try: + metadata = pickle.loads(buf) + except pickle.UnpicklingError as exc: + if (isinstance(exc.args[0], str) + and exc.args[0].startswith('invalid load key')): + raise ChecksumError('Invalid metadata') + raise + + if metadata is None: + return dict() + else: + return metadata - `metadata` can be a dict of additional attributes to store with the - object. + def open_read(self, key): + """Open object for reading + + Return a tuple of a file-like object. Bucket contents can be read from + the file-like object, metadata is stored in its *metadata* attribute and + can be modified by the caller at will. The object must be closed explicitly. - Returns the size of the stored object (after compression). + If the bucket has a password set but the object is not encrypted, + `ObjectNotEncrypted` is raised. """ - (size, fn) = self.prep_store_fh(key, fh, metadata) - fn() - return size - def prep_store_fh(self, key, fh, metadata=None): - """Prepare to store data in `fh` under `key` + fh = self.bucket.open_read(key) + convert_legacy_metadata(fh.metadata) - `metadata` can be a dict of additional attributes to store with the - object. The method compresses and encrypts the data and returns a tuple - `(size, fn)`, where `fn` is a function that does the actual network - transaction and `size` is the size of the object after compression - and encryption. - """ + compr_alg = fh.metadata['compression'] + encr_alg = fh.metadata['encryption'] + + metadata = self._unwrap_meta(fh.metadata) - if not isinstance(key, str): - raise TypeError('key must be of type str') + if compr_alg == 'BZIP2': + decompressor = bz2.BZ2Decompressor() + elif compr_alg == 'LZMA': + decompressor = lzma.LZMADecompressor() + elif compr_alg == 'ZLIB': + decompressor = zlib.decompressobj() + elif compr_alg == 'None': + decompressor = None + else: + raise RuntimeError('Unsupported compression: %s' % compr_alg) + + if encr_alg == 'AES': + fh = LegacyDecryptDecompressFilter(fh, self.passphrase, decompressor) + else: + if encr_alg == 'AES_v2': + fh = DecryptFilter(fh, self.passphrase) + elif encr_alg != 'None': + raise RuntimeError('Unsupported encryption: %s' % encr_alg) + + if decompressor: + fh = DecompressFilter(fh, decompressor) + + fh.metadata = metadata + + return fh + def open_write(self, key, metadata=None): + """Open object for writing + + `metadata` can be a dict of additional attributes to store with the + object. Returns a file-like object. + """ + # We always store metadata (even if it's just None), so that we can # verify that the object has been created by us when we call lookup(). meta_buf = pickle.dumps(metadata, 2) @@ -293,12 +371,13 @@ class AbstractBucket(object): meta_raw = dict() if self.passphrase: - meta_raw['encryption'] = 'AES' + meta_raw['encryption'] = 'AES_v2' nonce = struct.pack(b'<f', time.time() - time.timezone) + bytes(key) meta_raw['meta'] = b64encode(encrypt(meta_buf, self.passphrase, nonce)) else: meta_raw['encryption'] = 'None' meta_raw['meta'] = b64encode(meta_buf) + nonce = None if self.compression == 'zlib': compr = zlib.compressobj(9) @@ -310,254 +389,448 @@ class AbstractBucket(object): compr = lzma.LZMACompressor(options={ 'level': 7 }) meta_raw['compression'] = 'LZMA' elif not self.compression: - compr = DummyCompressor() + compr = None meta_raw['compression'] = 'None' - else: - raise ValueError('Invalid compression algorithm') - # We need to generate a temporary copy to determine the size of the - # object (which needs to transmitted as Content-Length) - tmp = tempfile.TemporaryFile() - fh.seek(0) - if self.passphrase: - compress_encrypt_fh(fh, tmp, self.passphrase, nonce, compr) - else: - compress_fh(fh, tmp, compr) - tmp.seek(0, os.SEEK_END) - size = tmp.tell() - tmp.seek(0) - return (size, lambda: self.raw_store(key, tmp, meta_raw)) + fh = self.bucket.open_write(key, meta_raw) - @abstractmethod - def read_after_create_consistent(self): - '''Does this backend provide read-after-create consistency?''' - pass - - @abstractmethod - def read_after_write_consistent(self): - '''Does this backend provide read-after-write consistency? + if nonce: + fh = EncryptFilter(fh, self.passphrase, nonce) + if compr: + fh = CompressFilter(fh, compr) + + return fh + + def is_get_consistent(self): + '''If True, objects retrievals are guaranteed to be up-to-date - (This does not includes read-after-delete) + If this method returns True, then creating, deleting, or overwriting an + object is guaranteed to be immediately reflected in subsequent object + retrieval attempts. ''' - pass + return self.bucket.is_get_consistent() + + def is_list_create_consistent(self): + '''If True, new objects are guaranteed to show up in object listings - @abstractmethod - def read_after_delete_consistent(self): - '''Does this backend provide read-after-delete consistency?''' - pass - - @abstractmethod - def __str__(self): - pass - - @abstractmethod + If this method returns True, creation of objects will immediately be + reflected when retrieving the list of available objects. + ''' + return self.bucket.is_get_consistent() + def clear(self): """Delete all objects in bucket""" - pass + return self.bucket.clear() - @abstractmethod def contains(self, key): '''Check if `key` is in bucket''' - pass + return self.bucket.contains(key) - @abstractmethod - def raw_lookup(self, key): - '''Return meta data for `key`''' - pass - - @abstractmethod def delete(self, key, force=False): """Delete object stored under `key` ``bucket.delete(key)`` can also be written as ``del bucket[key]``. If `force` is true, do not return an error if the key does not exist. """ - pass + return self.bucket.delete(key, force) - @abstractmethod def list(self, prefix=''): '''List keys in bucket Returns an iterator over all keys in the bucket. ''' - pass - - @abstractmethod - def raw_fetch(self, key, fh): - '''Fetch contents stored under `key` and write them into `fh`''' - pass - - @abstractmethod - def raw_store(self, key, fh, metadata): - '''Store contents of `fh` in `key` with metadata - - `metadata` has to be a dict with lower-case keys. - ''' - pass + return self.bucket.list(prefix) def copy(self, src, dest): """Copy data stored under key `src` to key `dest` If `dest` already exists, it will be overwritten. The copying - is done on the remote side. If the backend does not support - this operation, raises `UnsupportedError`. + is done on the remote side. """ - # Unused arguments - #pylint: disable=W0613 - raise UnsupportedError('Backend does not support remote copy') + return self.bucket.copy(src, dest) def rename(self, src, dest): """Rename key `src` to `dest` If `dest` already exists, it will be overwritten. The rename - is done on the remote side. If the backend does not support - this operation, raises `UnsupportedError`. + is done on the remote side. """ - # Unused arguments - #pylint: disable=W0613 - raise UnsupportedError('Backend does not support remote rename') - - -class UnsupportedError(Exception): - '''Raised if a backend does not support a particular operation''' - - pass - - -def decrypt_uncompress_fh(ifh, ofh, passphrase, decomp): - '''Read `ofh` and write decrypted, uncompressed data to `ofh`''' - - bs = 256 * 1024 - - # Read nonce - len_ = struct.unpack(b'<B', ifh.read(struct.calcsize(b'<B')))[0] - nonce = ifh.read(len_) - - key = sha256(passphrase + nonce) - cipher = pycryptopp.cipher.aes.AES(key) - hmac_ = hmac.new(key, digestmod=hashlib.sha256) - - # Read (encrypted) hmac - hash_ = ifh.read(32) # Length of hash - - while True: - buf = ifh.read(bs) - if not buf: - break - - buf = cipher.process(buf) - try: - buf = decomp.decompress(buf) - except IOError: - raise ChecksumError('Invalid compressed stream') - - if buf: - hmac_.update(buf) - ofh.write(buf) - - if decomp.unused_data: - raise ChecksumError('Data after end of compressed stream') - - # Decompress hmac - hash_ = cipher.process(hash_) - - if hash_ != hmac_.digest(): - raise ChecksumError('HMAC mismatch') - -def uncompress_fh(ifh, ofh, decomp): - '''Read `ofh` and write uncompressed data to `ofh`''' - - bs = 256 * 1024 - while True: - buf = ifh.read(bs) - if not buf: - break - - try: - buf = decomp.decompress(buf) - except IOError: - raise ChecksumError('Invalid compressed stream') - - if buf: - ofh.write(buf) - - if decomp.unused_data: - raise ChecksumError('Data after end of compressed stream') + return self.bucket.rename(src, dest) -class DummyDecompressor(object): +class AbstractInputFilter(object): + '''Process data while reading''' + + __metaclass__ = ABCMeta + def __init__(self): - super(DummyDecompressor, self).__init__() - self.unused_data = None + super(AbstractInputFilter, self).__init__() + self.bs = 256 * 1024 + self.buffer = '' + + def read(self, size=None): + '''Try to read *size* bytes + + If *None*, read until EOF. + ''' + + if size is None: + remaining = 1<<31 + else: + remaining = size - len(self.buffer) + + while remaining > 0: + buf = self._read(self.bs) + if not buf: + break + remaining -= len(buf) + self.buffer += buf + + if size is None: + buf = self.buffer + self.buffer = '' + else: + buf = self.buffer[:size] + self.buffer = self.buffer[size:] + + return buf + + @abstractmethod + def _read(self, size): + '''Read roughly *size* bytes''' + pass + +class CompressFilter(object): + '''Compress data while writing + + The `compr_size` attribute is used to keep track of the + compressed size. + ''' + + def __init__(self, fh, compr): + '''Initialize + + *fh* should be a file-like object. *decomp* should be a fresh compressor + instance with a *compress* method. + ''' + super(CompressFilter, self).__init__() + + self.fh = fh + self.compr = compr + self.compr_size = 0 + + def write(self, data): + '''Write *data*''' + + buf = self.compr.compress(data) + if buf: + self.fh.write(buf) + self.compr_size += len(buf) + + def close(self): + buf = self.compr.flush() + self.fh.write(buf) + self.compr_size += len(buf) + self.fh.close() - def decompress(self, buf): + def __enter__(self): + return self + + def __exit__(self, *a): + self.close() + return False + +class DecompressFilter(AbstractInputFilter): + '''Decompress data while reading''' + + def __init__(self, fh, decomp, metadata=None): + '''Initialize + + *fh* should be a file-like object. *decomp* should be a + fresh decompressor instance with a *decompress* method. + ''' + super(DecompressFilter, self).__init__() + + self.fh = fh + self.decomp = decomp + self.metadata = metadata + + def _read(self, size): + '''Read roughly *size* bytes''' + + buf = '' + while not buf: + buf = self.fh.read(size) + if not buf: + if self.decomp.unused_data: + raise ChecksumError('Data after end of compressed stream') + return '' + + try: + buf = self.decomp.decompress(buf) + except IOError as exc: + if exc.args == ('invalid data stream',): + raise ChecksumError('Invalid compressed stream') + raise + except lzma.error as exc: + if exc.args == ('unknown file format',): + raise ChecksumError('Invalid compressed stream') + raise + except zlib.error as exc: + if exc.args[0].startswith('Error -3 while decompressing:'): + log.warn('LegacyDecryptDecompressFilter._read(): %s', + exc.args[0]) + raise ChecksumError('Invalid compressed stream') + raise + return buf + + def close(self): + self.fh.close() + + def __enter__(self): + return self + + def __exit__(self, *a): + self.close() + return False + +class EncryptFilter(object): + '''Encrypt data while writing''' + + def __init__(self, fh, passphrase, nonce): + '''Initialize + + *fh* should be a file-like object. + ''' + super(EncryptFilter, self).__init__() + + self.fh = fh + + if isinstance(nonce, unicode): + nonce = nonce.encode('utf-8') + + self.key = sha256(passphrase + nonce) + self.cipher = aes.AES(self.key) #IGNORE:E1102 + self.hmac = hmac.new(self.key, digestmod=hashlib.sha256) + + self.fh.write(struct.pack(b'<B', len(nonce))) + self.fh.write(nonce) + + def write(self, data): + '''Write *data* + + len(data) must be < 2**32. + + Every invocation of `write` generates a packet that contains both the + length of the data and the data, so the passed data should have + reasonable size (if the data is written in e.g. 4 byte chunks, it is + blown up by 100%) + ''' + + if len(data) == 0: + return + + buf = struct.pack(b'<I', len(data)) + data + self.hmac.update(buf) + buf = self.cipher.process(buf) + if buf: + self.fh.write(buf) + + def close(self): + # Packet length of 0 indicates end of stream, only HMAC follows + buf = struct.pack(b'<I', 0) + self.hmac.update(buf) + buf = self.cipher.process(buf + self.hmac.digest()) + self.fh.write(buf) + self.fh.close() + + def __enter__(self): + return self + + def __exit__(self, *a): + self.close() + return False + +class DecryptFilter(AbstractInputFilter): + '''Decrypt data while reading + + Reader has to read the entire stream in order for HMAC + checking to work. + ''' + + def __init__(self, fh, passphrase, metadata=None): + '''Initialize + + *fh* should be a file-like object. + ''' + super(DecryptFilter, self).__init__() + + self.fh = fh + self.off_size = struct.calcsize(b'<I') + self.remaining = 0 # Remaining length of current packet + self.metadata = metadata + self.hmac_checked = False + + # Read nonce + len_ = struct.unpack(b'<B', fh.read(struct.calcsize(b'<B')))[0] + nonce = fh.read(len_) + + key = sha256(passphrase + nonce) + self.cipher = aes.AES(key) #IGNORE:E1102 + self.hmac = hmac.new(key, digestmod=hashlib.sha256) + + def _read(self, size): + '''Read roughly *size* bytes''' + + buf = self.fh.read(size) + if not buf: + if not self.hmac_checked: + raise ChecksumError('HMAC mismatch') + return '' + + inbuf = self.cipher.process(buf) + outbuf = '' + while True: + + if len(inbuf) <= self.remaining: + self.remaining -= len(inbuf) + self.hmac.update(inbuf) + outbuf += inbuf + break + + outbuf += inbuf[:self.remaining] + self.hmac.update(inbuf[:self.remaining+self.off_size]) + paket_size = struct.unpack(b'<I', inbuf[self.remaining + :self.remaining+self.off_size])[0] + inbuf = inbuf[self.remaining + self.off_size:] + self.remaining = paket_size + + # End of file, read and check HMAC + if paket_size == 0: + if len(inbuf) != HMAC_SIZE: + inbuf += self.cipher.process(self.fh.read(HMAC_SIZE - len(inbuf))) + if inbuf != self.hmac.digest(): + raise ChecksumError('HMAC mismatch') + self.hmac_checked = True + break + + return outbuf -class DummyCompressor(object): - def flush(self): - return '' + def close(self): + self.fh.close() - def compress(self, buf): - return buf + def __enter__(self): + return self + + def __exit__(self, *a): + self.close() + return False + +class LegacyDecryptDecompressFilter(AbstractInputFilter): + '''Decrypt and Decompress data while reading + + Reader has to read the entire stream in order for HMAC + checking to work. + ''' + + def __init__(self, fh, passphrase, decomp, metadata=None): + '''Initialize + + *fh* should be a file-like object. + ''' + super(LegacyDecryptDecompressFilter, self).__init__() + + self.fh = fh + self.metadata = metadata + self.decomp = decomp + self.hmac_checked = False + + # Read nonce + len_ = struct.unpack(b'<B', fh.read(struct.calcsize(b'<B')))[0] + nonce = fh.read(len_) + self.hash = fh.read(HMAC_SIZE) + + key = sha256(passphrase + nonce) + self.cipher = aes.AES(key) #IGNORE:E1102 + self.hmac = hmac.new(key, digestmod=hashlib.sha256) + + def _read(self, size): + '''Read roughly *size* bytes''' + + buf = None + while not buf: + buf = self.fh.read(size) + if not buf and not self.hmac_checked: + if self.cipher.process(self.hash) != self.hmac.digest(): + raise ChecksumError('HMAC mismatch') + elif self.decomp and self.decomp.unused_data: + raise ChecksumError('Data after end of compressed stream') + else: + self.hmac_checked = True + return '' + elif not buf: + return '' + + buf = self.cipher.process(buf) + if not self.decomp: + break + + try: + buf = self.decomp.decompress(buf) + except IOError as exc: + if exc.args == ('invalid data stream',): + raise ChecksumError('Invalid compressed stream') + raise + except lzma.error as exc: + if exc.args == ('unknown file format',): + raise ChecksumError('Invalid compressed stream') + raise + except zlib.error as exc: + if exc.args[0].startswith('Error -3 while decompressing:'): + log.warn('LegacyDecryptDecompressFilter._read(): %s', + exc.args[0]) + raise ChecksumError('Invalid compressed stream') + raise + + self.hmac.update(buf) + return buf + def close(self): + self.fh.close() -def compress_encrypt_fh(ifh, ofh, passphrase, nonce, compr): - '''Read `ifh` and write compressed, encrypted data to `ofh`''' + def __enter__(self): + return self + + def __exit__(self, *a): + self.close() + return False + + # pickle requires a readline() method for unpickling, but isn't actually + # using it if we're using the binary protocols. + def readline(self): + raise RuntimeError('not implemented') + +def encrypt(buf, passphrase, nonce): + '''Encrypt *buf*''' if isinstance(nonce, unicode): nonce = nonce.encode('utf-8') - bs = 1024 * 1024 key = sha256(passphrase + nonce) - cipher = pycryptopp.cipher.aes.AES(key) + cipher = aes.AES(key) #IGNORE:E1102 hmac_ = hmac.new(key, digestmod=hashlib.sha256) - # Write nonce - ofh.write(struct.pack(b'<B', len(nonce))) - ofh.write(nonce) - off = ofh.tell() - - # Reserve space for hmac - ofh.write(b'0' * 32) - - while True: - buf = ifh.read(bs) - if not buf: - buf = compr.flush() - buf = cipher.process(buf) - ofh.write(buf) - break - - hmac_.update(buf) - buf = compr.compress(buf) - if buf: - buf = cipher.process(buf) - ofh.write(buf) - - buf = hmac_.digest() + hmac_.update(buf) buf = cipher.process(buf) - ofh.seek(off) - ofh.write(buf) - -def compress_fh(ifh, ofh, compr): - '''Read `ifh` and write compressed data to `ofh`''' - - bs = 1024 * 1024 - while True: - buf = ifh.read(bs) - if not buf: - buf = compr.flush() - ofh.write(buf) - break - - buf = compr.compress(buf) - if buf: - ofh.write(buf) - - + hash_ = cipher.process(hmac_.digest()) + return b''.join( + (struct.pack(b'<B', len(nonce)), + nonce, hash_, buf)) + def decrypt(buf, passphrase): - '''Decrypt given string''' + '''Decrypt *buf''' fh = StringIO(buf) @@ -565,11 +838,11 @@ def decrypt(buf, passphrase): nonce = fh.read(len_) key = sha256(passphrase + nonce) - cipher = pycryptopp.cipher.aes.AES(key) + cipher = aes.AES(key) #IGNORE:E1102 hmac_ = hmac.new(key, digestmod=hashlib.sha256) # Read (encrypted) hmac - hash_ = fh.read(32) # Length of hash + hash_ = fh.read(HMAC_SIZE) buf = fh.read() buf = cipher.process(buf) @@ -628,27 +901,8 @@ class NoSuchBucket(Exception): def __str__(self): return 'Bucket %r does not exist' % self.name -def encrypt(buf, passphrase, nonce): - '''Encrypt given string''' - - if isinstance(nonce, unicode): - nonce = nonce.encode('utf-8') - - key = sha256(passphrase + nonce) - cipher = pycryptopp.cipher.aes.AES(key) - hmac_ = hmac.new(key, digestmod=hashlib.sha256) - - hmac_.update(buf) - buf = cipher.process(buf) - hash_ = cipher.process(hmac_.digest()) - - return b''.join( - (struct.pack(b'<B', len(nonce)), - nonce, hash_, buf)) - def convert_legacy_metadata(meta): - if ('encryption' in meta and 'compression' in meta): return @@ -685,6 +939,108 @@ def convert_legacy_metadata(meta): meta['compression'] = 'None' +def get_bucket(options, plain=False): + '''Return bucket for given storage-url + + If *plain* is true, don't attempt to unlock and don't wrap into + BetterBucket. + ''' + return get_bucket_factory(options, plain)() + +def get_bucket_factory(options, plain=False): + '''Return factory producing bucket objects for given storage-url + + If *plain* is true, don't attempt to unlock and don't wrap into + BetterBucket. + ''' + + hit = re.match(r'^([a-zA-Z0-9]+)://(.+)$', options.storage_url) + if not hit: + raise QuietError('Unknown storage url: %s' % options.storage_url) + backend_name = 's3ql.backends.%s' % hit.group(1) + bucket_name = hit.group(2) + try: + __import__(backend_name) + except ImportError: + raise QuietError('No such backend: %s' % hit.group(1)) + + bucket_class = getattr(sys.modules[backend_name], 'Bucket') + + # Read authfile + config = ConfigParser.SafeConfigParser() + if os.path.isfile(options.authfile): + mode = os.stat(options.authfile).st_mode + if mode & (stat.S_IRGRP | stat.S_IROTH): + raise QuietError("%s has insecure permissions, aborting." % options.authfile) + config.read(options.authfile) + + backend_login = None + backend_pw = None + bucket_passphrase = None + for section in config.sections(): + def getopt(name): + try: + return config.get(section, name) + except ConfigParser.NoOptionError: + return None + + pattern = getopt('storage-url') + + if not pattern or not options.storage_url.startswith(pattern): + continue + + backend_login = backend_login or getopt('backend-login') + backend_pw = backend_pw or getopt('backend-password') + bucket_passphrase = bucket_passphrase or getopt('bucket-passphrase') + + if not backend_login and bucket_class.needs_login: + if sys.stdin.isatty(): + backend_login = getpass("Enter backend login: ") + else: + backend_login = sys.stdin.readline().rstrip() + + if not backend_pw and bucket_class.needs_login: + if sys.stdin.isatty(): + backend_pw = getpass("Enter backend password: ") + else: + backend_pw = sys.stdin.readline().rstrip() + + if plain: + return lambda: bucket_class(bucket_name, backend_login, backend_pw) + + bucket = bucket_class(bucket_name, backend_login, backend_pw) + + try: + encrypted = 's3ql_passphrase' in bucket + except NoSuchBucket: + raise QuietError('Bucket %d does not exist' % bucket_name) + + if encrypted and not bucket_passphrase: + if sys.stdin.isatty(): + bucket_passphrase = getpass("Enter bucket encryption passphrase: ") + else: + bucket_passphrase = sys.stdin.readline().rstrip() + elif not encrypted: + bucket_passphrase = None + + if hasattr(options, 'compress'): + compress = options.compress + else: + compress = 'zlib' + + if not encrypted: + return lambda: BetterBucket(None, compress, + bucket_class(bucket_name, backend_login, backend_pw)) + + tmp_bucket = BetterBucket(bucket_passphrase, compress, bucket) + + try: + data_pw = tmp_bucket['s3ql_passphrase'] + except ChecksumError: + raise QuietError('Wrong bucket passphrase') + + return lambda: BetterBucket(data_pw, compress, + bucket_class(bucket_name, backend_login, backend_pw))
\ No newline at end of file diff --git a/src/s3ql/backends/ftp.py b/src/s3ql/backends/ftp.py deleted file mode 100644 index 39c53f3..0000000 --- a/src/s3ql/backends/ftp.py +++ /dev/null @@ -1,27 +0,0 @@ -''' -__init__.py - this file is part of S3QL (http://s3ql.googlecode.com) - -Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> - -This program can be distributed under the terms of the GNU LGPL. -''' - -from __future__ import division, print_function, absolute_import - -from .common import AbstractConnection -from ..common import QuietError -import logging - -log = logging.getLogger("backend.ftp") - -class Connection(AbstractConnection): - - def __init__(self, host, port, login, password): - super(Connection, self).__init__() - raise QuietError('FTP backend is not yet implemented.') - -class TLSConnection(Connection): - - def __init__(self, host, port, login, password): - super(Connection, self).__init__() - raise QuietError('FTP backend is not yet implemented.') diff --git a/src/s3ql/backends/ftplib.py b/src/s3ql/backends/ftplib.py deleted file mode 100644 index 70884f1..0000000 --- a/src/s3ql/backends/ftplib.py +++ /dev/null @@ -1,1038 +0,0 @@ -# Stolen from Python 2.7 to get TLS support - -#pylint: disable-all -#@PydevCodeAnalysisIgnore - - -"""An FTP client class and some helper functions. - -Based on RFC 959: File Transfer Protocol (FTP), by J. Postel and J. Reynolds - -Example: - ->>> from ftplib import FTP ->>> ftp = FTP('ftp.python.org') # connect to host, default port ->>> ftp.login() # default, i.e.: user anonymous, passwd anonymous@ -'230 Guest login ok, access restrictions apply.' ->>> ftp.retrlines('LIST') # list directory contents -total 9 -drwxr-xr-x 8 root wheel 1024 Jan 3 1994 . -drwxr-xr-x 8 root wheel 1024 Jan 3 1994 .. -drwxr-xr-x 2 root wheel 1024 Jan 3 1994 bin -drwxr-xr-x 2 root wheel 1024 Jan 3 1994 etc -d-wxrwxr-x 2 ftp wheel 1024 Sep 5 13:43 incoming -drwxr-xr-x 2 root wheel 1024 Nov 17 1993 lib -drwxr-xr-x 6 1094 wheel 1024 Sep 13 19:07 pub -drwxr-xr-x 3 root wheel 1024 Jan 3 1994 usr --rw-r--r-- 1 root root 312 Aug 1 1994 welcome.msg -'226 Transfer complete.' ->>> ftp.quit() -'221 Goodbye.' ->>> - -A nice test that reveals some of the network dialogue would be: -python ftplib.py -d localhost -l -p -l -""" - - -# -# Changes and improvements suggested by Steve Majewski. -# Modified by Jack to work on the mac. -# Modified by Siebren to support docstrings and PASV. -# Modified by Phil Schwartz to add storbinary and storlines callbacks. -# Modified by Giampaolo Rodola' to add TLS support. -# - -import os -import sys - -# Import SOCKS module if it exists, else standard socket module socket -try: - import SOCKS; socket = SOCKS; del SOCKS # import SOCKS as socket - from socket import getfqdn; socket.getfqdn = getfqdn; del getfqdn -except ImportError: - import socket -from socket import _GLOBAL_DEFAULT_TIMEOUT - -__all__ = ["FTP", "Netrc"] - -# Magic number from <socket.h> -MSG_OOB = 0x1 # Process data out of band - - -# The standard FTP server control port -FTP_PORT = 21 - - -# Exception raised when an error or invalid response is received -class Error(Exception): pass -class error_reply(Error): pass # unexpected [123]xx reply -class error_temp(Error): pass # 4xx errors -class error_perm(Error): pass # 5xx errors -class error_proto(Error): pass # response does not begin with [1-5] - - -# All exceptions (hopefully) that may be raised here and that aren't -# (always) programming errors on our side -all_errors = (Error, IOError, EOFError) - - -# Line terminators (we always output CRLF, but accept any of CRLF, CR, LF) -CRLF = '\r\n' - -# The class itself -class FTP: - - '''An FTP client class. - - To create a connection, call the class using these arguments: - host, user, passwd, acct, timeout - - The first four arguments are all strings, and have default value ''. - timeout must be numeric and defaults to None if not passed, - meaning that no timeout will be set on any ftp socket(s) - If a timeout is passed, then this is now the default timeout for all ftp - socket operations for this instance. - - Then use self.connect() with optional host and port argument. - - To download a file, use ftp.retrlines('RETR ' + filename), - or ftp.retrbinary() with slightly different arguments. - To upload a file, use ftp.storlines() or ftp.storbinary(), - which have an open file as argument (see their definitions - below for details). - The download/upload functions first issue appropriate TYPE - and PORT or PASV commands. -''' - - debugging = 0 - host = '' - port = FTP_PORT - sock = None - file = None - welcome = None - passiveserver = 1 - - # Initialization method (called by class instantiation). - # Initialize host to localhost, port to standard ftp port - # Optional arguments are host (for connect()), - # and user, passwd, acct (for login()) - def __init__(self, host='', user='', passwd='', acct='', - timeout=_GLOBAL_DEFAULT_TIMEOUT): - self.timeout = timeout - if host: - self.connect(host) - if user: - self.login(user, passwd, acct) - - def connect(self, host='', port=0, timeout= -999): - '''Connect to host. Arguments are: - - host: hostname to connect to (string, default previous host) - - port: port to connect to (integer, default previous port) - ''' - if host != '': - self.host = host - if port > 0: - self.port = port - if timeout != -999: - self.timeout = timeout - self.sock = socket.create_connection((self.host, self.port), self.timeout) - self.af = self.sock.family - self.file = self.sock.makefile('rb') - self.welcome = self.getresp() - return self.welcome - - def getwelcome(self): - '''Get the welcome message from the server. - (this is read and squirreled away by connect())''' - if self.debugging: - print '*welcome*', self.sanitize(self.welcome) - return self.welcome - - def set_debuglevel(self, level): - '''Set the debugging level. - The required argument level means: - 0: no debugging output (default) - 1: print commands and responses but not body text etc. - 2: also print raw lines read and sent before stripping CR/LF''' - self.debugging = level - debug = set_debuglevel - - def set_pasv(self, val): - '''Use passive or active mode for data transfers. - With a false argument, use the normal PORT mode, - With a true argument, use the PASV command.''' - self.passiveserver = val - - # Internal: "sanitize" a string for printing - def sanitize(self, s): - if s[:5] == 'pass ' or s[:5] == 'PASS ': - i = len(s) - while i > 5 and s[i - 1] in '\r\n': - i = i - 1 - s = s[:5] + '*' * (i - 5) + s[i:] - return repr(s) - - # Internal: send one line to the server, appending CRLF - def putline(self, line): - line = line + CRLF - if self.debugging > 1: print '*put*', self.sanitize(line) - self.sock.sendall(line) - - # Internal: send one command to the server (through putline()) - def putcmd(self, line): - if self.debugging: print '*cmd*', self.sanitize(line) - self.putline(line) - - # Internal: return one line from the server, stripping CRLF. - # Raise EOFError if the connection is closed - def getline(self): - line = self.file.readline() - if self.debugging > 1: - print '*get*', self.sanitize(line) - if not line: raise EOFError - if line[-2:] == CRLF: line = line[:-2] - elif line[-1:] in CRLF: line = line[:-1] - return line - - # Internal: get a response from the server, which may possibly - # consist of multiple lines. Return a single string with no - # trailing CRLF. If the response consists of multiple lines, - # these are separated by '\n' characters in the string - def getmultiline(self): - line = self.getline() - if line[3:4] == '-': - code = line[:3] - while 1: - nextline = self.getline() - line = line + ('\n' + nextline) - if nextline[:3] == code and \ - nextline[3:4] != '-': - break - return line - - # Internal: get a response from the server. - # Raise various errors if the response indicates an error - def getresp(self): - resp = self.getmultiline() - if self.debugging: print '*resp*', self.sanitize(resp) - self.lastresp = resp[:3] - c = resp[:1] - if c in ('1', '2', '3'): - return resp - if c == '4': - raise error_temp, resp - if c == '5': - raise error_perm, resp - raise error_proto, resp - - def voidresp(self): - """Expect a response beginning with '2'.""" - resp = self.getresp() - if resp[:1] != '2': - raise error_reply, resp - return resp - - def abort(self): - '''Abort a file transfer. Uses out-of-band data. - This does not follow the procedure from the RFC to send Telnet - IP and Synch; that doesn't seem to work with the servers I've - tried. Instead, just send the ABOR command as OOB data.''' - line = 'ABOR' + CRLF - if self.debugging > 1: print '*put urgent*', self.sanitize(line) - self.sock.sendall(line, MSG_OOB) - resp = self.getmultiline() - if resp[:3] not in ('426', '226'): - raise error_proto, resp - - def sendcmd(self, cmd): - '''Send a command and return the response.''' - self.putcmd(cmd) - return self.getresp() - - def voidcmd(self, cmd): - """Send a command and expect a response beginning with '2'.""" - self.putcmd(cmd) - return self.voidresp() - - def sendport(self, host, port): - '''Send a PORT command with the current host and the given - port number. - ''' - hbytes = host.split('.') - pbytes = [repr(port // 256), repr(port % 256)] - bytes = hbytes + pbytes - cmd = 'PORT ' + ','.join(bytes) - return self.voidcmd(cmd) - - def sendeprt(self, host, port): - '''Send a EPRT command with the current host and the given port number.''' - af = 0 - if self.af == socket.AF_INET: - af = 1 - if self.af == socket.AF_INET6: - af = 2 - if af == 0: - raise error_proto, 'unsupported address family' - fields = ['', repr(af), host, repr(port), ''] - cmd = 'EPRT ' + '|'.join(fields) - return self.voidcmd(cmd) - - def makeport(self): - '''Create a new socket and send a PORT command for it.''' - msg = "getaddrinfo returns an empty list" - sock = None - for res in socket.getaddrinfo(None, 0, self.af, socket.SOCK_STREAM, 0, socket.AI_PASSIVE): - af, socktype, proto, canonname, sa = res - try: - sock = socket.socket(af, socktype, proto) - sock.bind(sa) - except socket.error, msg: - if sock: - sock.close() - sock = None - continue - break - if not sock: - raise socket.error, msg - sock.listen(1) - port = sock.getsockname()[1] # Get proper port - host = self.sock.getsockname()[0] # Get proper host - if self.af == socket.AF_INET: - resp = self.sendport(host, port) - else: - resp = self.sendeprt(host, port) - return sock - - def makepasv(self): - if self.af == socket.AF_INET: - host, port = parse227(self.sendcmd('PASV')) - else: - host, port = parse229(self.sendcmd('EPSV'), self.sock.getpeername()) - return host, port - - def ntransfercmd(self, cmd, rest=None): - """Initiate a transfer over the data connection. - - If the transfer is active, send a port command and the - transfer command, and accept the connection. If the server is - passive, send a pasv command, connect to it, and start the - transfer command. Either way, return the socket for the - connection and the expected size of the transfer. The - expected size may be None if it could not be determined. - - Optional `rest' argument can be a string that is sent as the - argument to a REST command. This is essentially a server - marker used to tell the server to skip over any data up to the - given marker. - """ - size = None - if self.passiveserver: - host, port = self.makepasv() - conn = socket.create_connection((host, port), self.timeout) - if rest is not None: - self.sendcmd("REST %s" % rest) - resp = self.sendcmd(cmd) - # Some servers apparently send a 200 reply to - # a LIST or STOR command, before the 150 reply - # (and way before the 226 reply). This seems to - # be in violation of the protocol (which only allows - # 1xx or error messages for LIST), so we just discard - # this response. - if resp[0] == '2': - resp = self.getresp() - if resp[0] != '1': - raise error_reply, resp - else: - sock = self.makeport() - if rest is not None: - self.sendcmd("REST %s" % rest) - resp = self.sendcmd(cmd) - # See above. - if resp[0] == '2': - resp = self.getresp() - if resp[0] != '1': - raise error_reply, resp - conn, sockaddr = sock.accept() - if resp[:3] == '150': - # this is conditional in case we received a 125 - size = parse150(resp) - return conn, size - - def transfercmd(self, cmd, rest=None): - """Like ntransfercmd() but returns only the socket.""" - return self.ntransfercmd(cmd, rest)[0] - - def login(self, user='', passwd='', acct=''): - '''Login, default anonymous.''' - if not user: user = 'anonymous' - if not passwd: passwd = '' - if not acct: acct = '' - if user == 'anonymous' and passwd in ('', '-'): - # If there is no anonymous ftp password specified - # then we'll just use anonymous@ - # We don't send any other thing because: - # - We want to remain anonymous - # - We want to stop SPAM - # - We don't want to let ftp sites to discriminate by the user, - # host or country. - passwd = passwd + 'anonymous@' - resp = self.sendcmd('USER ' + user) - if resp[0] == '3': resp = self.sendcmd('PASS ' + passwd) - if resp[0] == '3': resp = self.sendcmd('ACCT ' + acct) - if resp[0] != '2': - raise error_reply, resp - return resp - - def retrbinary(self, cmd, callback, blocksize=8192, rest=None): - """Retrieve data in binary mode. A new port is created for you. - - Args: - cmd: A RETR command. - callback: A single parameter callable to be called on each - block of data read. - blocksize: The maximum number of bytes to read from the - socket at one time. [default: 8192] - rest: Passed to transfercmd(). [default: None] - - Returns: - The response code. - """ - self.voidcmd('TYPE I') - conn = self.transfercmd(cmd, rest) - while 1: - data = conn.recv(blocksize) - if not data: - break - callback(data) - conn.close() - return self.voidresp() - - def retrlines(self, cmd, callback=None): - """Retrieve data in line mode. A new port is created for you. - - Args: - cmd: A RETR, LIST, NLST, or MLSD command. - callback: An optional single parameter callable that is called - for each line with the trailing CRLF stripped. - [default: print_line()] - - Returns: - The response code. - """ - if callback is None: callback = print_line - resp = self.sendcmd('TYPE A') - conn = self.transfercmd(cmd) - fp = conn.makefile('rb') - while 1: - line = fp.readline() - if self.debugging > 2: print '*retr*', repr(line) - if not line: - break - if line[-2:] == CRLF: - line = line[:-2] - elif line[-1:] == '\n': - line = line[:-1] - callback(line) - fp.close() - conn.close() - return self.voidresp() - - def storbinary(self, cmd, fp, blocksize=8192, callback=None): - """Store a file in binary mode. A new port is created for you. - - Args: - cmd: A STOR command. - fp: A file-like object with a read(num_bytes) method. - blocksize: The maximum data size to read from fp and send over - the connection at once. [default: 8192] - callback: An optional single parameter callable that is called on - on each block of data after it is sent. [default: None] - - Returns: - The response code. - """ - self.voidcmd('TYPE I') - conn = self.transfercmd(cmd) - while 1: - buf = fp.read(blocksize) - if not buf: break - conn.sendall(buf) - if callback: callback(buf) - conn.close() - return self.voidresp() - - def storlines(self, cmd, fp, callback=None): - """Store a file in line mode. A new port is created for you. - - Args: - cmd: A STOR command. - fp: A file-like object with a readline() method. - callback: An optional single parameter callable that is called on - on each line after it is sent. [default: None] - - Returns: - The response code. - """ - self.voidcmd('TYPE A') - conn = self.transfercmd(cmd) - while 1: - buf = fp.readline() - if not buf: break - if buf[-2:] != CRLF: - if buf[-1] in CRLF: buf = buf[:-1] - buf = buf + CRLF - conn.sendall(buf) - if callback: callback(buf) - conn.close() - return self.voidresp() - - def acct(self, password): - '''Send new account name.''' - cmd = 'ACCT ' + password - return self.voidcmd(cmd) - - def nlst(self, *args): - '''Return a list of files in a given directory (default the current).''' - cmd = 'NLST' - for arg in args: - cmd = cmd + (' ' + arg) - files = [] - self.retrlines(cmd, files.append) - return files - - def dir(self, *args): - '''List a directory in long form. - By default list current directory to stdout. - Optional last argument is callback function; all - non-empty arguments before it are concatenated to the - LIST command. (This *should* only be used for a pathname.)''' - cmd = 'LIST' - func = None - if args[-1:] and type(args[-1]) != type(''): - args, func = args[:-1], args[-1] - for arg in args: - if arg: - cmd = cmd + (' ' + arg) - self.retrlines(cmd, func) - - def rename(self, fromname, toname): - '''Rename a file.''' - resp = self.sendcmd('RNFR ' + fromname) - if resp[0] != '3': - raise error_reply, resp - return self.voidcmd('RNTO ' + toname) - - def delete(self, filename): - '''Delete a file.''' - resp = self.sendcmd('DELE ' + filename) - if resp[:3] in ('250', '200'): - return resp - else: - raise error_reply, resp - - def cwd(self, dirname): - '''Change to a directory.''' - if dirname == '..': - try: - return self.voidcmd('CDUP') - except error_perm, msg: - if msg.args[0][:3] != '500': - raise - elif dirname == '': - dirname = '.' # does nothing, but could return error - cmd = 'CWD ' + dirname - return self.voidcmd(cmd) - - def size(self, filename): - '''Retrieve the size of a file.''' - # The SIZE command is defined in RFC-3659 - resp = self.sendcmd('SIZE ' + filename) - if resp[:3] == '213': - s = resp[3:].strip() - try: - return int(s) - except (OverflowError, ValueError): - return long(s) - - def mkd(self, dirname): - '''Make a directory, return its full pathname.''' - resp = self.sendcmd('MKD ' + dirname) - return parse257(resp) - - def rmd(self, dirname): - '''Remove a directory.''' - return self.voidcmd('RMD ' + dirname) - - def pwd(self): - '''Return current working directory.''' - resp = self.sendcmd('PWD') - return parse257(resp) - - def quit(self): - '''Quit, and close the connection.''' - resp = self.voidcmd('QUIT') - self.close() - return resp - - def close(self): - '''Close the connection without assuming anything about it.''' - if self.file: - self.file.close() - self.sock.close() - self.file = self.sock = None - - -try: - import ssl -except ImportError: - pass -else: - class FTP_TLS(FTP): - '''A FTP subclass which adds TLS support to FTP as described - in RFC-4217. - - Connect as usual to port 21 implicitly securing the FTP control - connection before authenticating. - - Securing the data connection requires user to explicitly ask - for it by calling prot_p() method. - - Usage example: - >>> from ftplib import FTP_TLS - >>> ftps = FTP_TLS('ftp.python.org') - >>> ftps.login() # login anonimously previously securing control channel - '230 Guest login ok, access restrictions apply.' - >>> ftps.prot_p() # switch to secure data connection - '200 Protection level set to P' - >>> ftps.retrlines('LIST') # list directory content securely - total 9 - drwxr-xr-x 8 root wheel 1024 Jan 3 1994 . - drwxr-xr-x 8 root wheel 1024 Jan 3 1994 .. - drwxr-xr-x 2 root wheel 1024 Jan 3 1994 bin - drwxr-xr-x 2 root wheel 1024 Jan 3 1994 etc - d-wxrwxr-x 2 ftp wheel 1024 Sep 5 13:43 incoming - drwxr-xr-x 2 root wheel 1024 Nov 17 1993 lib - drwxr-xr-x 6 1094 wheel 1024 Sep 13 19:07 pub - drwxr-xr-x 3 root wheel 1024 Jan 3 1994 usr - -rw-r--r-- 1 root root 312 Aug 1 1994 welcome.msg - '226 Transfer complete.' - >>> ftps.quit() - '221 Goodbye.' - >>> - ''' - ssl_version = ssl.PROTOCOL_TLSv1 - - def __init__(self, host='', user='', passwd='', acct='', keyfile=None, - certfile=None, timeout=_GLOBAL_DEFAULT_TIMEOUT): - self.keyfile = keyfile - self.certfile = certfile - self._prot_p = False - FTP.__init__(self, host, user, passwd, acct, timeout) - - def login(self, user='', passwd='', acct='', secure=True): - if secure and not isinstance(self.sock, ssl.SSLSocket): - self.auth() - return FTP.login(self, user, passwd, acct) - - def auth(self): - '''Set up secure control connection by using TLS/SSL.''' - if isinstance(self.sock, ssl.SSLSocket): - raise ValueError("Already using TLS") - if self.ssl_version == ssl.PROTOCOL_TLSv1: - resp = self.voidcmd('AUTH TLS') - else: - resp = self.voidcmd('AUTH SSL') - self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile, - ssl_version=self.ssl_version) - self.file = self.sock.makefile(mode='rb') - return resp - - def prot_p(self): - '''Set up secure data connection.''' - # PROT defines whether or not the data channel is to be protected. - # Though RFC-2228 defines four possible protection levels, - # RFC-4217 only recommends two, Clear and Private. - # Clear (PROT C) means that no security is to be used on the - # data-channel, Private (PROT P) means that the data-channel - # should be protected by TLS. - # PBSZ command MUST still be issued, but must have a parameter of - # '0' to indicate that no buffering is taking place and the data - # connection should not be encapsulated. - self.voidcmd('PBSZ 0') - resp = self.voidcmd('PROT P') - self._prot_p = True - return resp - - def prot_c(self): - '''Set up clear text data connection.''' - resp = self.voidcmd('PROT C') - self._prot_p = False - return resp - - # --- Overridden FTP methods - - def ntransfercmd(self, cmd, rest=None): - conn, size = FTP.ntransfercmd(self, cmd, rest) - if self._prot_p: - conn = ssl.wrap_socket(conn, self.keyfile, self.certfile, - ssl_version=self.ssl_version) - return conn, size - - def retrbinary(self, cmd, callback, blocksize=8192, rest=None): - self.voidcmd('TYPE I') - conn = self.transfercmd(cmd, rest) - try: - while 1: - data = conn.recv(blocksize) - if not data: - break - callback(data) - # shutdown ssl layer - if isinstance(conn, ssl.SSLSocket): - conn.unwrap() - finally: - conn.close() - return self.voidresp() - - def retrlines(self, cmd, callback=None): - if callback is None: callback = print_line - resp = self.sendcmd('TYPE A') - conn = self.transfercmd(cmd) - fp = conn.makefile('rb') - try: - while 1: - line = fp.readline() - if self.debugging > 2: print '*retr*', repr(line) - if not line: - break - if line[-2:] == CRLF: - line = line[:-2] - elif line[-1:] == '\n': - line = line[:-1] - callback(line) - # shutdown ssl layer - if isinstance(conn, ssl.SSLSocket): - conn.unwrap() - finally: - fp.close() - conn.close() - return self.voidresp() - - def storbinary(self, cmd, fp, blocksize=8192, callback=None): - self.voidcmd('TYPE I') - conn = self.transfercmd(cmd) - try: - while 1: - buf = fp.read(blocksize) - if not buf: break - conn.sendall(buf) - if callback: callback(buf) - # shutdown ssl layer - if isinstance(conn, ssl.SSLSocket): - conn.unwrap() - finally: - conn.close() - return self.voidresp() - - def storlines(self, cmd, fp, callback=None): - self.voidcmd('TYPE A') - conn = self.transfercmd(cmd) - try: - while 1: - buf = fp.readline() - if not buf: break - if buf[-2:] != CRLF: - if buf[-1] in CRLF: buf = buf[:-1] - buf = buf + CRLF - conn.sendall(buf) - if callback: callback(buf) - # shutdown ssl layer - if isinstance(conn, ssl.SSLSocket): - conn.unwrap() - finally: - conn.close() - return self.voidresp() - - __all__.append(FTP_TLS) - all_errors = (Error, IOError, EOFError, ssl.SSLError) - - -_150_re = None - -def parse150(resp): - '''Parse the '150' response for a RETR request. - Returns the expected transfer size or None; size is not guaranteed to - be present in the 150 message. - ''' - if resp[:3] != '150': - raise error_reply, resp - global _150_re - if _150_re is None: - import re - _150_re = re.compile("150 .* \((\d+) bytes\)", re.IGNORECASE) - m = _150_re.match(resp) - if not m: - return None - s = m.group(1) - try: - return int(s) - except (OverflowError, ValueError): - return long(s) - - -_227_re = None - -def parse227(resp): - '''Parse the '227' response for a PASV request. - Raises error_proto if it does not contain '(h1,h2,h3,h4,p1,p2)' - Return ('host.addr.as.numbers', port#) tuple.''' - - if resp[:3] != '227': - raise error_reply, resp - global _227_re - if _227_re is None: - import re - _227_re = re.compile(r'(\d+),(\d+),(\d+),(\d+),(\d+),(\d+)') - m = _227_re.search(resp) - if not m: - raise error_proto, resp - numbers = m.groups() - host = '.'.join(numbers[:4]) - port = (int(numbers[4]) << 8) + int(numbers[5]) - return host, port - - -def parse229(resp, peer): - '''Parse the '229' response for a EPSV request. - Raises error_proto if it does not contain '(|||port|)' - Return ('host.addr.as.numbers', port#) tuple.''' - - if resp[:3] != '229': - raise error_reply, resp - left = resp.find('(') - if left < 0: raise error_proto, resp - right = resp.find(')', left + 1) - if right < 0: - raise error_proto, resp # should contain '(|||port|)' - if resp[left + 1] != resp[right - 1]: - raise error_proto, resp - parts = resp[left + 1:right].split(resp[left + 1]) - if len(parts) != 5: - raise error_proto, resp - host = peer[0] - port = int(parts[3]) - return host, port - - -def parse257(resp): - '''Parse the '257' response for a MKD or PWD request. - This is a response to a MKD or PWD request: a directory name. - Returns the directoryname in the 257 reply.''' - - if resp[:3] != '257': - raise error_reply, resp - if resp[3:5] != ' "': - return '' # Not compliant to RFC 959, but UNIX ftpd does this - dirname = '' - i = 5 - n = len(resp) - while i < n: - c = resp[i] - i = i + 1 - if c == '"': - if i >= n or resp[i] != '"': - break - i = i + 1 - dirname = dirname + c - return dirname - - -def print_line(line): - '''Default retrlines callback to print a line.''' - print line - - -def ftpcp(source, sourcename, target, targetname='', type='I'): - '''Copy file from one FTP-instance to another.''' - if not targetname: targetname = sourcename - type = 'TYPE ' + type - source.voidcmd(type) - target.voidcmd(type) - sourcehost, sourceport = parse227(source.sendcmd('PASV')) - target.sendport(sourcehost, sourceport) - # RFC 959: the user must "listen" [...] BEFORE sending the - # transfer request. - # So: STOR before RETR, because here the target is a "user". - treply = target.sendcmd('STOR ' + targetname) - if treply[:3] not in ('125', '150'): raise error_proto # RFC 959 - sreply = source.sendcmd('RETR ' + sourcename) - if sreply[:3] not in ('125', '150'): raise error_proto # RFC 959 - source.voidresp() - target.voidresp() - - -class Netrc: - """Class to parse & provide access to 'netrc' format files. - - See the netrc(4) man page for information on the file format. - - WARNING: This class is obsolete -- use module netrc instead. - - """ - __defuser = None - __defpasswd = None - __defacct = None - - def __init__(self, filename=None): - if filename is None: - if "HOME" in os.environ: - filename = os.path.join(os.environ["HOME"], - ".netrc") - else: - raise IOError, \ - "specify file to load or set $HOME" - self.__hosts = {} - self.__macros = {} - fp = open(filename, "r") - in_macro = 0 - while 1: - line = fp.readline() - if not line: break - if in_macro and line.strip(): - macro_lines.append(line) - continue - elif in_macro: - self.__macros[macro_name] = tuple(macro_lines) - in_macro = 0 - words = line.split() - host = user = passwd = acct = None - default = 0 - i = 0 - while i < len(words): - w1 = words[i] - if i + 1 < len(words): - w2 = words[i + 1] - else: - w2 = None - if w1 == 'default': - default = 1 - elif w1 == 'machine' and w2: - host = w2.lower() - i = i + 1 - elif w1 == 'login' and w2: - user = w2 - i = i + 1 - elif w1 == 'password' and w2: - passwd = w2 - i = i + 1 - elif w1 == 'account' and w2: - acct = w2 - i = i + 1 - elif w1 == 'macdef' and w2: - macro_name = w2 - macro_lines = [] - in_macro = 1 - break - i = i + 1 - if default: - self.__defuser = user or self.__defuser - self.__defpasswd = passwd or self.__defpasswd - self.__defacct = acct or self.__defacct - if host: - if host in self.__hosts: - ouser, opasswd, oacct = \ - self.__hosts[host] - user = user or ouser - passwd = passwd or opasswd - acct = acct or oacct - self.__hosts[host] = user, passwd, acct - fp.close() - - def get_hosts(self): - """Return a list of hosts mentioned in the .netrc file.""" - return self.__hosts.keys() - - def get_account(self, host): - """Returns login information for the named host. - - The return value is a triple containing userid, - password, and the accounting field. - - """ - host = host.lower() - user = passwd = acct = None - if host in self.__hosts: - user, passwd, acct = self.__hosts[host] - user = user or self.__defuser - passwd = passwd or self.__defpasswd - acct = acct or self.__defacct - return user, passwd, acct - - def get_macros(self): - """Return a list of all defined macro names.""" - return self.__macros.keys() - - def get_macro(self, macro): - """Return a sequence of lines which define a named macro.""" - return self.__macros[macro] - - - -def test(): - '''Test program. - Usage: ftp [-d] [-r[file]] host [-l[dir]] [-d[dir]] [-p] [file] ... - - -d dir - -l list - -p password - ''' - - if len(sys.argv) < 2: - print test.__doc__ - sys.exit(0) - - debugging = 0 - rcfile = None - while sys.argv[1] == '-d': - debugging = debugging + 1 - del sys.argv[1] - if sys.argv[1][:2] == '-r': - # get name of alternate ~/.netrc file: - rcfile = sys.argv[1][2:] - del sys.argv[1] - host = sys.argv[1] - ftp = FTP(host) - ftp.set_debuglevel(debugging) - userid = passwd = acct = '' - try: - netrc = Netrc(rcfile) - except IOError: - if rcfile is not None: - sys.stderr.write("Could not open account file" - " -- using anonymous login.") - else: - try: - userid, passwd, acct = netrc.get_account(host) - except KeyError: - # no account for host - sys.stderr.write( - "No account -- using anonymous login.") - ftp.login(userid, passwd, acct) - for file in sys.argv[2:]: - if file[:2] == '-l': - ftp.dir(file[2:]) - elif file[:2] == '-d': - cmd = 'CWD' - if file[2:]: cmd = cmd + ' ' + file[2:] - resp = ftp.sendcmd(cmd) - elif file == '-p': - ftp.set_pasv(not ftp.passiveserver) - else: - ftp.retrbinary('RETR ' + file, \ - sys.stdout.write, 1024) - ftp.quit() - - -if __name__ == '__main__': - test() diff --git a/src/s3ql/backends/gs.py b/src/s3ql/backends/gs.py new file mode 100644 index 0000000..d71e82e --- /dev/null +++ b/src/s3ql/backends/gs.py @@ -0,0 +1,79 @@ +''' +backends/gs.py - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright (C) Nikolaus Rath <Nikolaus@rath.org> + +This program can be distributed under the terms of the GNU GPLv3. +''' + +from __future__ import division, print_function, absolute_import +from . import s3 +from .s3 import retry +import httplib +import logging +import xml.etree.cElementTree as ElementTree + +# Pylint goes berserk with false positives +#pylint: disable=E1002,E1101,W0201 + +log = logging.getLogger("backends.gs") + +class Bucket(s3.Bucket): + """A bucket stored in Google Storage + + This class uses standard HTTP connections to connect to GS. + + The bucket guarantees immediate get consistency and eventual list + consistency. + """ + + def __init__(self, bucket_name, gs_key, gs_secret): + super(Bucket, self).__init__(bucket_name, gs_key, gs_secret) + + self.namespace = 'http://doc.s3.amazonaws.com/2006-03-01' + + def _init(self): + '''Additional initialization code + + Called by constructor and provided solely for easier subclassing. + ''' + + self.conn = self._get_conn() + + def _get_conn(self): + '''Return connection to server''' + + return httplib.HTTPConnection('%s.commondatastorage.googleapis.com' % self.bucket_name) + + @retry + def _get_region(self): + ''''Return bucket region''' + + log.debug('_get_region()') + resp = self._do_request('GET', '/', subres='location') + region = ElementTree.parse(resp).getroot().text + + return region + + def is_get_consistent(self): + '''If True, objects retrievals are guaranteed to be up-to-date + + If this method returns True, then creating, deleting, or overwriting an + object is guaranteed to be immediately reflected in subsequent object + retrieval attempts. + ''' + + return True + + def is_list_create_consistent(self): + '''If True, new objects are guaranteed to show up in object listings + + If this method returns True, creation of objects will immediately be + reflected when retrieving the list of available objects. + ''' + + return False + + def __str__(self): + return 'gs://%s/%s' % (self.bucket_name, self.prefix) + diff --git a/src/s3ql/backends/gss.py b/src/s3ql/backends/gss.py new file mode 100644 index 0000000..7b35cc8 --- /dev/null +++ b/src/s3ql/backends/gss.py @@ -0,0 +1,33 @@ +''' +backends/gss.py - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright (C) Nikolaus Rath <Nikolaus@rath.org> + +This program can be distributed under the terms of the GNU GPLv3. +''' + +from __future__ import division, print_function, absolute_import + +from . import gs +import httplib + +# Pylint goes berserk with false positives +#pylint: disable=E1002,E1101,W0232 + + +class Bucket(gs.Bucket): + """A bucket stored in Google Storage + + This class uses secure (SSL) connections to connect to GS. + + The bucket guarantees immediate get consistency and eventual list + consistency. + """ + + def _get_conn(self): + '''Return connection to server''' + + return httplib.HTTPSConnection('%s.commondatastorage.googleapis.com' % self.bucket_name) + + def __str__(self): + return 'gss://%s/%s' % (self.bucket_name, self.prefix)
\ No newline at end of file diff --git a/src/s3ql/backends/local.py b/src/s3ql/backends/local.py index 136c7d5..4184a26 100644 --- a/src/s3ql/backends/local.py +++ b/src/s3ql/backends/local.py @@ -3,258 +3,276 @@ local.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import -from .common import AbstractConnection, AbstractBucket, NoSuchBucket, NoSuchObject +from .common import AbstractBucket, NoSuchBucket, NoSuchObject import shutil import logging import cPickle as pickle import os import errno -import threading +import thread +from s3ql.backends.common import ChecksumError log = logging.getLogger("backend.local") -class Connection(AbstractConnection): - """ - A connection that stores buckets on the local disk - - This class is threadsafe. All methods (except for internal methods - starting with underscore) may be called concurrently by different - threads. - """ - - def __init__(self): - super(Connection, self).__init__() - self.lock = threading.RLock() - - def delete_bucket(self, name, recursive=False): - """Delete bucket""" +class Bucket(AbstractBucket): + ''' + A bucket that is stored on the local hard disk + ''' - with self.lock: - if not os.path.exists(name): - raise NoSuchBucket(name) + needs_login = False - if recursive: - shutil.rmtree(name) - else: - os.rmdir(name) + def __init__(self, name, backend_login, backend_pw): #IGNORE:W0613 + '''Initialize local bucket + + Login and password are ignored. + ''' + super(Bucket, self).__init__() + self.name = name + + if not os.path.exists(name): + raise NoSuchBucket(name) - def create_bucket(self, name, passphrase=None, compression='bzip2'): - """Create and return a bucket""" + def __str__(self): + return 'local://%s' % self.name - with self.lock: - if os.path.exists(name): - raise RuntimeError('Bucket already exists') - os.mkdir(name) - - return self.get_bucket(name, passphrase, compression) + def lookup(self, key): + """Return metadata for given key. - def get_bucket(self, name, passphrase=None, compression='bzip2'): - """Return a bucket instance for the bucket `name` - - Raises `NoSuchBucket` if the bucket does not exist. + If the key does not exist, `NoSuchObject` is raised. """ - with self.lock: - if not os.path.exists(name): - raise NoSuchBucket(name) - return Bucket(name, self.lock, passphrase, compression) - + path = self._key_to_path(key) + try: + with open(path, 'rb') as src: + return pickle.load(src) + except IOError as exc: + if exc.errno == errno.ENOENT: + raise NoSuchObject(key) + else: + raise + except pickle.UnpicklingError as exc: + if (isinstance(exc.args[0], str) + and exc.args[0].startswith('invalid load key')): + raise ChecksumError('Invalid metadata') + raise + + def open_read(self, key): + """Open object for reading -class Bucket(AbstractBucket): - ''' - A bucket that is stored on the local hard disk + Return a tuple of a file-like object. Bucket contents can be read from + the file-like object, metadata is stored in its *metadata* attribute and + can be modified by the caller at will. + """ + + path = self._key_to_path(key) + try: + fh = ObjectR(path) + except IOError as exc: + if exc.errno == errno.ENOENT: + raise NoSuchObject(key) + else: + raise + + try: + fh.metadata = pickle.load(fh) + except pickle.UnpicklingError as exc: + if (isinstance(exc.args[0], str) + and exc.args[0].startswith('invalid load key')): + raise ChecksumError('Invalid metadata') + raise + return fh - This class is threadsafe. All methods (except for internal methods - starting with underscore) may be called concurrently by different - threads. - ''' + def open_write(self, key, metadata=None): + """Open object for writing - def __init__(self, name, lock, passphrase, compression): - super(Bucket, self).__init__(passphrase, compression) - self.name = name - self.lock = lock - - def __str__(self): - return '<local bucket, name=%r>' % self.name - - def read_after_create_consistent(self): - return True + `metadata` can be a dict of additional attributes to store with the + object. Returns a file-like object. + """ + + if metadata is None: + metadata = dict() + + + path = self._key_to_path(key) - def read_after_write_consistent(self): + # By renaming, we make sure that there are no + # conflicts between parallel reads, the last one wins + tmpname = '%s#%d-%d' % (path, os.getpid(), thread.get_ident()) + + try: + dest = open(tmpname, 'wb') + except IOError as exc: + if exc.errno != errno.ENOENT: + raise + try: + os.makedirs(os.path.dirname(path)) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + else: + # Another thread may have created the directory already + pass + dest = open(tmpname, 'wb', 0) + + os.rename(tmpname, path) + pickle.dump(metadata, dest, 2) + return dest + + def is_get_consistent(self): + '''If True, objects retrievals are guaranteed to be up-to-date + + If this method returns True, then creating, deleting, or overwriting an + object is guaranteed to be immediately reflected in subsequent object + retrieval attempts. + ''' + return True + + def is_list_create_consistent(self): + '''If True, new objects are guaranteed to show up in object listings + + If this method returns True, creation of objects will immediately be + reflected when retrieving the list of available objects. + ''' - def read_after_delete_consistent(self): return True - + def clear(self): """Delete all objects in bucket""" - with self.lock: - for name in os.listdir(self.name): - path = os.path.join(self.name, name) - if os.path.isdir(path): - shutil.rmtree(path) - else: - os.unlink(path) + + for name in os.listdir(self.name): + path = os.path.join(self.name, name) + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.unlink(path) def contains(self, key): - with self.lock: - path = self._key_to_path(key) + '.dat' - try: - os.lstat(path) - except OSError as exc: - if exc.errno == errno.ENOENT: - return False - raise - return True - - - def raw_lookup(self, key): - with self.lock: - path = self._key_to_path(key) - try: - with open(path + '.meta', 'rb') as src: - return pickle.load(src) - except IOError as exc: - if exc.errno == errno.ENOENT: - raise NoSuchObject(key) - else: - raise + '''Check if `key` is in bucket''' + + path = self._key_to_path(key) + try: + os.lstat(path) + except OSError as exc: + if exc.errno == errno.ENOENT: + return False + raise + return True def delete(self, key, force=False): - with self.lock: - path = self._key_to_path(key) - try: - os.unlink(path + '.dat') - os.unlink(path + '.meta') - except OSError as exc: - if exc.errno == errno.ENOENT: - if force: - pass - else: - raise NoSuchObject(key) + """Delete object stored under `key` + + ``bucket.delete(key)`` can also be written as ``del bucket[key]``. + If `force` is true, do not return an error if the key does not exist. + """ + path = self._key_to_path(key) + try: + os.unlink(path) + except OSError as exc: + if exc.errno == errno.ENOENT: + if force: + pass else: - raise + raise NoSuchObject(key) + else: + raise + def list(self, prefix=''): + '''List keys in bucket - def list(self, prefix=None): - with self.lock: + Returns an iterator over all keys in the bucket. + ''' + if prefix: + base = os.path.dirname(self._key_to_path(prefix)) + else: + base = self.name + + for (path, dirnames, filenames) in os.walk(base, topdown=True): + + # Do not look in wrong directories if prefix: - base = os.path.dirname(self._key_to_path(prefix)) - else: - base = self.name + rpath = path[len(self.name):] # path relative to base + prefix_l = ''.join(rpath.split('/')) - for (path, dirnames, filenames) in os.walk(base, topdown=True): + dirs_to_walk = list() + for name in dirnames: + prefix_ll = unescape(prefix_l + name) + if prefix_ll.startswith(prefix[:len(prefix_ll)]): + dirs_to_walk.append(name) + dirnames[:] = dirs_to_walk + + for name in filenames: + key = unescape(name) - # Do not look in wrong directories - if prefix: - rpath = path[len(self.name):] # path relative to base - prefix_l = ''.join(rpath.split('/')) - - dirs_to_walk = list() - for name in dirnames: - prefix_ll = unescape(prefix_l + name) - if prefix_ll.startswith(prefix[:len(prefix_ll)]): - dirs_to_walk.append(name) - dirnames[:] = dirs_to_walk - - for name in filenames: - if not name.endswith('.dat'): - continue - key = unescape(name[:-4]) - - if not prefix or key.startswith(prefix): - yield key - - def raw_fetch(self, key, fh): - with self.lock: - path = self._key_to_path(key) - try: - with open(path + '.dat', 'rb') as src: - fh.seek(0) - shutil.copyfileobj(src, fh) - with open(path + '.meta', 'rb') as src: - metadata = pickle.load(src) - except IOError as exc: - if exc.errno == errno.ENOENT: - raise NoSuchObject(key) - else: - raise - - return metadata - - def raw_store(self, key, fh, metadata): - with self.lock: - path = self._key_to_path(key) - fh.seek(0) - try: - dest = open(path + '.dat', 'wb') - except IOError as exc: - if exc.errno != errno.ENOENT: - raise - os.makedirs(os.path.dirname(path)) - dest = open(path + '.dat', 'wb') - - shutil.copyfileobj(fh, dest) - dest.close() - - with open(path + '.meta', 'wb') as dest: - pickle.dump(metadata, dest, 2) + if not prefix or key.startswith(prefix): + yield key def copy(self, src, dest): - with self.lock: - if not isinstance(src, str): - raise TypeError('key must be of type str') - - if not isinstance(dest, str): - raise TypeError('key must be of type str') - - path_src = self._key_to_path(src) - path_dest = self._key_to_path(dest) - - # Can't use shutil.copyfile() here, need to make - # sure destination path exists + """Copy data stored under key `src` to key `dest` + + If `dest` already exists, it will be overwritten. + """ + + path_src = self._key_to_path(src) + path_dest = self._key_to_path(dest) + + # Can't use shutil.copyfile() here, need to make + # sure destination path exists + try: + dest = open(path_dest, 'wb') + except IOError as exc: + if exc.errno != errno.ENOENT: + raise try: - dest = open(path_dest + '.dat', 'wb') - except IOError as exc: - if exc.errno != errno.ENOENT: - raise os.makedirs(os.path.dirname(path_dest)) - dest = open(path_dest + '.dat', 'wb') - - try: - with open(path_src + '.dat', 'rb') as src: - shutil.copyfileobj(src, dest) - except IOError as exc: - if exc.errno == errno.ENOENT: - raise NoSuchObject(src) - else: + except OSError as exc: + if exc.errno != errno.EEXIST: raise - finally: - dest.close() - - shutil.copyfile(path_src + '.meta', path_dest + '.meta') + else: + # Another thread may have created the directory already + pass + dest = open(path_dest, 'wb') + + try: + with open(path_src, 'rb') as src: + shutil.copyfileobj(src, dest) + except IOError as exc: + if exc.errno == errno.ENOENT: + raise NoSuchObject(src) + else: + raise + finally: + dest.close() def rename(self, src, dest): - with self.lock: - src_path = self._key_to_path(src) - dest_path = self._key_to_path(dest) - if not os.path.exists(src_path + '.dat'): - raise NoSuchObject(src) - - try: - os.rename(src_path + '.dat', dest_path + '.dat') - os.rename(src_path + '.meta', dest_path + '.meta') + """Rename key `src` to `dest` + + If `dest` already exists, it will be overwritten. + """ + src_path = self._key_to_path(src) + dest_path = self._key_to_path(dest) + if not os.path.exists(src_path): + raise NoSuchObject(src) + + try: + os.rename(src_path, dest_path) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + try: + os.makedirs(os.path.dirname(dest_path)) except OSError as exc: - if exc.errno != errno.ENOENT: + if exc.errno != errno.EEXIST: raise - os.makedirs(os.path.dirname(dest_path)) - os.rename(src_path + '.dat', dest_path + '.dat') - os.rename(src_path + '.meta', dest_path + '.meta') + else: + # Another thread may have created the directory already + pass + os.rename(src_path, dest_path) def _key_to_path(self, key): '''Return path for given key''' @@ -276,21 +294,26 @@ class Bucket(AbstractBucket): return os.path.join(*path) def escape(s): - '''Escape '/', '=' and '\0' in s''' + '''Escape '/', '=' and '.' in s''' s = s.replace('=', '=3D') s = s.replace('/', '=2F') - s = s.replace('\0', '=00') + s = s.replace('#', '=23') return s def unescape(s): - '''Un-Escape '/', '=' and '\0' in s''' + '''Un-Escape '/', '=' and '.' in s''' s = s.replace('=2F', '/') - s = s.replace('=00', '\0') + s = s.replace('=23', '#') s = s.replace('=3D', '=') return s - +class ObjectR(file): + '''A local storage object opened for reading''' + + def __init__(self, name, metadata=None): + super(ObjectR, self).__init__(name, 'rb', buffering=0) + self.metadata = metadata
\ No newline at end of file diff --git a/src/s3ql/backends/s3.py b/src/s3ql/backends/s3.py index a536c1c..8ea740e 100644 --- a/src/s3ql/backends/s3.py +++ b/src/s3ql/backends/s3.py @@ -1,382 +1,713 @@ ''' s3.py - this file is part of S3QL (http://s3ql.googlecode.com) -Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> +Copyright (C) Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import -# Python boto uses several deprecated modules, deactivate warnings for them -import warnings -warnings.filterwarnings("ignore", "", DeprecationWarning, "boto") - -from .common import AbstractConnection, AbstractBucket, NoSuchBucket, NoSuchObject -from time import sleep -from .boto.s3.connection import S3Connection -from contextlib import contextmanager -from .boto import exception -from s3ql.common import (TimeoutError, AsyncFn) +from .common import AbstractBucket, NoSuchObject import logging -import errno import httplib import re import time -import threading +from base64 import b64encode +import hmac +import hashlib +import tempfile +import urllib +import xml.etree.cElementTree as ElementTree +import os +import errno +from functools import wraps log = logging.getLogger("backend.s3") -class Connection(AbstractConnection): - """Represents a connection to Amazon S3 +C_DAY_NAMES = [ 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun' ] +C_MONTH_NAMES = [ 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec' ] + +XML_CONTENT_RE = re.compile('^application/xml(?:;\s+|$)', re.IGNORECASE) - This class just dispatches everything to boto. It uses a separate boto - connection object for each thread. +RETRY_TIMEOUT=60*60*24 +def retry(fn): + '''Decorator for retrying a method on some exceptions + + If the decorated method raises an exception for which the instance's + `_retry_on(exc)` method is true, the decorated method is called again at + increasing intervals. If this persists for more than *timeout* seconds, + the most-recently caught exception is re-raised. + ''' + + @wraps(fn) + def wrapped(self, *a, **kw): + interval = 1/50 + waited = 0 + while True: + try: + return fn(self, *a, **kw) + except Exception as exc: + # Access to protected member ok + #pylint: disable=W0212 + if not self._retry_on(exc): + raise + if waited > RETRY_TIMEOUT: + log.error('%s.%s(*): Timeout exceeded, re-raising %r exception', + self.__class__.__name__, fn.__name__, exc) + raise + + log.debug('%s.%s(*): trying again after %r exception:', + self.__class__.__name__, fn.__name__, exc) + + time.sleep(interval) + waited += interval + if interval < 20*60: + interval *= 2 + + # False positive + #pylint: disable=E1101 + wrapped.__doc__ += ''' +This method has been decorated and will automatically recall itself in +increasing intervals for up to s3ql.common.RETRY_TIMEOUT seconds if it raises an +exception for which the instance's `_retry_on` method returns True. +''' + return wrapped + + +class Bucket(AbstractBucket): + """A bucket stored in Amazon S3 - This class is threadsafe. All methods (except for internal methods - starting with underscore) may be called concurrently by different - threads. + This class uses standard HTTP connections to connect to S3. + + The bucket guarantees get after create consistency, i.e. a newly created + object will be immediately retrievable. Additional consistency guarantees + may or may not be available and can be queried for with instance methods. """ - def __init__(self, awskey, awspass, use_ssl, - reduced_redundancy=False): - super(Connection, self).__init__() - self.awskey = awskey - self.awspass = awspass - self.pool = list() - self.lock = threading.RLock() - self.conn_cnt = 0 - self.use_ssl = use_ssl - self.reduced_redundancy = reduced_redundancy - - def _pop_conn(self): - '''Get boto connection object from the pool''' - - with self.lock: - try: - conn = self.pool.pop() - except IndexError: - # Need to create a new connection - log.debug("Creating new boto connection (active conns: %d)...", - self.conn_cnt) - conn = S3Connection(self.awskey, self.awspass, - is_secure=self.use_ssl) - self.conn_cnt += 1 + def __init__(self, bucket_name, aws_key_id, aws_key): + super(Bucket, self).__init__() + + try: + idx = bucket_name.index('/') + except ValueError: + idx = len(bucket_name) + + self.bucket_name = bucket_name[:idx] + self.prefix = bucket_name[idx+1:] + self.aws_key = aws_key + self.aws_key_id = aws_key_id + self.namespace = 'http://s3.amazonaws.com/doc/2006-03-01/' + + self._init() + + def _init(self): + '''Additional initialization code + + Called by constructor and provided solely for easier subclassing. + ''' + # Attributes defined outside init + #pylint: disable=W0201 + + self.conn = self._get_conn() + self.region = self._get_region() + + if self.region not in ('EU', 'us-west-1', 'ap-southeast-1'): + log.warn('Warning: bucket provides insufficient consistency guarantees!') + + def _get_conn(self): + '''Return connection to server''' + + return httplib.HTTPConnection('%s.s3.amazonaws.com' % self.bucket_name) + + @staticmethod + def is_temp_failure(exc): + '''Return true if exc indicates a temporary error + + Return true if the given exception is used by this bucket's backend + to indicate a temporary problem. Most instance methods automatically + retry the request in this case, so the caller does not need to + worry about temporary failures. + + However, in same cases (e.g. when reading or writing an object), the + request cannot automatically be retried. In these case this method can + be used to check for temporary problems and so that the request can + be manually restarted if applicable. + ''' - return conn - - def _push_conn(self, conn): - '''Return boto connection object to pool''' - with self.lock: - self.pool.append(conn) - - def delete_bucket(self, name, recursive=False): - """Delete bucket""" - - if not recursive: - with self._get_boto() as boto: - boto.delete_bucket(name) - return - - # Delete recursively - with self._get_boto() as boto: - step = 1 - waited = 0 - while waited < 600: - try: - boto.delete_bucket(name) - except exception.S3ResponseError as exc: - if exc.code != 'BucketNotEmpty': - raise - else: - return - self.get_bucket(name, passphrase=None).clear() - time.sleep(step) - waited += step - step *= 2 - - raise RuntimeError('Bucket does not seem to get empty') - - - @contextmanager - def _get_boto(self): - """Provide boto connection object""" - - conn = self._pop_conn() + if isinstance(exc, (InternalError, BadDigest, IncompleteBody, RequestTimeout, + OperationAborted, SlowDown, RequestTimeTooSkewed, + httplib.IncompleteRead)): + return True + + # Server closed connection + elif (isinstance(exc, httplib.BadStatusLine) + and (not exc.line or exc.line == "''")): + return True + + elif isinstance(exc, IOError) and exc.errno == errno.EPIPE: + return True + + return False + + _retry_on = is_temp_failure + + @retry + def delete(self, key, force=False): + '''Delete the specified object''' + + log.debug('delete(%s)', key) try: - yield conn - finally: - self._push_conn(conn) + resp = self._do_request('DELETE', '/%s%s' % (self.prefix, key)) + assert resp.length == 0 + except NoSuchKey: + if force: + pass + else: + raise NoSuchObject(key) + + def list(self, prefix=''): + '''List keys in bucket - def create_bucket(self, name, location, passphrase=None, - compression='lzma'): - """Create and return an S3 bucket + Returns an iterator over all keys in the bucket. + ''' - Note that a call to `get_bucket` right after creation may fail, - since the changes do not propagate instantaneously through AWS. - """ - # Argument number deliberately differs from base class - #pylint: disable-msg=W0221 + log.debug('list(%s): start', prefix) - self.check_name(name) - with self._get_boto() as boto: + marker = '' + waited = 0 + interval = 1/50 + iterator = self._list(prefix, marker) + while True: try: - boto.create_bucket(name, location=location) - except exception.S3ResponseError as exc: - if exc.code == 'InvalidBucketName': - raise InvalidBucketNameError() - else: + marker = iterator.next() + waited = 0 + except StopIteration: + break + except Exception as exc: + if not self.is_temp_failure(exc): + raise + if waited > 60*60: + log.error('list(): Timeout exceeded, re-raising %r exception', exc) raise + + log.debug('list(): trying again after %r exception:', exc) + time.sleep(interval) + waited += interval + if interval < 20*60: + interval *= 2 + iterator = self._list(prefix, marker) - return Bucket(self, name, passphrase, compression) + else: + yield marker - def check_name(self, name): - '''Check if bucket name conforms to requirements - - Raises `InvalidBucketName` for invalid names. + def _list(self, prefix='', start=''): + '''List keys in bucket, starting with *start* + + Returns an iterator over all keys in the bucket. ''' - - if (not re.match('^[a-z0-9][a-z0-9.-]{1,60}[a-z0-9]$', name) - or '..' in name - or '.-' in name - or '-.' in name - or re.match('^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$', name)): - raise InvalidBucketNameError() - - def get_bucket(self, name, passphrase=None, compression='lzma'): - """Return a bucket instance for the bucket `name`""" - self.check_name(name) + keys_remaining = True + marker = start + prefix = self.prefix + prefix - with self._get_boto() as boto: - try: - boto.get_bucket(name) - except exception.S3ResponseError as e: - if e.status == 404: - raise NoSuchBucket(name) - elif e.code == 'InvalidBucketName': - raise InvalidBucketNameError() - else: - raise - return Bucket(self, name, passphrase, compression) + while keys_remaining: + log.debug('list(%s): requesting with marker=%s', prefix, marker) + + keys_remaining = None + resp = self._do_request('GET', '/', query_string={ 'prefix': prefix, + 'marker': marker, + 'max-keys': 1000 }) + + if not XML_CONTENT_RE.match(resp.getheader('Content-Type')): + raise RuntimeError('unexpected content type: %s' % resp.getheader('Content-Type')) + + itree = iter(ElementTree.iterparse(resp, events=("start", "end"))) + (event, root) = itree.next() -class Bucket(AbstractBucket): - """Represents a bucket stored in Amazon S3. + namespace = re.sub(r'^\{(.+)\}.+$', r'\1', root.tag) + if namespace != self.namespace: + raise RuntimeError('Unsupported namespace: %s' % namespace) + + try: + for (event, el) in itree: + if event != 'end': + continue + + if el.tag == '{%s}IsTruncated' % self.namespace: + keys_remaining = (el.text == 'true') + + elif el.tag == '{%s}Contents' % self.namespace: + marker = el.findtext('{%s}Key' % self.namespace) + yield marker + root.clear() + + except GeneratorExit: + # Need to read rest of response + while True: + buf = resp.read(8192) + if buf == '': + break + break + + if keys_remaining is None: + raise RuntimeError('Could not parse body') - This class should not be instantiated directly, but using - `Connection.get_bucket()`. + @retry + def _get_region(self): + ''''Return bucket region''' + + log.debug('_get_region()') + resp = self._do_request('GET', '/', subres='location') + + region = ElementTree.parse(resp).getroot().text + + if not region: + region = 'us-standard' + + if region not in ('EU', 'us-west-1', 'ap-southeast-1', + 'ap-northeast-1', 'us-standard'): + raise RuntimeError('Unknown bucket region: %s' % region) + + return region + + @retry + def lookup(self, key): + """Return metadata for given key""" + + log.debug('lookup(%s)', key) + + try: + resp = self._do_request('HEAD', '/%s%s' % (self.prefix, key)) + assert resp.length == 0 + except HTTPError as exc: + if exc.status == 404: + raise NoSuchObject(key) + else: + raise - Due to AWS' eventual propagation model, we may receive e.g. a 'unknown - bucket' error when we try to upload a key into a newly created bucket. For - this reason, many boto calls are wrapped with `retry_boto`. Note that this - assumes that no one else is messing with the bucket at the same time. + return extractmeta(resp) - This class is threadsafe. All methods (except for internal methods - starting with underscore) may be called concurrently by different - threads. - """ + @retry + def open_read(self, key): + ''''Open object for reading - @contextmanager - def _get_boto(self): - '''Provide boto bucket object''' - # Access to protected methods ok - #pylint: disable-msg=W0212 - - boto_conn = self.conn._pop_conn() + Return a tuple of a file-like object. Bucket contents can be read from + the file-like object, metadata is stored in its *metadata* attribute and + can be modified by the caller at will. The object must be closed explicitly. + ''' + try: - yield retry_boto(boto_conn.get_bucket, self.name) - finally: - self.conn._push_conn(boto_conn) + resp = self._do_request('GET', '/%s%s' % (self.prefix, key)) + except NoSuchKey: + raise NoSuchObject(key) + + return ObjectR(key, resp, self, extractmeta(resp)) + + def open_write(self, key, metadata=None): + """Open object for writing - def __init__(self, conn, name, passphrase, compression): - super(Bucket, self).__init__(passphrase, compression) - self.conn = conn - self.name = name - with self._get_boto() as boto: - self.rac_consistent = (boto.get_location() != '') + `metadata` can be a dict of additional attributes to store with the + object. Returns a file-like object that must be closed when all data has + been written. + + Since Amazon S3 does not support chunked uploads, the entire data will + be buffered in memory before upload. + """ + + log.debug('open_write(%s): start', key) + + headers = dict() + if metadata: + for (hdr, val) in metadata.iteritems(): + headers['x-amz-meta-%s' % hdr] = val + + return ObjectW(key, self, headers) + + def is_get_consistent(self): + '''If True, objects retrievals are guaranteed to be up-to-date + + If this method returns True, then creating, deleting, or overwriting an + object is guaranteed to be immediately reflected in subsequent object + retrieval attempts. + ''' + + return False + + def is_list_create_consistent(self): + '''If True, new objects are guaranteed to show up in object listings + + If this method returns True, creation of objects will immediately be + reflected when retrieving the list of available objects. + ''' + return self.region in ('EU', 'us-west-1', 'ap-southeast-1') + + @retry + def copy(self, src, dest): + """Copy data stored under key `src` to key `dest` + + If `dest` already exists, it will be overwritten. The copying is done on + the remote side. + """ + + log.debug('copy(%s, %s): start', src, dest) + + try: + resp = self._do_request('PUT', '/%s%s' % (self.prefix, dest), + headers={ 'x-amz-copy-source': '/%s/%s%s' % (self.bucket_name, + self.prefix, src)}) + # Discard response body + resp.read() + except NoSuchKey: + raise NoSuchObject(src) + + def _do_request(self, method, url, subres=None, query_string=None, + headers=None, body=None ): + '''Send request, read and return response object''' + + log.debug('_do_request(): start with parameters (%r, %r, %r, %r, %r, %r)', + method, url, subres, query_string, headers, body) + + if headers is None: + headers = dict() + + headers['connection'] = 'keep-alive' + + if not body: + headers['content-length'] = '0' + + full_url = self._add_auth(method, url, headers, subres, query_string) + + redirect_count = 0 + while True: + log.debug('_do_request(): sending request') + self.conn.request(method, full_url, body, headers) + + log.debug('_do_request(): Reading response') + try: + resp = self.conn.getresponse() + except httplib.BadStatusLine as exc: + if exc.line == "''" or not exc.line: + # Server closed connection, reconnect + self.conn.close() + raise + except httplib.CannotSendRequest: + log.warn('_do_request(): httplib can not send request, ' + 'retrying (current state: %s)..', self.conn.__state) + raise + + log.debug('_do_request(): request-id: %s', resp.getheader('x-amz-request-id')) + + if (resp.status < 300 or resp.status > 399): + break + + redirect_count += 1 + if redirect_count > 10: + raise RuntimeError('Too many chained redirections') + full_url = resp.getheader('Location') + + log.debug('_do_request(): redirecting to %s', full_url) + + if body and not isinstance(body, bytes): + body.seek(0) + + # Read and discard body + resp.read() + + # We need to call read() at least once for httplib to consider this + # request finished, even if there is no response body. + if resp.length == 0: + resp.read() + + # Success + if resp.status >= 200 and resp.status <= 299: + return resp + + # If method == HEAD, server must not return response body + # even in case of errors + if method.upper() == 'HEAD': + raise HTTPError(resp.status, resp.reason) + + + content_type = resp.getheader('Content-Type') + if not content_type or not XML_CONTENT_RE.match(content_type): + raise RuntimeError('\n'.join(['Unexpected S3 reply:', + '%d %s' % (resp.status, resp.reason) ] + + [ '%s: %s' % x for x in resp.getheaders() ]) + + '\n' + resp.read()) + # Error + tree = ElementTree.parse(resp).getroot() + raise get_S3Error(tree.findtext('Code'), tree.findtext('Message')) + + def clear(self): """Delete all objects in bucket - This function starts multiple threads.""" + Note that this method may not be able to see (and therefore also not + delete) recently uploaded objects if `is_list_create_consistent` is + False. + """ - threads = list() for (no, s3key) in enumerate(self): if no != 0 and no % 1000 == 0: - log.info('Deleted %d objects so far..', no) + log.info('clear(): deleted %d objects so far..', no) - log.debug('Deleting key %s', s3key) + log.debug('clear(): deleting key %s', s3key) # Ignore missing objects when clearing bucket - t = AsyncFn(self.delete, s3key, True) - t.start() - threads.append(t) - - if len(threads) > 50: - log.debug('50 threads reached, waiting..') - threads.pop(0).join_and_raise() - - log.debug('Waiting for removal threads') - for t in threads: - t.join_and_raise() + self.delete(s3key, True) def __str__(self): - if self.passphrase: - return '<encrypted s3 bucket, name=%r>' % self.name - else: - return '<s3 bucket, name=%r>' % self.name - - def contains(self, key): - with self._get_boto() as boto: - bkey = retry_boto(boto.get_key, key) - - return bkey is not None - - def read_after_create_consistent(self): - return self.rac_consistent + return 's3://%s/%s' % (self.bucket_name, self.prefix) - def read_after_write_consistent(self): - return False - - def read_after_delete_consistent(self): - return False - - def raw_lookup(self, key): - '''Retrieve metadata for `key` + def _add_auth(self, method, url, headers, subres=None, query_string=None): + '''Add authentication to *headers* - If the key has been lost (S3 returns 405), it is automatically - deleted so that it will no longer be returned by list_keys. + Note that *headers* is modified in-place. As a convenience, + this method returns the encoded URL with query string and + sub resource appended. + + *query_string* must be a dict or *None*. *subres* must be a + string or *None*. ''' - with self._get_boto() as boto: - bkey = _get_boto_key(boto, key) - - if bkey is None: - raise NoSuchObject(key) - - return bkey.metadata - - def delete(self, key, force=False): - """Deletes the specified key - - ``bucket.delete(key)`` can also be written as ``del bucket[key]``. - If `force` is true, do not return an error if the key does not exist. - """ - - if not isinstance(key, str): - raise TypeError('key must be of type str') - - with self._get_boto() as boto: - if not force and retry_boto(boto.get_key, key) is None: - raise NoSuchObject(key) - - retry_boto(boto.delete_key, key) - - def list(self, prefix=''): - with self._get_boto() as boto: - for bkey in boto.list(prefix): - yield bkey.name - - def raw_fetch(self, key, fh): - '''Fetch `key` and store in `fh` + + # See http://docs.amazonwebservices.com/AmazonS3/latest/dev/RESTAuthentication.html - If the key has been lost (S3 returns 405), it is automatically - deleted so that it will no longer be returned by list_keys. - ''' + # Lowercase headers + keys = list(headers.iterkeys()) + for key in keys: + key_l = key.lower() + if key_l == key: + continue + headers[key_l] = headers[key] + del headers[key] + + # Date, can't use strftime because it's locale dependent + now = time.gmtime() + headers['date'] = ('%s, %02d %s %04d %02d:%02d:%02d GMT' + % (C_DAY_NAMES[now.tm_wday], + now.tm_mday, + C_MONTH_NAMES[now.tm_mon - 1], + now.tm_year, now.tm_hour, + now.tm_min, now.tm_sec)) - with self._get_boto() as boto: - bkey = _get_boto_key(boto, key) - - if bkey is None: - raise NoSuchObject(key) - fh.seek(0) - retry_boto(bkey.get_contents_to_file, fh) - - return bkey.metadata - - def raw_store(self, key, fh, metadata): - with self._get_boto() as boto: - bkey = boto.new_key(key) - bkey.metadata.update(metadata) - retry_boto(bkey.set_contents_from_file, fh, - reduced_redundancy=self.conn.reduced_redundancy) - + auth_strs = [method, '\n'] + + for hdr in ('content-md5', 'content-type', 'date'): + if hdr in headers: + auth_strs.append(headers[hdr]) + auth_strs.append('\n') + + for hdr in sorted(x for x in headers if x.startswith('x-amz-')): + val = ' '.join(re.split(r'\s*\n\s*', headers[hdr].strip())) + auth_strs.append('%s:%s\n' % (hdr,val)) + + auth_strs.append('/' + self.bucket_name) + url = urllib.quote(url) + auth_strs.append(url) + if subres: + auth_strs.append('?%s' % subres) + + # False positive, hashlib *does* have sha1 member + #pylint: disable=E1101 + signature = b64encode(hmac.new(self.aws_key, ''.join(auth_strs), hashlib.sha1).digest()) + + headers['authorization'] = 'AWS %s:%s' % (self.aws_key_id, signature) + + full_url = url + if query_string: + s = urllib.urlencode(query_string, doseq=True) + if subres: + full_url += '?%s&%s' % (subres, s) + else: + full_url += '?%s' % s + elif subres: + full_url += '?%s' % subres + + return full_url - def copy(self, src, dest): - if not isinstance(src, str): - raise TypeError('key must be of type str') + +class ObjectR(object): + '''An S3 object open for reading''' + + def __init__(self, key, resp, bucket, metadata=None): + self.key = key + self.resp = resp + self.closed = False + self.bucket = bucket + self.metadata = metadata + + # False positive, hashlib *does* have md5 member + #pylint: disable=E1101 + self.md5 = hashlib.md5() + + def read(self, size=None): + '''Read object data''' + + # chunked encoding handled by httplib + buf = self.resp.read(size) + self.md5.update(buf) + return buf + + def __enter__(self): + return self + + def __exit__(self, *a): + self.close() + return False + + def _retry_on(self, exc): + return self.bucket._retry_on(exc) #IGNORE:W0212 + + @retry + def close(self): + '''Close object''' - if not isinstance(dest, str): - raise TypeError('key must be of type str') + # Access to protected member ok + #pylint: disable=W0212 + + log.debug('ObjectR(%s).close(): start', self.key) + self.closed = True - with self._get_boto() as boto: - retry_boto(boto.copy_key, dest, self.name, src) + while True: + buf = self.read(8192) + if buf == '': + break -def _get_boto_key(boto, key): - '''Get boto key object for `key` + etag = self.resp.getheader('ETag').strip('"') + + if etag != self.md5.hexdigest(): + log.warn('ObjectR(%s).close(): MD5 mismatch: %s vs %s', self.key, etag, self.md5.hexdigest()) + raise BadDigest('BadDigest', 'Received ETag does not agree with our calculations.') + + def __del__(self): + if not self.closed: + try: + self.close() + except: + pass + raise RuntimeError('ObjectR %s has been destroyed without calling close()!' % self.key) - If the key has been lost (S3 returns 405), it is automatically - deleted so that it will no longer be returned by list_keys. +class ObjectW(object): + '''An S3 object open for writing + + All data is first cached in memory, upload only starts when + the close() method is called. ''' - try: - return retry_boto(boto.get_key, key) - except exception.S3ResponseError as exc: - if exc.error_code != 'MethodNotAllowed': - raise - - # Object was lost - log.warn('Object %s has been lost by Amazon, deleting..', key) - retry_boto(boto.delete_key, key) - return None - -def retry_boto(fn, *a, **kw): - """Wait for fn(*a, **kw) to succeed + def __init__(self, key, bucket, headers): + self.key = key + self.bucket = bucket + self.headers = headers + self.closed = False + self.fh = tempfile.TemporaryFile(bufsize=0) # no Python buffering + + # False positive, hashlib *does* have md5 member + #pylint: disable=E1101 + self.md5 = hashlib.md5() + + def write(self, buf): + '''Write object data''' + + self.fh.write(buf) + self.md5.update(buf) + + def _retry_on(self, exc): + return self.bucket._retry_on(exc) #IGNORE:W0212 + + @retry + def close(self): + '''Close object and upload data''' + + # Access to protected member ok + #pylint: disable=W0212 + + log.debug('ObjectW(%s).close(): start', self.key) + + self.closed = True + self.headers['Content-Length'] = os.fstat(self.fh.fileno()).st_size + + self.fh.seek(0) + resp = self.bucket._do_request('PUT', '/%s%s' % (self.bucket.prefix, self.key), + headers=self.headers, body=self.fh) + etag = resp.getheader('ETag').strip('"') + assert resp.length == 0 + + if etag != self.md5.hexdigest(): + log.warn('ObjectW(%s).close(): MD5 mismatch (%s vs %s)', self.key, etag, + self.md5.hexdigest) + raise BadDigest('BadDigest', 'Received ETag does not agree with our calculations.') + + def __del__(self): + if not self.closed: + try: + self.close() + except: + pass + raise RuntimeError('ObjectW %s has been destroyed without calling close()!' % self.key) + + def __enter__(self): + return self - If `fn(*a, **kw)` raises any of + def __exit__(self, *a): + self.close() + return False - - `boto.exception.S3ResponseError` with errorcode in - (`NoSuchBucket`, `RequestTimeout`) - - `IOError` with errno 104 - - `httplib.IncompleteRead` - - the function is called again. If the timeout is reached, `TimeoutError` is raised. - """ - - step = 0.2 - timeout = 300 - waited = 0 - while waited < timeout: - try: - return fn(*a, **kw) - except exception.S3ResponseError as exc: - if exc.error_code in ('NoSuchBucket', 'RequestTimeout', 'InternalError'): - log.warn('Encountered %s error when calling %s, retrying...', - exc.error_code, fn.__name__) - else: - raise - except IOError as exc: - if exc.errno == errno.ECONNRESET: - pass - else: - raise - except exception.S3CopyError as exc: - if exc.error_code in ('RequestTimeout', 'InternalError'): - log.warn('Encountered %s error when calling %s, retrying...', - exc.error_code, fn.__name__) - else: - raise - except httplib.IncompleteRead as exc: - log.warn('Encountered IncompleteRead error when calling %s, retrying...', - fn.__name__) - - sleep(step) - waited += step - if step < timeout / 30: - step *= 2 - - raise TimeoutError() - -class InvalidBucketNameError(Exception): - + +def get_S3Error(code, msg): + '''Instantiate most specific S3Error subclass''' + + return globals().get(code, S3Error)(code, msg) + +def extractmeta(resp): + '''Extract metadata from HTTP response object''' + + meta = dict() + for (name, val) in resp.getheaders(): + hit = re.match(r'^x-amz-meta-(.+)$', name) + if not hit: + continue + meta[hit.group(1)] = val + + return meta + +class HTTPError(Exception): + ''' + Represents an HTTP error returned by S3 in response to a + HEAD request. + ''' + + def __init__(self, status, msg): + super(HTTPError, self).__init__() + self.status = status + self.msg = msg + def __str__(self): - return 'Bucket name contains invalid characters.' + return '%d %s' % (self.status, self.msg) + +class S3Error(Exception): + ''' + Represents an error returned by S3. For possible codes, see + http://docs.amazonwebservices.com/AmazonS3/latest/API/ErrorResponses.html + ''' + + def __init__(self, code, msg): + super(S3Error, self).__init__() + self.code = code + self.msg = msg + + def __str__(self): + return '%s: %s' % (self.code, self.msg) + +class NoSuchKey(S3Error): pass +class AccessDenied(S3Error): pass +class BadDigest(S3Error): pass +class IncompleteBody(S3Error): pass +class InternalError(S3Error): pass +class InvalidAccessKeyId(S3Error): pass +class InvalidSecurity(S3Error): pass +class OperationAborted(S3Error): pass +class RequestTimeout(S3Error): pass +class SlowDown(S3Error): pass +class RequestTimeTooSkewed(S3Error): pass diff --git a/src/s3ql/backends/s3c.py b/src/s3ql/backends/s3c.py new file mode 100644 index 0000000..088843e --- /dev/null +++ b/src/s3ql/backends/s3c.py @@ -0,0 +1,100 @@ +''' +backends/s3c.py - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright (C) Nikolaus Rath <Nikolaus@rath.org> + +This program can be distributed under the terms of the GNU GPLv3. +''' + +from __future__ import division, print_function, absolute_import +from . import s3 +import httplib +import logging + +log = logging.getLogger("backends.s3c") + +# Pylint goes berserk with false positives +#pylint: disable=E1002,E1101 + +class Bucket(s3.Bucket): + """A bucket stored in some S3 compatible storage service. + + This class uses standard HTTP connections to connect to GS. + + The bucket guarantees only immediate get after create consistency. + """ + + + def __init__(self, bucket_name, login, password): + + try: + idx = bucket_name.index('/') + except ValueError: + idx = len(bucket_name) + self.hostname = bucket_name[:idx] + bucket_name = bucket_name[idx+1:] + + super(Bucket, self).__init__(bucket_name, login, password) + + if ':' in self.hostname: + idx = self.hostname.index(':') + self.conn = httplib.HTTPConnection(self.hostname[:idx], + port=int(self.hostname[idx+1:])) + else: + self.conn = httplib.HTTPConnection(self.hostname) + + self.namespace = 'http://s3.amazonaws.com/doc/2006-03-01/' + + def _init(self): + '''Additional initialization code + + Called by constructor and provided solely for easier subclassing. + ''' + pass + + def _do_request(self, method, url, subres=None, query_string=None, + headers=None, body=None ): + '''Send request, read and return response object''' + + # Need to add bucket name + url = '/%s%s' % (self.bucket_name, url) + return super(Bucket, self)._do_request(method, url, subres, + query_string, headers, body) + + def _add_auth(self, method, url, headers, subres=None, query_string=None): + '''Add authentication to *headers* + + Note that *headers* is modified in-place. As a convenience, + this method returns the encoded URL with query string and + sub resource appended. + + *query_string* must be a dict or *None*. *subres* must be a + string or *None*. + ''' + + # Bucket name is already part of URL + url = url[len(self.bucket_name)+1:] + return '/%s%s' % (self.bucket_name, + super(Bucket, self)._add_auth(method, url, headers, + subres, query_string)) + + def is_get_consistent(self): + '''If True, objects retrievals are guaranteed to be up-to-date + + If this method returns True, then creating, deleting, or overwriting an + object is guaranteed to be immediately reflected in subsequent object + retrieval attempts. + ''' + return False + + def is_list_create_consistent(self): + '''If True, new objects are guaranteed to show up in object listings + + If this method returns True, creation of objects will immediately be + reflected when retrieving the list of available objects. + ''' + return False + + def __str__(self): + return 's3c://%s/%s/%s' % (self.hostname, self.bucket_name, self.prefix) + diff --git a/src/s3ql/backends/s3cs.py b/src/s3ql/backends/s3cs.py new file mode 100644 index 0000000..f2ae747 --- /dev/null +++ b/src/s3ql/backends/s3cs.py @@ -0,0 +1,32 @@ +''' +backends/s3cs.py - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright (C) Nikolaus Rath <Nikolaus@rath.org> + +This program can be distributed under the terms of the GNU GPLv3. +''' + +from __future__ import division, print_function, absolute_import + +from . import s3c +import httplib + +# Pylint goes berserk with false positives +#pylint: disable=E1002,E1101,W0232 + + +class Bucket(s3c.Bucket): + """A bucket stored in some S3 compatible storage service. + + This class uses secure (SSL) connections. + + The bucket guarantees only immediate get after create consistency. + """ + + def _get_conn(self): + '''Return connection to server''' + + return httplib.HTTPSConnection(self.bucket_name) + + def __str__(self): + return 's3cs://%s/%s' % (self.bucket_name, self.prefix)
\ No newline at end of file diff --git a/src/s3ql/backends/s3s.py b/src/s3ql/backends/s3s.py new file mode 100644 index 0000000..2a5ecc4 --- /dev/null +++ b/src/s3ql/backends/s3s.py @@ -0,0 +1,30 @@ +''' +s3s.py - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright (C) Nikolaus Rath <Nikolaus@rath.org> + +This program can be distributed under the terms of the GNU GPLv3. +''' + +from __future__ import division, print_function, absolute_import + +from . import s3 +import httplib + +class Bucket(s3.Bucket): + """A bucket stored in Amazon S3 + + This class uses secure (SSL) connections to connect to S3. + + The bucket guarantees get after create consistency, i.e. a newly created + object will be immediately retrievable. Additional consistency guarantees + may or may not be available and can be queried for with instance methods. + """ + + def _get_conn(self): + '''Return connection to server''' + + return httplib.HTTPSConnection('%s.s3.amazonaws.com' % self.bucket_name) + + def __str__(self): + return 's3s://%s/%s' % (self.bucket_name, self.prefix)
\ No newline at end of file diff --git a/src/s3ql/backends/sftp.py b/src/s3ql/backends/sftp.py deleted file mode 100644 index 5900901..0000000 --- a/src/s3ql/backends/sftp.py +++ /dev/null @@ -1,349 +0,0 @@ -''' -__init__.py - this file is part of S3QL (http://s3ql.googlecode.com) - -Copyright (C) 2010 Nikolaus Rath <Nikolaus@rath.org> -Copyright (C) 2010 Ron Knapp <ron.siesta@gmail.com> - -This program can be distributed under the terms of the GNU LGPL. -''' - -from __future__ import division, print_function, absolute_import - -from .common import AbstractConnection, AbstractBucket, NoSuchBucket, NoSuchObject -import logging -import errno -import shutil -import cPickle as pickle -import os -import stat -import paramiko -import threading - -log = logging.getLogger("backend.sftp") - - -class Connection(AbstractConnection): - ''' - Provides a connection to an SFTP server. - - This class is threadsafe. All methods (except for internal methods - starting with underscore) may be called concurrently by different - threads. - ''' - - def __init__(self, host, port, login, password): - super(Connection, self).__init__() - - self.port = port or 22 - self.host = host - self.login = login - self.password = password - - self._client = None - self.sftp = None - - self._setup_ssh_connection() - - self.lock = threading.RLock() - - def _setup_ssh_connection(self): - - self._client = paramiko.SSHClient() - # Probably not a good idea to do this by default - #self._client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - self._client.load_host_keys(os.path.expanduser('~/.ssh/known_hosts')) - self._client.connect(self.host, port=self.port, username=self.login, password=self.password) - self.sftp = self._client.open_sftp() - - # We don't want the connection to time out - self._client.get_transport().set_keepalive(300) - - def __contains__(self, entry): - with self.lock: - try: - self.sftp.stat(entry) - except IOError as exc: - if exc.errno == errno.ENOENT: - return False - else: - raise - else: - return True - - def delete_bucket(self, name, recursive=False): - """Delete bucket""" - - with self.lock: - if name not in self: - raise NoSuchBucket(name) - - if recursive: - self._rmtree(name) - - self.sftp.rmdir(name) - - def _rmtree(self, path): - '''Recursively delete contents of remote path''' - - for attr in self.sftp.listdir_attr(path): - fullname = '%s/%s' % (path, attr.filename) - if stat.S_ISDIR(attr.st_mode): - self._rmtree(fullname) - self.sftp.rmdir(fullname) - else: - self.sftp.remove(fullname) - - - def create_bucket(self, name, passphrase=None, compression='lzma'): - """Create and return bucket""" - - with self.lock: - self.sftp.mkdir(name) - return self.get_bucket(name, passphrase, compression) - - def get_bucket(self, name, passphrase=None, compression='lzma'): - """Return Bucket instance for the bucket `name`""" - - with self.lock: - if name not in self: - raise NoSuchBucket(name) - - return Bucket(self, name, passphrase, compression) - - def close(self): - with self.lock: - self._client.close() - - def prepare_fork(self): - with self.lock: - self._client.close() - - def finish_fork(self): - with self.lock: - self._setup_ssh_connection() - -class Bucket(AbstractBucket): - ''' - Stores data remotely on an SFTP server. - - This class is threadsafe. All methods (except for internal methods - starting with underscore) may be called concurrently by different - threads. - ''' - - def __init__(self, conn, name, passphrase, compression): - super(Bucket, self).__init__(passphrase, compression) - self.conn = conn - self.name = name - - def _key_to_path(self, key): - '''Return path for given key''' - - key = _escape(key) - - if not key.startswith('s3ql_data_'): - return os.path.join(self.name, key) - - no = key[10:] - path = [ self.name, 's3ql_data'] - for i in range(0, len(no), 3): - path.append(no[:i]) - path.append(key) - - return os.path.join(*path) - - def __str__(self): - return '<sftp bucket, name=%r>' % self.name - - def read_after_create_consistent(self): - return True - - def read_after_write_consistent(self): - return True - - def read_after_delete_consistent(self): - return True - - def clear(self): - # Access to protected member ok - #pylint: disable=W0212 - with self.conn.lock: - self.conn._rmtree(self.name) - - def contains(self, key): - with self.conn.lock: - return (self._key_to_path(key) + '.dat') in self.conn - - def raw_lookup(self, key): - with self.conn.lock: - path = self._key_to_path(key) - try: - src = self.conn.sftp.open(path + '.meta', 'rb') - return pickle.load(src) - except IOError as exc: - if exc.errno == errno.ENOENT: - raise NoSuchObject(key) - else: - raise - - def delete(self, key, force=False): - with self.conn.lock: - path = self._key_to_path(key) - - try: - self.conn.sftp.remove(path + '.dat') - self.conn.sftp.remove(path + '.meta') - except IOError as exc: - if exc.errno == errno.ENOENT: - if force: - pass - else: - raise NoSuchObject(key) - else: - raise - - def list(self, prefix=''): - with self.conn.lock: - if prefix: - base = os.path.dirname(self._key_to_path(prefix)) - else: - base = self.name - - for (path, dirnames, filenames) in self._walk(base): - - # Do not look in wrong directories - if prefix: - rpath = path[len(self.name):] # path relative to base - prefix_l = ''.join(rpath.split('/')) - - dirs_to_walk = list() - for name in dirnames: - prefix_ll = _unescape(prefix_l + name) - if prefix_ll.startswith(prefix[:len(prefix_ll)]): - dirs_to_walk.append(name) - dirnames[:] = dirs_to_walk - - for name in filenames: - if not name.endswith('.dat'): - continue - key = _unescape(name[:-4]) - - if not prefix or key.startswith(prefix): - yield key - - def _walk(self, base): - '''Iterate recursively over directories, like os.walk''' - - to_visit = [ base ] - while to_visit: - base = to_visit.pop() - files = list() - for attr in self.conn.sftp.listdir_attr(base): - if stat.S_ISDIR(attr.st_mode): - to_visit.append('%s/%s' % (base, attr.filename)) - else: - files.append(attr.filename) - yield (base, to_visit, files) - - def _makedirs(self, path): - '''Like os.makedirs, but over sftp''' - - cur = '/' - done = False - for el in path.split('/'): - cur = '%s/%s' % (cur, el) - if cur not in self.conn: - self.conn.sftp.mkdir(cur) - done = True - - if not done: - err = OSError('Entry already exists: %s' % cur) - err.errno = errno.EEXIST - raise err - - def raw_fetch(self, key, fh): - with self.conn.lock: - path = self._key_to_path(key) - try: - src = self.conn.sftp.open(path + '.dat', 'r') - src.prefetch() - fh.seek(0) - shutil.copyfileobj(src, fh) - src.close() - - src = self.conn.sftp.open(path + '.meta', 'r') - src.prefetch() - metadata = pickle.load(src) - src.close() - - except IOError as exc: - if exc.errno == errno.ENOENT: - raise NoSuchObject(key) - else: - raise - - return metadata - - def raw_store(self, key, fh, metadata): - with self.conn.lock: - path = self._key_to_path(key) - fh.seek(0) - - try: - dest = self.conn.sftp.open(path + '.dat', 'w') - dest.set_pipelined(True) - except IOError as exc: - if exc.errno != errno.ENOENT: - raise - self._makedirs(os.path.dirname(path)) - dest = self.conn.sftp.open(path + '.dat', 'w') - dest.set_pipelined(True) - - shutil.copyfileobj(fh, dest) - dest.close() - - dest = self.conn.sftp.open(path + '.meta', 'w') - dest.set_pipelined(True) - pickle.dump(metadata, dest, 2) - dest.close() - - def rename(self, src, dest): - with self.conn.lock: - src_path = self._key_to_path(src) - dest_path = self._key_to_path(dest) - - try: - self.conn.sftp.lstat(src_path + '.dat') - except IOError as exc: - if exc.errno == errno.ENOENT: - raise NoSuchObject(src) - else: - raise - - try: - self.conn.sftp.rename(src_path + '.dat', dest_path + '.dat') - self.conn.sftp.rename(src_path + '.meta', dest_path + '.meta') - except IOError as exc: - if exc.errno != errno.ENOENT: - raise - self._makedirs(os.path.dirname(dest_path)) - self.conn.sftp.rename(src_path + '.dat', dest_path + '.dat') - self.conn.sftp.rename(src_path + '.meta', dest_path + '.meta') - -def _escape(s): - '''Escape '/', '=' and '\0' in s''' - - s = s.replace('=', '=3D') - s = s.replace('/', '=2F') - s = s.replace('\0', '=00') - - return s - -def _unescape(s): - '''Un-Escape '/', '=' and '\0' in s''' - - s = s.replace('=2F', '/') - s = s.replace('=00', '\0') - s = s.replace('=3D', '=') - - return s diff --git a/src/s3ql/block_cache.py b/src/s3ql/block_cache.py index f4bf152..0b694d5 100644 --- a/src/s3ql/block_cache.py +++ b/src/s3ql/block_cache.py @@ -3,33 +3,109 @@ block_cache.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import - -from contextlib import contextmanager -from .multi_lock import MultiLock -from .backends.common import NoSuchObject -from .ordered_dict import OrderedDict -from .common import EmbeddedException, ExceptionStoringThread -from .thread_group import ThreadGroup -from .upload_manager import UploadManager, RemoveThread, retry_exc +from .backends.common import CompressFilter +from .common import sha256_fh from .database import NoSuchRowError +from .ordered_dict import OrderedDict +from Queue import Queue +from contextlib import contextmanager from llfuse import lock, lock_released import logging import os +import shutil import threading import time -import stat - -__all__ = [ "BlockCache" ] # standard logger for this module log = logging.getLogger("BlockCache") + +# Buffer size when writing objects +BUFSIZE = 256 * 1024 + +# Special queue entry that signals threads to terminate +QuitSentinel = object() + +class Distributor(object): + ''' + Distributes objects to consumers. + ''' + + def __init__(self): + super(Distributor, self).__init__() + + self.slot = None + self.cv = threading.Condition() + + def put(self, obj): + '''Offer *obj* for consumption - -class CacheEntry(file): + The method blocks until another thread calls `get()` to consume + the object. + ''' + + if obj is None: + raise ValueError("Can't put None into Queue") + + with self.cv: + while self.slot is not None: + self.cv.wait() + self.slot = obj + self.cv.notify_all() + + def get(self): + '''Consume and return an object + + The method blocks until another thread offers an object + by calling the `put` method. + ''' + with self.cv: + while not self.slot: + self.cv.wait() + tmp = self.slot + self.slot = None + self.cv.notify_all() + return tmp + + +class SimpleEvent(object): + ''' + Like threading.Event, but without any internal flag. Calls + to `wait` always block until some other thread calls + `notify` or `notify_all`. + ''' + + def __init__(self): + super(SimpleEvent, self).__init__() + self.__cond = threading.Condition(threading.Lock()) + + def notify_all(self): + self.__cond.acquire() + try: + self.__cond.notify_all() + finally: + self.__cond.release() + + def notify(self): + self.__cond.acquire() + try: + self.__cond.notify() + finally: + self.__cond.release() + + def wait(self): + self.__cond.acquire() + try: + self.__cond.wait() + return + finally: + self.__cond.release() + + +class CacheEntry(object): """An element in the block cache If `obj_id` is `None`, then the object has not yet been @@ -38,54 +114,71 @@ class CacheEntry(file): Attributes: ----------- - :modified_after_upload: - This attribute is only significant when the cache entry - is currently being uploaded. At the beginning of the upload, - it is set to False. For any write access, it is set to True. - If it is still False when the upload has completed, - `dirty` is set to False and the object looses the ``.d`` suffix - in its name. - + :dirty: + entry has been changed since it was last uploaded. + + :size: current file size + + :pos: current position in file """ - __slots__ = [ 'dirty', 'obj_id', 'inode', 'blockno', 'last_access', - 'modified_after_upload' ] + __slots__ = [ 'dirty', 'inode', 'blockno', 'last_access', + 'size', 'pos', 'fh' ] - def __init__(self, inode, blockno, obj_id, filename, mode): - super(CacheEntry, self).__init__(filename, mode) + def __init__(self, inode, blockno, filename): + super(CacheEntry, self).__init__() + # Writing 100MB in 128k chunks takes 90ms unbuffered and + # 116ms with 1 MB buffer. Reading time does not depend on + # buffer size. + self.fh = open(filename, "w+b", 0) self.dirty = False - self.modified_after_upload = False - self.obj_id = obj_id self.inode = inode self.blockno = blockno self.last_access = 0 + self.pos = 0 + self.size = os.fstat(self.fh.fileno()).st_size - def truncate(self, *a, **kw): - if not self.dirty: - os.rename(self.name, self.name + '.d') - self.dirty = True - self.modified_after_upload = True - return super(CacheEntry, self).truncate(*a, **kw) - - def write(self, *a, **kw): - if not self.dirty: - os.rename(self.name, self.name + '.d') - self.dirty = True - self.modified_after_upload = True - return super(CacheEntry, self).write(*a, **kw) - - def writelines(self, *a, **kw): - if not self.dirty: - os.rename(self.name, self.name + '.d') - self.dirty = True - self.modified_after_upload = True - return super(CacheEntry, self).writelines(*a, **kw) - + def read(self, size=None): + buf = self.fh.read(size) + self.pos += len(buf) + return buf + + def flush(self): + self.fh.flush() + + def seek(self, off): + if self.pos != off: + self.fh.seek(off) + self.pos = off + + def tell(self): + return self.pos + + def truncate(self, size=None): + self.dirty = True + self.fh.truncate(size) + if size is None: + if self.pos < self.size: + self.size = self.pos + elif size < self.size: + self.size = size + + def write(self, buf): + self.dirty = True + self.fh.write(buf) + self.pos += len(buf) + self.size = max(self.pos, self.size) + + def close(self): + self.fh.close() + + def unlink(self): + os.unlink(self.fh.name) + def __str__(self): - return ('<CacheEntry, inode=%d, blockno=%d, dirty=%s, obj_id=%r>' % - (self.inode, self.blockno, self.dirty, self.obj_id)) + return ('<%sCacheEntry, inode=%d, blockno=%d>' + % ('Dirty ' if self.dirty else '', self.inode, self.blockno)) -MAX_REMOVAL_THREADS = 25 class BlockCache(object): """Provides access to file blocks @@ -94,182 +187,400 @@ class BlockCache(object): This class uses the llfuse global lock. Methods which release the lock have are marked as such in their docstring. - - - Attributes: - ----------- - :mlock: locks on (inode, blockno) during `get`, so that we do not - download the same object with more than one thread. - :encountered_errors: This attribute is set if some non-fatal errors - were encountered during asynchronous operations (for - example, an object that was supposed to be deleted did - not exist). + Attributes + ---------- + + :path: where cached data is stored + :entries: ordered dictionary of cache entries + :size: current size of all cached entries + :max_size: maximum size to which cache can grow + :in_transit: set of objects currently in transit and + (inode, blockno) tuples currently being uploaded + :removed_in_transit: set of objects that have been removed from the db + while in transit, and should be removed from the backend as soon + as the transit completes. + :to_upload: distributes objects to upload to worker threads + :to_remove: distributes objects to remove to worker threads + :transfer_complete: signals completion of an object transfer + (either upload or download) + + The `in_transit` attribute is used to + - Prevent multiple threads from downloading the same object + - Prevent threads from downloading an object before it has been + uploaded completely (can happen when a cache entry is linked to a + block while the object containing the block is still being + uploaded) """ - def __init__(self, bucket, db, cachedir, max_size, max_entries=768): + def __init__(self, bucket_pool, db, cachedir, max_size, max_entries=768): log.debug('Initializing') - self.cache = OrderedDict() - self.cachedir = cachedir - self.max_size = max_size + + self.path = cachedir + self.db = db + self.bucket_pool = bucket_pool + self.entries = OrderedDict() self.max_entries = max_entries self.size = 0 - self.db = db - self.bucket = bucket - self.mlock = MultiLock() - self.removal_queue = ThreadGroup(MAX_REMOVAL_THREADS) - self.upload_manager = UploadManager(bucket, db, self.removal_queue) - self.commit_thread = CommitThread(self) - self.encountered_errors = False - - def init(self): - log.debug('init: start') - if not os.path.exists(self.cachedir): - os.mkdir(self.cachedir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) - self.commit_thread.start() - log.debug('init: end') + self.max_size = max_size + self.in_transit = set() + self.removed_in_transit = set() + self.to_upload = Distributor() + self.to_remove = Queue() + self.upload_threads = [] + self.removal_threads = [] + self.transfer_completed = SimpleEvent() + + if not os.path.exists(self.path): + os.mkdir(self.path) + + def __len__(self): + '''Get number of objects in cache''' + return len(self.entries) + def init(self, threads=1): + '''Start worker threads''' + + for _ in range(threads): + t = threading.Thread(target=self._upload_loop) + t.start() + self.upload_threads.append(t) + + for _ in range(10): + t = threading.Thread(target=self._removal_loop) + t.daemon = True # interruption will do no permanent harm + t.start() + self.removal_threads.append(t) + def destroy(self): - log.debug('destroy: start') + '''Clean up and stop worker threads''' - # If there were errors, we still want to try to finalize - # as much as we can - try: - self.commit_thread.stop() - except Exception as exc: - self.encountered_errors = True - if isinstance(exc, EmbeddedException): - log.error('CommitThread encountered exception.') - else: - log.exception('Error when stopping commit thread') + log.debug('destroy(): clearing cache...') + self.clear() + + with lock_released: + for t in self.upload_threads: + self.to_upload.put(QuitSentinel) + for t in self.removal_threads: + self.to_remove.put(QuitSentinel) + + log.debug('destroy(): waiting for upload threads...') + for t in self.upload_threads: + t.join() + + log.debug('destroy(): waiting for removal threads...') + for t in self.removal_threads: + t.join() + + self.upload_threads = [] + self.removal_threads = [] + + os.rmdir(self.path) + + def _upload_loop(self): + '''Process upload queue''' + + while True: + tmp = self.to_upload.get() + + if tmp is QuitSentinel: + break + + self._do_upload(*tmp) + + def _do_upload(self, el, obj_id): + '''Upload object''' + try: - self.clear() - except: - self.encountered_errors = True - log.exception('Error when clearing cache') + if log.isEnabledFor(logging.DEBUG): + time_ = time.time() + + with self.bucket_pool() as bucket: + with bucket.open_write('s3ql_data_%d' % obj_id) as fh: + el.seek(0) + while True: + buf = el.read(BUFSIZE) + if not buf: + break + fh.write(buf) + + if log.isEnabledFor(logging.DEBUG): + time_ = time.time() - time_ + rate = el.size / (1024**2 * time_) if time_ != 0 else 0 + log.debug('_do_upload(%s): uploaded %d bytes in %.3f seconds, %.2f MB/s', + obj_id, el.size, time_, rate) - while True: - try: - self.upload_manager.join_all() - except Exception as exc: - self.encountered_errors = True - if isinstance(exc, EmbeddedException): - log.error('UploadManager encountered exception.') - else: - log.exception('Error when joining UploadManager') - break + if isinstance(fh, CompressFilter): + obj_size = fh.compr_size else: - break + obj_size = el.size + + with lock: + self.db.execute('UPDATE objects SET compr_size=? WHERE id=?', + (obj_size, obj_id)) + + el.dirty = False + self.in_transit.remove(obj_id) + self.in_transit.remove((el.inode, el.blockno)) + self.transfer_completed.notify_all() + + except: + with lock: + self.in_transit.remove(obj_id) + self.in_transit.remove((el.inode, el.blockno)) + self.transfer_completed.notify_all() + raise + + + def wait(self): + '''Wait until an object has been transferred + + If there are no objects in transit, return immediately. This method + releases the global lock. + ''' + + if not self.transfer_in_progress(): + return + + with lock_released: + self.transfer_completed.wait() + + def upload(self, el): + '''Upload cache entry `el` asynchronously + + Return (uncompressed) size of cache entry. + + This method releases the global lock. + ''' + + log.debug('upload(%s): start', el) + + assert (el.inode, el.blockno) not in self.in_transit + self.in_transit.add((el.inode, el.blockno)) + + try: + el.seek(0) + hash_ = sha256_fh(el) - while True: try: - self.removal_queue.join_all() - except Exception as exc: - self.encountered_errors = True - if isinstance(exc, EmbeddedException): - log.error('RemovalQueue encountered exception.') + if el.blockno == 0: + old_block_id = self.db.get_val('SELECT block_id FROM inodes ' + 'WHERE id=? AND block_id IS NOT NULL', + (el.inode,)) else: - log.exception('Error when waiting for removal queue:') - break + old_block_id = self.db.get_val('SELECT block_id FROM inode_blocks ' + 'WHERE inode=? AND blockno=?', + (el.inode, el.blockno)) + except NoSuchRowError: + old_block_id = None + + try: + block_id = self.db.get_val('SELECT id FROM blocks WHERE hash=?', (hash_,)) + + # No block with same hash + except NoSuchRowError: + obj_id = self.db.rowid('INSERT INTO objects (refcount) VALUES(1)') + log.debug('upload(%s): created new object %d', el, obj_id) + block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, hash, size) ' + 'VALUES(?,?,?,?)', (1, obj_id, hash_, el.size)) + log.debug('upload(%s): created new block %d', el, block_id) + log.debug('upload(%s): adding to upload queue', el) + self.in_transit.add(obj_id) + with lock_released: + if not self.upload_threads: + log.warn("upload(%s): no upload threads, uploading synchronously", el) + self._do_upload(el, obj_id) + else: + self.to_upload.put((el, obj_id)) + + # There is a block with the same hash else: - break - - if self.upload_manager.encountered_errors: - self.encountered_errors = True + if old_block_id == block_id: + log.debug('upload(%s): unchanged, block_id=%d', el, block_id) + el.dirty = False + self.in_transit.remove((el.inode, el.blockno)) + return el.size + + log.debug('upload(%s): (re)linking to %d', el, block_id) + self.db.execute('UPDATE blocks SET refcount=refcount+1 WHERE id=?', + (block_id,)) + el.dirty = False + self.in_transit.remove((el.inode, el.blockno)) + except: + self.in_transit.remove((el.inode, el.blockno)) + raise + + + if el.blockno == 0: + self.db.execute('UPDATE inodes SET block_id=? WHERE id=?', (block_id, el.inode)) + else: + self.db.execute('INSERT OR REPLACE INTO inode_blocks (block_id, inode, blockno) ' + 'VALUES(?,?,?)', (block_id, el.inode, el.blockno)) + + # Check if we have to remove an old block + if not old_block_id: + log.debug('upload(%s): no old block, returning', el) + return el.size + + refcount = self.db.get_val('SELECT refcount FROM blocks WHERE id=?', (old_block_id,)) + if refcount > 1: + log.debug('upload(%s): decreased refcount for prev. block: %d', el, old_block_id) + self.db.execute('UPDATE blocks SET refcount=refcount-1 WHERE id=?', (old_block_id,)) + return el.size + + log.debug('upload(%s): removing prev. block %d', el, old_block_id) + old_obj_id = self.db.get_val('SELECT obj_id FROM blocks WHERE id=?', (old_block_id,)) + self.db.execute('DELETE FROM blocks WHERE id=?', (old_block_id,)) + refcount = self.db.get_val('SELECT refcount FROM objects WHERE id=?', (old_obj_id,)) + if refcount > 1: + log.debug('upload(%s): decreased refcount for prev. obj: %d', el, old_obj_id) + self.db.execute('UPDATE objects SET refcount=refcount-1 WHERE id=?', + (old_obj_id,)) + return el.size + + log.debug('upload(%s): removing object %d', el, old_obj_id) + self.db.execute('DELETE FROM objects WHERE id=?', (old_obj_id,)) + + while old_obj_id in self.in_transit: + log.debug('upload(%s): waiting for transfer of old object %d to complete', + el, old_obj_id) + self.wait() + + with lock_released: + if not self.removal_threads: + log.warn("upload(%s): no removal threads, removing synchronously", el) + self._do_removal(old_obj_id) + else: + log.debug('upload(%s): adding %d to removal queue', el, old_obj_id) + self.to_remove.put(old_obj_id) + + return el.size + - os.rmdir(self.cachedir) - log.debug('destroy: end') - - def get_bucket_size(self): - '''Return total size of the underlying bucket''' - - return self.bucket.get_size() - - def __len__(self): - '''Get number of objects in cache''' - return len(self.cache) + def transfer_in_progress(self): + '''Return True if there are any blocks in transit''' + + return len(self.in_transit) > 0 + def _removal_loop(self): + '''Process removal queue''' + + while True: + tmp = self.to_remove.get() + + if tmp is QuitSentinel: + break + + self._do_removal(tmp) + + def _do_removal(self, obj_id): + '''Remove object''' + + with self.bucket_pool() as bucket: + bucket.delete('s3ql_data_%d' % obj_id) + @contextmanager def get(self, inode, blockno): """Get file handle for block `blockno` of `inode` - This method releases the global lock. + This method releases the global lock, and the managed block + may do so as well. Note: if `get` and `remove` are called concurrently, then it is possible that a block that has been requested with `get` and passed to `remove` for deletion will not be deleted. """ - log.debug('get(inode=%d, block=%d): start', inode, blockno) + #log.debug('get(inode=%d, block=%d): start', inode, blockno) - if self.size > self.max_size or len(self.cache) > self.max_entries: + if self.size > self.max_size or len(self.entries) > self.max_entries: self.expire() - # Need to release global lock to acquire mlock to prevent deadlocking - lock.release() - with self.mlock(inode, blockno): - lock.acquire() + el = None + while el is None: + # Don't allow changing objects while they're being uploaded + if (inode, blockno) in self.in_transit: + log.debug('get(inode=%d, block=%d): inode/blockno in transit, waiting', + inode, blockno) + self.wait() + continue try: - el = self.cache[(inode, blockno)] + el = self.entries[(inode, blockno)] # Not in cache except KeyError: - filename = os.path.join(self.cachedir, - 'inode_%d_block_%d' % (inode, blockno)) + filename = os.path.join(self.path, '%d-%d' % (inode, blockno)) try: - obj_id = self.db.get_val("SELECT obj_id FROM blocks WHERE inode=? AND blockno=?", - (inode, blockno)) + if blockno == 0: + block_id = self.db.get_val('SELECT block_id FROM inodes ' + 'WHERE id=? AND block_id IS NOT NULL', (inode,)) + else: + block_id = self.db.get_val('SELECT block_id FROM inode_blocks ' + 'WHERE inode=? AND blockno=?', (inode, blockno)) # No corresponding object except NoSuchRowError: - log.debug('get(inode=%d, block=%d): creating new block', inode, blockno) - el = CacheEntry(inode, blockno, None, filename, "w+b") - + #log.debug('get(inode=%d, block=%d): creating new block', inode, blockno) + el = CacheEntry(inode, blockno, filename) + self.entries[(inode, blockno)] = el + # Need to download corresponding object else: - log.debug('get(inode=%d, block=%d): downloading block', inode, blockno) - el = CacheEntry(inode, blockno, obj_id, filename, "w+b") - with lock_released: - try: - if self.bucket.read_after_create_consistent(): - self.bucket.fetch_fh('s3ql_data_%d' % obj_id, el) - else: - retry_exc(300, [ NoSuchObject ], self.bucket.fetch_fh, - 's3ql_data_%d' % obj_id, el) - except: - os.unlink(filename) - raise - - # Writing will have set dirty flag - el.dirty = False - os.rename(el.name + '.d', el.name) + #log.debug('get(inode=%d, block=%d): downloading block', inode, blockno) + obj_id = self.db.get_val('SELECT obj_id FROM blocks WHERE id=?', (block_id,)) - self.size += os.fstat(el.fileno()).st_size - - self.cache[(inode, blockno)] = el - + if obj_id in self.in_transit: + log.debug('get(inode=%d, block=%d): object %d in transit, waiting', + inode, blockno, obj_id) + self.wait() + continue + + # We need to download + self.in_transit.add(obj_id) + log.debug('get(inode=%d, block=%d): downloading object %d..', + inode, blockno, obj_id) + try: + el = CacheEntry(inode, blockno, filename) + with lock_released: + with self.bucket_pool() as bucket: + with bucket.open_read('s3ql_data_%d' % obj_id) as fh: + shutil.copyfileobj(fh, el) + + # Note: We need to do this *before* releasing the global + # lock to notify other threads + self.entries[(inode, blockno)] = el + + # Writing will have set dirty flag + el.dirty = False + self.size += el.size + + except: + el.unlink() + raise + finally: + self.in_transit.remove(obj_id) + with lock_released: + self.transfer_completed.notify_all() + # In Cache else: - log.debug('get(inode=%d, block=%d): in cache', inode, blockno) - self.cache.to_head((inode, blockno)) + #log.debug('get(inode=%d, block=%d): in cache', inode, blockno) + self.entries.to_head((inode, blockno)) - el.last_access = time.time() - oldsize = os.fstat(el.fileno()).st_size + oldsize = el.size # Provide fh to caller try: - log.debug('get(inode=%d, block=%d): yield', inode, blockno) + #log.debug('get(inode=%d, block=%d): yield', inode, blockno) yield el finally: - # Update cachesize - el.flush() - newsize = os.fstat(el.fileno()).st_size - self.size += newsize - oldsize + # Update cachesize + self.size += el.size - oldsize - log.debug('get(inode=%d, block=%d): end', inode, blockno) + #log.debug('get(inode=%d, block=%d): end', inode, blockno) def expire(self): @@ -283,53 +594,61 @@ class BlockCache(object): log.debug('expire: start') - while (len(self.cache) > self.max_entries or - (len(self.cache) > 0 and self.size > self.max_size)): + did_nothing_count = 0 + while (len(self.entries) > self.max_entries or + (len(self.entries) > 0 and self.size > self.max_size)): need_size = self.size - self.max_size - need_entries = len(self.cache) - self.max_entries + need_entries = len(self.entries) - self.max_entries # Try to expire entries that are not dirty - for el in self.cache.values_rev(): + for el in self.entries.values_rev(): if el.dirty: - if (el.inode, el.blockno) not in self.upload_manager.in_transit: + if (el.inode, el.blockno) in self.in_transit: + log.debug('expire: %s is dirty, but already being uploaded', el) + continue + else: log.debug('expire: %s is dirty, trying to flush', el) break - else: - continue - - del self.cache[(el.inode, el.blockno)] - size = os.fstat(el.fileno()).st_size + + del self.entries[(el.inode, el.blockno)] el.close() - os.unlink(el.name) + el.unlink() need_entries -= 1 - self.size -= size - need_size -= size + self.size -= el.size + need_size -= el.size + did_nothing_count = 0 if need_size <= 0 and need_entries <= 0: break if need_size <= 0 and need_entries <= 0: break - # If nothing is being uploaded, try to upload just enough - if not self.upload_manager.upload_in_progress(): - for el in self.cache.values_rev(): + # Try to upload just enough + for el in self.entries.values_rev(): + if el.dirty and (el.inode, el.blockno) not in self.in_transit: log.debug('expire: uploading %s..', el) - if el.dirty and (el.inode, el.blockno) not in self.upload_manager.in_transit: - freed = self.upload_manager.add(el) # Releases global lock - need_size -= freed - else: - need_size -= os.fstat(el.fileno()).st_size - need_entries -= 1 - - if need_size <= 0 and need_entries <= 0: - break - + freed = self.upload(el) # Releases global lock + need_size -= freed + did_nothing_count = 0 + else: + log.debug('expire: %s can soon be expired..', el) + need_size -= el.size + need_entries -= 1 + + if need_size <= 0 and need_entries <= 0: + break + + did_nothing_count += 1 + if did_nothing_count > 50: + log.error('Problem in expire()') + break + # Wait for the next entry - log.debug('expire: waiting for upload threads..') - self.upload_manager.join_one() # Releases global lock - + log.debug('expire: waiting for transfer threads..') + self.wait() # Releases global lock + log.debug('expire: end') @@ -342,13 +661,12 @@ class BlockCache(object): This method releases the global lock. - Note: if `get` and `remove` are called concurrently, then it - is possible that a block that has been requested with `get` and - passed to `remove` for deletion will not be deleted. + Note: if `get` and `remove` are called concurrently, then it is possible + that a block that has been requested with `get` and passed to `remove` + for deletion will not be deleted. """ - log.debug('remove(inode=%d, start=%d, end=%s): start', - inode, start_no, end_no) + log.debug('remove(inode=%d, start=%d, end=%s): start', inode, start_no, end_no) if end_no is None: end_no = start_no + 1 @@ -356,126 +674,97 @@ class BlockCache(object): for blockno in range(start_no, end_no): # We can't use self.mlock here to prevent simultaneous retrieval # of the block with get(), because this could deadlock - if (inode, blockno) in self.cache: + if (inode, blockno) in self.entries: + log.debug('remove(inode=%d, blockno=%d): removing from cache', + inode, blockno) + # Type inference fails here #pylint: disable-msg=E1103 - el = self.cache.pop((inode, blockno)) + el = self.entries.pop((inode, blockno)) - self.size -= os.fstat(el.fileno()).st_size - el.close() - if el.dirty: - os.unlink(el.name + '.d') - else: - os.unlink(el.name) + self.size -= el.size + el.unlink() - if el.obj_id is None: - log.debug('remove(inode=%d, blockno=%d): block only in cache', - inode, blockno) - continue - - log.debug('remove(inode=%d, blockno=%d): block in cache and db', inode, blockno) - obj_id = el.obj_id + try: + if blockno == 0: + block_id = self.db.get_val('SELECT block_id FROM inodes ' + 'WHERE id=? AND block_id IS NOT NULL', + (inode,)) + else: + block_id = self.db.get_val('SELECT block_id FROM inode_blocks ' + 'WHERE inode=? AND blockno=?', (inode, blockno)) + except NoSuchRowError: + log.debug('remove(inode=%d, blockno=%d): block not in db', inode, blockno) + continue + # Detach inode from block + if blockno == 0: + self.db.execute('UPDATE inodes SET block_id=NULL WHERE id=?', (inode,)) else: - try: - obj_id = self.db.get_val('SELECT obj_id FROM blocks WHERE inode=? ' - 'AND blockno = ?', (inode, blockno)) - except NoSuchRowError: - log.debug('remove(inode=%d, blockno=%d): block does not exist', - inode, blockno) - continue - - log.debug('remove(inode=%d, blockno=%d): block only in db ', inode, blockno) - - self.db.execute('DELETE FROM blocks WHERE inode=? AND blockno=?', - (inode, blockno)) + self.db.execute('DELETE FROM inode_blocks WHERE inode=? AND blockno=?', + (inode, blockno)) + # Decrease block refcount + refcount = self.db.get_val('SELECT refcount FROM blocks WHERE id=?', (block_id,)) + if refcount > 1: + log.debug('remove(inode=%d, blockno=%d): decreasing refcount for block %d', + inode, blockno, block_id) + self.db.execute('UPDATE blocks SET refcount=refcount-1 WHERE id=?', + (block_id,)) + continue + + # Detach block from object + log.debug('remove(inode=%d, blockno=%d): deleting block %d', + inode, blockno, block_id) + obj_id = self.db.get_val('SELECT obj_id FROM blocks WHERE id=?', (block_id,)) + self.db.execute('DELETE FROM blocks WHERE id=?', (block_id,)) + + # Decrease object refcount refcount = self.db.get_val('SELECT refcount FROM objects WHERE id=?', (obj_id,)) if refcount > 1: log.debug('remove(inode=%d, blockno=%d): decreasing refcount for object %d', inode, blockno, obj_id) self.db.execute('UPDATE objects SET refcount=refcount-1 WHERE id=?', - (obj_id,)) - to_delete = False + (obj_id,)) else: - log.debug('remove(inode=%d, blockno=%d): deleting object %d', - inode, blockno, obj_id) + while obj_id in self.in_transit: + log.debug('remove(inode=%d, blockno=%d): waiting for transfer of ' + 'object %d to complete', inode, blockno, obj_id) + self.wait() self.db.execute('DELETE FROM objects WHERE id=?', (obj_id,)) - to_delete = True - - if to_delete: - try: - # Releases global lock: - self.removal_queue.add_thread(RemoveThread(obj_id, self.bucket, - (inode, blockno), - self.upload_manager)) - except EmbeddedException as exc: - exc = exc.exc_info[1] - if isinstance(exc, NoSuchObject): - log.warn('Backend seems to have lost object %s', exc.key) - self.encountered_errors = True - else: - raise - - log.debug('remove(inode=%d, start=%d, end=%s): end', - inode, start_no, end_no) + with lock_released: + if not self.removal_threads: + log.warn("remove(): no removal threads, removing synchronously") + self._do_removal(obj_id) + else: + self.to_remove.put(obj_id) + + log.debug('remove(inode=%d, start=%d, end=%s): end', inode, start_no, end_no) def flush(self, inode): """Flush buffers for `inode`""" - # Cache entries are automatically flushed after each read() - # and write() + # Cache entries are automatically flushed after each read() and write() pass def commit(self): - """Upload all dirty blocks + """Initiate upload of all dirty blocks - This method uploads all dirty blocks. The object itself may - still be in transit when the method returns, but the - blocks table is guaranteed to refer to the correct objects. + When the method returns, all blocks have been registered + in the database (but the actual uploads may still be + in progress). This method releases the global lock. """ - in_transit = set() - - for el in self.cache.itervalues(): - if not el.dirty: + for el in self.entries.itervalues(): + if not (el.dirty and (el.inode, el.blockno) not in self.in_transit): continue - if (el.inode, el.blockno) in self.upload_manager.in_transit: - if not el.modified_after_upload: - continue - - # We need to wait for the current upload to complete - in_transit.add(el) - else: - self.upload_manager.add(el) # Releases global lock - - while in_transit: - log.warn('commit(): in_transit: %s', in_transit) - self.upload_manager.join_one() - finished = in_transit.difference(self.upload_manager.in_transit) - in_transit = in_transit.intersection(self.upload_manager.in_transit) - - for el in finished: - # Object may no longer be dirty or already in transit - # if a different thread initiated the object while - # the global lock was released in a previous iteration. - if el.dirty: - continue - if (el.inode, el.blockno) in self.upload_manager.in_transit: - continue - - self.upload_manager.add(el) # Releases global lock - - + self.upload(el) # Releases global lock + def clear(self): - """Upload all dirty data and clear cache - - When the method returns, all blocks have been registered - in the database, but the actual uploads may still be - in progress. + """Clear cache This method releases the global lock. """ @@ -485,63 +774,11 @@ class BlockCache(object): self.max_entries = 0 self.expire() # Releases global lock self.max_entries = bak + log.debug('clear: end') + def __del__(self): - if len(self.cache) > 0: + if len(self.entries) > 0: raise RuntimeError("BlockCache instance was destroyed without calling destroy()!") -class CommitThread(ExceptionStoringThread): - ''' - Periodically upload dirty blocks. - - This class uses the llfuse global lock. When calling objects - passed in the constructor, the global lock is acquired first. - ''' - - - def __init__(self, bcache): - super(CommitThread, self).__init__() - self.bcache = bcache - self.stop_event = threading.Event() - self.name = 'CommitThread' - - def run_protected(self): - log.debug('CommitThread: start') - - while not self.stop_event.is_set(): - did_sth = False - stamp = time.time() - for el in self.bcache.cache.values_rev(): - if stamp - el.last_access < 10: - break - if (not el.dirty or - (el.inode, el.blockno) in self.bcache.upload_manager.in_transit): - continue - - # Acquire global lock to access UploadManager instance - with lock: - if (not el.dirty or # Object may have been accessed - (el.inode, el.blockno) in self.bcache.upload_manager.in_transit): - continue - self.bcache.upload_manager.add(el) - did_sth = True - - if self.stop_event.is_set(): - break - - if not did_sth: - self.stop_event.wait(5) - - log.debug('CommitThread: end') - - def stop(self): - '''Wait for thread to finish, raise any occurred exceptions. - - This method releases the global lock. - ''' - - self.stop_event.set() - with lock_released: - self.join_and_raise() -
\ No newline at end of file diff --git a/src/s3ql/cli/__init__.py b/src/s3ql/cli/__init__.py index 019cbde..f716dc8 100644 --- a/src/s3ql/cli/__init__.py +++ b/src/s3ql/cli/__init__.py @@ -3,7 +3,7 @@ __init__.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import diff --git a/src/s3ql/cli/adm.py b/src/s3ql/cli/adm.py index 1e991b0..c09fdae 100644 --- a/src/s3ql/cli/adm.py +++ b/src/s3ql/cli/adm.py @@ -3,30 +3,37 @@ adm.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import - -import logging -from s3ql.common import (get_backend, QuietError, unlock_bucket, - cycle_metadata, dump_metadata, restore_metadata, - setup_logging, get_bucket_home) -from s3ql.backends import s3 -from s3ql.parse_args import ArgumentParser -from s3ql import CURRENT_FS_REV +from Queue import Queue +from datetime import datetime as Datetime from getpass import getpass -import sys -from s3ql.backends.common import ChecksumError -import os +from s3ql import CURRENT_FS_REV +from s3ql.backends.common import (BetterBucket, get_bucket, NoSuchBucket, + ChecksumError, AbstractBucket, NoSuchObject) +from s3ql.backends.local import Bucket as LocalBucket, ObjectR, unescape, escape +from s3ql.common import (QuietError, restore_metadata, cycle_metadata, + dump_metadata, create_tables, setup_logging, get_bucket_cachedir) from s3ql.database import Connection -import tempfile -from datetime import datetime as Datetime -import textwrap +from s3ql.fsck import Fsck +from s3ql.parse_args import ArgumentParser +from threading import Thread +import ConfigParser +import cPickle as pickle +import errno +import hashlib +import logging +import os +import re import shutil import stat +import sys +import tempfile +import textwrap import time -import cPickle as pickle + log = logging.getLogger("adm") @@ -50,7 +57,7 @@ def parse_args(args): parents=[pparser]) subparsers.add_parser("upgrade", help="upgrade file system to newest revision", parents=[pparser]) - subparsers.add_parser("delete", help="completely delete a bucket with all contents", + subparsers.add_parser("clear", help="delete all S3QL data from the bucket", parents=[pparser]) subparsers.add_parser("download-metadata", help="Interactively download metadata backups. " @@ -59,14 +66,12 @@ def parse_args(args): parser.add_debug_modules() parser.add_quiet() - parser.add_homedir() + parser.add_log() + parser.add_authfile() + parser.add_cachedir() parser.add_version() - parser.add_ssl() options = parser.parse_args(args) - - if not os.path.exists(options.homedir): - os.mkdir(options.homedir, 0700) return options @@ -77,33 +82,30 @@ def main(args=None): args = sys.argv[1:] options = parse_args(args) - setup_logging(options, 'adm.log') - - with get_backend(options.storage_url, - options.homedir, options.ssl) as (conn, bucketname): - home = get_bucket_home(options.storage_url, options.homedir) + setup_logging(options) + + # Check if fs is mounted on this computer + # This is not foolproof but should prevent common mistakes + match = options.storage_url + ' /' + with open('/proc/mounts', 'r') as fh: + for line in fh: + if line.startswith(match): + raise QuietError('Can not work on mounted file system.') - if options.action == 'delete': - return delete_bucket(conn, bucketname, home) - - if not bucketname in conn: - raise QuietError("Bucket does not exist.") - - bucket = conn.get_bucket(bucketname) + if options.action == 'clear': + return clear(get_bucket(options, plain=True), + get_bucket_cachedir(options.storage_url, options.cachedir)) + + if options.action == 'upgrade': + return upgrade(get_possibly_old_bucket(options)) - try: - unlock_bucket(options.homedir, options.storage_url, bucket) - except ChecksumError: - raise QuietError('Checksum error - incorrect password?') - - if options.action == 'passphrase': - return change_passphrase(bucket) - - if options.action == 'upgrade': - return upgrade(bucket) + bucket = get_bucket(options) + + if options.action == 'passphrase': + return change_passphrase(bucket) - if options.action == 'download-metadata': - return download_metadata(bucket, options.storage_url) + if options.action == 'download-metadata': + return download_metadata(bucket, options.storage_url) def download_metadata(bucket, storage_url): @@ -136,35 +138,35 @@ def download_metadata(bucket, storage_url): log.info('Downloading %s...', name) - home = get_bucket_home(storage_url, '.') + cachepath = get_bucket_cachedir(storage_url, '.') for i in ('.db', '.params'): - if os.path.exists(home + i): - raise QuietError('%s already exists, aborting.' % home+i) + if os.path.exists(cachepath + i): + raise QuietError('%s already exists, aborting.' % cachepath+i) - fh = os.fdopen(os.open(home + '.db', os.O_RDWR | os.O_CREAT, - stat.S_IRUSR | stat.S_IWUSR), 'w+b') + os.close(os.open(cachepath + '.db', os.O_RDWR | os.O_CREAT, + stat.S_IRUSR | stat.S_IWUSR), 'w+b') param = bucket.lookup(name) try: - fh.close() - db = Connection(home + '.db') - fh = tempfile.TemporaryFile() - bucket.fetch_fh(name, fh) - fh.seek(0) + db = Connection(cachepath + '.db') log.info('Reading metadata...') + with bucket.open_read(name) as fh: + restore_metadata(fh, db) restore_metadata(fh, db) - fh.close() except: # Don't keep file if it doesn't contain anything sensible - os.unlink(home + '.db') + os.unlink(cachepath + '.db') raise - pickle.dump(param, open(home + '.params', 'wb'), 2) - + # Raise sequence number so that fsck.s3ql actually uses the + # downloaded backup + seq_nos = [ int(x[len('s3ql_seq_no_'):]) for x in bucket.list('s3ql_seq_no_') ] + param['seq_no'] = max(seq_nos) + 1 + pickle.dump(param, open(cachepath + '.params', 'wb'), 2) def change_passphrase(bucket): '''Change bucket passphrase''' - if 's3ql_passphrase' not in bucket: + if not isinstance(bucket, BetterBucket) and bucket.passphrase: raise QuietError('Bucket is not encrypted.') data_pw = bucket.passphrase @@ -178,9 +180,10 @@ def change_passphrase(bucket): bucket.passphrase = wrap_pw bucket['s3ql_passphrase'] = data_pw + bucket.passphrase = data_pw -def delete_bucket(conn, bucketname, home): - print('I am about to delete the bucket %s with ALL contents.' % bucketname, +def clear(bucket, cachepath): + print('I am about to delete the S3QL file system in %s.' % bucket, 'Please enter "yes" to continue.', '> ', sep='\n', end='') if sys.stdin.readline().strip().lower() != 'yes': @@ -189,28 +192,132 @@ def delete_bucket(conn, bucketname, home): log.info('Deleting...') for suffix in ('.db', '.params'): - name = home + suffix + name = cachepath + suffix if os.path.exists(name): os.unlink(name) - name = home + '-cache' + name = cachepath + '-cache' if os.path.exists(name): shutil.rmtree(name) + + bucket.clear() + + print('File system deleted.') + + if not bucket.is_get_consistent(): + log.info('Note: it may take a while for the removals to propagate through the backend.') + + +def get_possibly_old_bucket(options, plain=False): + '''Return factory producing bucket objects for given storage-url + + If *plain* is true, don't attempt to unlock and don't wrap into + BetterBucket. + ''' + + hit = re.match(r'^([a-zA-Z0-9]+)://(.+)$', options.storage_url) + if not hit: + raise QuietError('Unknown storage url: %s' % options.storage_url) + + backend_name = 's3ql.backends.%s' % hit.group(1) + bucket_name = hit.group(2) + try: + __import__(backend_name) + except ImportError: + raise QuietError('No such backend: %s' % hit.group(1)) + + bucket_class = getattr(sys.modules[backend_name], 'Bucket') + + # Read authfile + config = ConfigParser.SafeConfigParser() + if os.path.isfile(options.authfile): + mode = os.stat(options.authfile).st_mode + if mode & (stat.S_IRGRP | stat.S_IROTH): + raise QuietError("%s has insecure permissions, aborting." % options.authfile) + config.read(options.authfile) + + backend_login = None + backend_pw = None + bucket_passphrase = None + for section in config.sections(): + def getopt(name): + try: + return config.get(section, name) + except ConfigParser.NoOptionError: + return None + + pattern = getopt('storage-url') + + if not pattern or not options.storage_url.startswith(pattern): + continue - if bucketname in conn: - conn.delete_bucket(bucketname, recursive=True) + backend_login = backend_login or getopt('backend-login') + backend_pw = backend_pw or getopt('backend-password') + bucket_passphrase = bucket_passphrase or getopt('bucket-passphrase') + + if not backend_login and bucket_class.needs_login: + if sys.stdin.isatty(): + backend_login = getpass("Enter backend login: ") + else: + backend_login = sys.stdin.readline().rstrip() - print('Bucket deleted.') - if isinstance(conn, s3.Connection): - print('Note that it may take a while until the removal becomes visible.') + if not backend_pw and bucket_class.needs_login: + if sys.stdin.isatty(): + backend_pw = getpass("Enter backend password: ") + else: + backend_pw = sys.stdin.readline().rstrip() -def upgrade(bucket): + bucket = bucket_class(bucket_name, backend_login, backend_pw) + if bucket_class == LocalBucket and 's3ql_metadata.dat' in bucket: + bucket_class = LegacyLocalBucket + + if plain: + return lambda: bucket_class(bucket_name, backend_login, backend_pw) + + bucket = bucket_class(bucket_name, backend_login, backend_pw) + + try: + encrypted = 's3ql_passphrase' in bucket + except NoSuchBucket: + raise QuietError('Bucket %d does not exist' % bucket_name) + + if encrypted and not bucket_passphrase: + if sys.stdin.isatty(): + bucket_passphrase = getpass("Enter bucket encryption passphrase: ") + else: + bucket_passphrase = sys.stdin.readline().rstrip() + elif not encrypted: + bucket_passphrase = None + + if hasattr(options, 'compress'): + compress = options.compress + else: + compress = 'zlib' + + if not encrypted: + return lambda: BetterBucket(None, compress, + bucket_class(bucket_name, backend_login, backend_pw)) + + tmp_bucket = BetterBucket(bucket_passphrase, compress, bucket) + + try: + data_pw = tmp_bucket['s3ql_passphrase'] + except ChecksumError: + raise QuietError('Wrong bucket passphrase') + + return lambda: BetterBucket(data_pw, compress, + bucket_class(bucket_name, backend_login, backend_pw)) + +def upgrade(bucket_factory): '''Upgrade file system to newest revision''' + bucket = bucket_factory() + # Access to protected member #pylint: disable=W0212 log.info('Getting file system parameters..') seq_nos = [ int(x[len('s3ql_seq_no_'):]) for x in bucket.list('s3ql_seq_no_') ] + seq_no = max(seq_nos) if not seq_nos: raise QuietError(textwrap.dedent(''' File system revision too old to upgrade! @@ -219,13 +326,11 @@ def upgrade(bucket): revision before you can use this version to upgrade to the newest revision. ''')) - seq_no = max(seq_nos) param = bucket.lookup('s3ql_metadata') # Check for unclean shutdown if param['seq_no'] < seq_no: - if (bucket.read_after_write_consistent() and - bucket.read_after_delete_consistent()): + if bucket.is_get_consistent(): raise QuietError(textwrap.fill(textwrap.dedent('''\ It appears that the file system is still mounted somewhere else. If this is not the case, the file system may have not been unmounted cleanly and you should try @@ -257,7 +362,7 @@ def upgrade(bucket): elif param['revision'] >= CURRENT_FS_REV: print('File system already at most-recent revision') return - + print(textwrap.dedent(''' I am about to update the file system to the newest revision. You will not be able to access the file system with any older version @@ -266,6 +371,10 @@ def upgrade(bucket): You should make very sure that this command is not interrupted and that no one else tries to mount, fsck or upgrade the file system at the same time. + + When using the local backend, metadata and data of each stored + object will be merged into one file. This requires every object + to be rewritten and may thus take some time. ''')) print('Please enter "yes" to continue.', '> ', sep='\n', end='') @@ -273,43 +382,433 @@ def upgrade(bucket): if sys.stdin.readline().strip().lower() != 'yes': raise QuietError() - # Download metadata - log.info("Downloading & uncompressing metadata...") - dbfile = tempfile.NamedTemporaryFile() - db = Connection(dbfile.name, fast_mode=True) - fh = tempfile.TemporaryFile() - bucket.fetch_fh("s3ql_metadata", fh) - fh.seek(0) - log.info('Reading metadata...') - restore_metadata(fh, db) - fh.close() - log.info('Upgrading from revision %d to %d...', CURRENT_FS_REV - 1, CURRENT_FS_REV) - param['revision'] = CURRENT_FS_REV - for (id_, mode, target) in db.query('SELECT id, mode, target FROM inodes'): - if stat.S_ISLNK(mode): - db.execute('UPDATE inodes SET size=? WHERE id=?', - (len(target), id_)) + if 's3ql_hash_check_status' not in bucket: + if (isinstance(bucket, LegacyLocalBucket) or + (isinstance(bucket, BetterBucket) and + isinstance(bucket.bucket, LegacyLocalBucket))): + log.info('Merging metadata into datafiles...') + if isinstance(bucket, LegacyLocalBucket): + bucketpath = bucket.name + else: + bucketpath = bucket.bucket.name + i = 0 + for (path, _, filenames) in os.walk(bucketpath, topdown=True): + for name in filenames: + if not name.endswith('.meta'): + continue + + basename = os.path.splitext(name)[0] + if '=00' in basename: + raise RuntimeError("No, seriously, you tried to break things, didn't you?") + + with open(os.path.join(path, name), 'r+b') as dst: + dst.seek(0, os.SEEK_END) + with open(os.path.join(path, basename + '.dat'), 'rb') as src: + shutil.copyfileobj(src, dst) + + basename = basename.replace('#', '=23') + os.rename(os.path.join(path, name), + os.path.join(path, basename)) + os.unlink(os.path.join(path, basename + '.dat')) + + i += 1 + if i % 100 == 0: + log.info('..processed %d objects so far..', i) + + print("Merging complete. Please restart s3qladm upgrade to complete the upgrade.") + return + + # Download metadata + log.info("Downloading & uncompressing metadata...") + dbfile = tempfile.NamedTemporaryFile() + with tempfile.TemporaryFile() as tmp: + with bucket.open_read("s3ql_metadata") as fh: + shutil.copyfileobj(fh, tmp) + + db = Connection(dbfile.name, fast_mode=True) + tmp.seek(0) + restore_legacy_metadata(tmp, db) + + # Increase metadata sequence no + param['seq_no'] += 1 + bucket['s3ql_seq_no_%d' % param['seq_no']] = 'Empty' + for i in seq_nos: + if i < param['seq_no'] - 5: + del bucket['s3ql_seq_no_%d' % i ] - # Increase metadata sequence no + log.info("Uploading database..") + cycle_metadata(bucket) + param['last-modified'] = time.time() - time.timezone + with bucket.open_write("s3ql_metadata", param) as fh: + dump_metadata(fh, db) + + else: + log.info("Downloading & uncompressing metadata...") + dbfile = tempfile.NamedTemporaryFile() + with tempfile.TemporaryFile() as tmp: + with bucket.open_read("s3ql_metadata") as fh: + shutil.copyfileobj(fh, tmp) + + db = Connection(dbfile.name, fast_mode=True) + tmp.seek(0) + restore_metadata(tmp, db) + + print(textwrap.dedent(''' + The following process may take a long time, but can be interrupted + with Ctrl-C and resumed from this point by calling `s3qladm upgrade` + again. Please see Changes.txt for why this is necessary. + ''')) + + if 's3ql_hash_check_status' not in bucket: + log.info("Starting hash verification..") + start_obj = 0 + bucket['s3ql_hash_check_status'] = '%d' % start_obj + else: + start_obj = int(bucket['s3ql_hash_check_status']) + log.info("Resuming hash verification with object %d..", start_obj) + + try: + total = db.get_val('SELECT COUNT(id) FROM objects WHERE id > ?', (start_obj,)) + i = 0 + queue = Queue(1) + queue.error = None + threads = [] + if (isinstance(bucket, LocalBucket) or + (isinstance(bucket, BetterBucket) and + isinstance(bucket.bucket,LocalBucket))): + thread_count = 1 + else: + thread_count = 25 + + for _ in range(thread_count): + t = Thread(target=check_hash, args=(queue, bucket_factory())) + t.daemon = True + t.start() + threads.append(t) + + for tmp in db.query('SELECT obj_id, hash FROM blocks JOIN objects ' + 'ON obj_id == objects.id WHERE obj_id > ? ' + 'ORDER BY obj_id ASC', (start_obj,)): + queue.put(tmp) + i += 1 + if i % 100 == 0: + log.info(' ..checked %d/%d objects..', i, total) + + if queue.error: + raise queue.error[0], queue.error[1], queue.error[2] + + for t in threads: + queue.put(None) + for t in threads: + t.join() + + except KeyboardInterrupt: + log.info("Storing verification status...") + for t in threads: + queue.put(None) + for t in threads: + t.join() + bucket['s3ql_hash_check_status'] = '%d' % tmp[0] + raise QuietError('Aborting..') + + except: + log.info("Storing verification status...") + bucket['s3ql_hash_check_status'] = '%d' % tmp[0] + raise + + log.info('Running fsck...') + bucket['s3ql_hash_check_status'] = '%d' % tmp[0] + fsck = Fsck(tempfile.mkdtemp(), bucket, param, db) + fsck.check() + + if fsck.uncorrectable_errors: + raise QuietError("Uncorrectable errors found, aborting.") + + param['revision'] = CURRENT_FS_REV param['seq_no'] += 1 - bucket.store('s3ql_seq_no_%d' % param['seq_no'], 'Empty') - for i in seq_nos: - if i < param['seq_no'] - 5: - del bucket['s3ql_seq_no_%d' % i ] - - # Upload metadata - fh = tempfile.TemporaryFile() - dump_metadata(fh, db) - fh.seek(0) + bucket['s3ql_seq_no_%d' % param['seq_no']] = 'Empty' + log.info("Uploading database..") cycle_metadata(bucket) param['last-modified'] = time.time() - time.timezone - bucket.store_fh("s3ql_metadata", fh, param) - fh.close() + with bucket.open_write("s3ql_metadata", param) as fh: + dump_metadata(fh, db) + + +def check_hash(queue, bucket): + + try: + while True: + tmp = queue.get() + if tmp is None: + break + + (obj_id, hash_) = tmp + + sha = hashlib.sha256() + try: + with bucket.open_read("s3ql_data_%d" % obj_id) as fh: + while True: + buf = fh.read(128*1024) + if not buf: + break + sha.update(buf) + except ChecksumError: + log.warn('Object %d corrupted! Deleting..', obj_id) + bucket.delete('s3ql_data_%d' % obj_id) + + except NoSuchObject: + log.warn('Object %d seems to have disappeared', obj_id) + + else: + if sha.digest() != hash_: + log.warn('Object %d corrupted! Deleting..', obj_id) + bucket.delete('s3ql_data_%d' % obj_id) + except: + queue.error = sys.exc_info() + queue.get() + + +def restore_legacy_metadata(ifh, conn): + unpickler = pickle.Unpickler(ifh) + (data_start, to_dump, sizes, columns) = unpickler.load() + ifh.seek(data_start) + create_tables(conn) + create_legacy_tables(conn) + for (table, _) in to_dump: + log.info('Loading %s', table) + col_str = ', '.join(columns[table]) + val_str = ', '.join('?' for _ in columns[table]) + if table in ('inodes', 'blocks', 'objects', 'contents'): + sql_str = 'INSERT INTO leg_%s (%s) VALUES(%s)' % (table, col_str, val_str) + else: + sql_str = 'INSERT INTO %s (%s) VALUES(%s)' % (table, col_str, val_str) + for _ in xrange(sizes[table]): + buf = unpickler.load() + for row in buf: + conn.execute(sql_str, row) + + # Create a block for each object + conn.execute(''' + INSERT INTO blocks (id, hash, refcount, obj_id, size) + SELECT id, hash, refcount, id, size FROM leg_objects + ''') + conn.execute(''' + INSERT INTO objects (id, refcount, compr_size) + SELECT id, 1, compr_size FROM leg_objects + ''') + conn.execute('DROP TABLE leg_objects') + + # Create new inode_blocks table for inodes with multiple blocks + conn.execute(''' + CREATE TEMP TABLE multi_block_inodes AS + SELECT inode FROM leg_blocks + GROUP BY inode HAVING COUNT(inode) > 1 + ''') + conn.execute(''' + INSERT INTO inode_blocks (inode, blockno, block_id) + SELECT inode, blockno, obj_id + FROM leg_blocks JOIN multi_block_inodes USING(inode) + ''') + + # Create new inodes table for inodes with multiple blocks + conn.execute(''' + INSERT INTO inodes (id, uid, gid, mode, mtime, atime, ctime, + refcount, size, rdev, locked, block_id) + SELECT id, uid, gid, mode, mtime, atime, ctime, + refcount, size, rdev, locked, NULL + FROM leg_inodes JOIN multi_block_inodes ON inode == id + ''') + + # Add inodes with just one block or no block + conn.execute(''' + INSERT INTO inodes (id, uid, gid, mode, mtime, atime, ctime, + refcount, size, rdev, locked, block_id) + SELECT id, uid, gid, mode, mtime, atime, ctime, + refcount, size, rdev, locked, obj_id + FROM leg_inodes LEFT JOIN leg_blocks ON leg_inodes.id == leg_blocks.inode + GROUP BY leg_inodes.id HAVING COUNT(leg_inodes.id) <= 1 + ''') + + conn.execute(''' + INSERT INTO symlink_targets (inode, target) + SELECT id, target FROM leg_inodes WHERE target IS NOT NULL + ''') + + conn.execute('DROP TABLE leg_inodes') + conn.execute('DROP TABLE leg_blocks') + + # Sort out names + conn.execute(''' + INSERT INTO names (name, refcount) + SELECT name, COUNT(name) FROM leg_contents GROUP BY name + ''') + conn.execute(''' + INSERT INTO contents (name_id, inode, parent_inode) + SELECT names.id, inode, parent_inode + FROM leg_contents JOIN names ON leg_contents.name == names.name + ''') + conn.execute('DROP TABLE leg_contents') + + conn.execute('ANALYZE') + +def create_legacy_tables(conn): + conn.execute(""" + CREATE TABLE leg_inodes ( + id INTEGER PRIMARY KEY, + uid INT NOT NULL, + gid INT NOT NULL, + mode INT NOT NULL, + mtime REAL NOT NULL, + atime REAL NOT NULL, + ctime REAL NOT NULL, + refcount INT NOT NULL, + target BLOB(256) , + size INT NOT NULL DEFAULT 0, + rdev INT NOT NULL DEFAULT 0, + locked BOOLEAN NOT NULL DEFAULT 0 + ) + """) + conn.execute(""" + CREATE TABLE leg_objects ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + refcount INT NOT NULL, + hash BLOB(16) UNIQUE, + size INT NOT NULL, + compr_size INT + )""") + conn.execute(""" + CREATE TABLE leg_blocks ( + inode INTEGER NOT NULL REFERENCES leg_inodes(id), + blockno INT NOT NULL, + obj_id INTEGER NOT NULL REFERENCES leg_objects(id), + PRIMARY KEY (inode, blockno) + )""") + conn.execute(""" + CREATE TABLE leg_contents ( + rowid INTEGER PRIMARY KEY AUTOINCREMENT, + name BLOB(256) NOT NULL, + inode INT NOT NULL REFERENCES leg_inodes(id), + parent_inode INT NOT NULL REFERENCES leg_inodes(id), + + UNIQUE (name, parent_inode) + )""") + +def create_legacy_indices(conn): + conn.execute('CREATE INDEX ix_leg_contents_parent_inode ON contents(parent_inode)') + conn.execute('CREATE INDEX ix_leg_contents_inode ON contents(inode)') + conn.execute('CREATE INDEX ix_leg_objects_hash ON objects(hash)') + conn.execute('CREATE INDEX ix_leg_blocks_obj_id ON blocks(obj_id)') + conn.execute('CREATE INDEX ix_leg_blocks_inode ON blocks(inode)') +class LegacyLocalBucket(AbstractBucket): + needs_login = False + + def __init__(self, name, backend_login, backend_pw): #IGNORE:W0613 + super(LegacyLocalBucket, self).__init__() + self.name = name + if not os.path.exists(name): + raise NoSuchBucket(name) + + def lookup(self, key): + path = self._key_to_path(key) + '.meta' + try: + with open(path, 'rb') as src: + return pickle.load(src) + except IOError as exc: + if exc.errno == errno.ENOENT: + raise NoSuchObject(key) + else: + raise + + def open_read(self, key): + path = self._key_to_path(key) + try: + fh = ObjectR(path + '.dat') + except IOError as exc: + if exc.errno == errno.ENOENT: + raise NoSuchObject(key) + else: + raise + + fh.metadata = pickle.load(open(path + '.meta', 'rb')) + + return fh + + def open_write(self, key, metadata=None): + raise RuntimeError('Not implemented') + + def clear(self): + raise RuntimeError('Not implemented') + + def copy(self, src, dest): + raise RuntimeError('Not implemented') + + def contains(self, key): + path = self._key_to_path(key) + try: + os.lstat(path + '.meta') + except OSError as exc: + if exc.errno == errno.ENOENT: + return False + raise + return True + + def delete(self, key, force=False): + raise RuntimeError('Not implemented') + + def list(self, prefix=''): + if prefix: + base = os.path.dirname(self._key_to_path(prefix)) + else: + base = self.name + + for (path, dirnames, filenames) in os.walk(base, topdown=True): + + # Do not look in wrong directories + if prefix: + rpath = path[len(self.name):] # path relative to base + prefix_l = ''.join(rpath.split('/')) + + dirs_to_walk = list() + for name in dirnames: + prefix_ll = unescape(prefix_l + name) + if prefix_ll.startswith(prefix[:len(prefix_ll)]): + dirs_to_walk.append(name) + dirnames[:] = dirs_to_walk + + for name in filenames: + key = unescape(name) + + if not prefix or key.startswith(prefix): + if key.endswith('.meta'): + yield key[:-5] + + def _key_to_path(self, key): + key = escape(key) + + if not key.startswith('s3ql_data_'): + return os.path.join(self.name, key) + + no = key[10:] + path = [ self.name, 's3ql_data_'] + for i in range(0, len(no), 3): + path.append(no[:i]) + path.append(key) + + return os.path.join(*path) + + def is_get_consistent(self): + return True + + def is_list_create_consistent(self): + return True + + if __name__ == '__main__': main(sys.argv[1:]) + diff --git a/src/s3ql/cli/cp.py b/src/s3ql/cli/cp.py index 51e9dbc..c1820e2 100644 --- a/src/s3ql/cli/cp.py +++ b/src/s3ql/cli/cp.py @@ -3,7 +3,7 @@ cp.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import diff --git a/src/s3ql/cli/ctrl.py b/src/s3ql/cli/ctrl.py index 637e321..e975fc2 100644 --- a/src/s3ql/cli/ctrl.py +++ b/src/s3ql/cli/ctrl.py @@ -3,7 +3,7 @@ ctrl.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2010 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import diff --git a/src/s3ql/cli/fsck.py b/src/s3ql/cli/fsck.py index c1bc04b..89292d9 100644 --- a/src/s3ql/cli/fsck.py +++ b/src/s3ql/cli/fsck.py @@ -3,28 +3,27 @@ fsck.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import - -import os -import stat -import time -from s3ql.common import (get_bucket_home, cycle_metadata, setup_logging, - unlock_bucket, QuietError, get_backend, get_seq_no, - restore_metadata, dump_metadata) -from s3ql.parse_args import ArgumentParser from s3ql import CURRENT_FS_REV +from s3ql.backends.common import get_bucket +from s3ql.common import (get_bucket_cachedir, cycle_metadata, setup_logging, + QuietError, get_seq_no, restore_metadata, dump_metadata) from s3ql.database import Connection -import logging from s3ql.fsck import Fsck -from s3ql.backends.common import ChecksumError -import sys +from s3ql.parse_args import ArgumentParser import apsw -import tempfile import cPickle as pickle +import logging +import os +import shutil +import stat +import sys +import tempfile import textwrap +import time log = logging.getLogger("fsck") @@ -33,12 +32,13 @@ def parse_args(args): parser = ArgumentParser( description="Checks and repairs an S3QL filesystem.") - parser.add_homedir() + parser.add_log('~/.s3ql/fsck.log') + parser.add_cachedir() + parser.add_authfile() parser.add_debug_modules() parser.add_quiet() parser.add_version() parser.add_storage_url() - parser.add_ssl() parser.add_argument("--batch", action="store_true", default=False, help="If user input is required, exit without prompting.") @@ -47,9 +47,6 @@ def parse_args(args): options = parser.parse_args(args) - if not os.path.exists(options.homedir): - os.mkdir(options.homedir, 0700) - return options @@ -59,155 +56,143 @@ def main(args=None): args = sys.argv[1:] options = parse_args(args) - setup_logging(options, 'fsck.log') + setup_logging(options) - with get_backend(options.storage_url, options.homedir, - options.ssl) as (conn, bucketname): - - # Check if fs is mounted on this computer - # This is not foolproof but should prevent common mistakes - match = options.storage_url + ' /' - with open('/proc/mounts', 'r') as fh: - for line in fh: - if line.startswith(match): - raise QuietError('Can not check mounted file system.') - - if not bucketname in conn: - raise QuietError("Bucket does not exist.") - bucket = conn.get_bucket(bucketname) - - try: - unlock_bucket(options.homedir, options.storage_url, bucket) - except ChecksumError: - raise QuietError('Checksum error - incorrect password?') - - home = get_bucket_home(options.storage_url, options.homedir) - seq_no = get_seq_no(bucket) - param_remote = bucket.lookup('s3ql_metadata') - db = None - - if os.path.exists(home + '.params'): - assert os.path.exists(home + '.db') - param = pickle.load(open(home + '.params', 'rb')) - if param['seq_no'] < seq_no: - log.info('Ignoring locally cached metadata (outdated).') - param = bucket.lookup('s3ql_metadata') - else: - log.info('Using cached metadata.') - db = Connection(home + '.db') - assert not os.path.exists(home + '-cache') or param['needs_fsck'] - - if param_remote['seq_no'] != param['seq_no']: - log.warn('Remote metadata is outdated.') - param['needs_fsck'] = True - - else: - param = param_remote - assert not os.path.exists(home + '-cache') - # .db might exist if mount.s3ql is killed at exactly the right instant - # and should just be ignored. - - # Check revision - if param['revision'] < CURRENT_FS_REV: - raise QuietError('File system revision too old, please run `s3qladm upgrade` first.') - elif param['revision'] > CURRENT_FS_REV: - raise QuietError('File system revision too new, please update your ' - 'S3QL installation.') - + # Check if fs is mounted on this computer + # This is not foolproof but should prevent common mistakes + match = options.storage_url + ' /' + with open('/proc/mounts', 'r') as fh: + for line in fh: + if line.startswith(match): + raise QuietError('Can not check mounted file system.') + + + bucket = get_bucket(options) + + cachepath = get_bucket_cachedir(options.storage_url, options.cachedir) + seq_no = get_seq_no(bucket) + param_remote = bucket.lookup('s3ql_metadata') + db = None + + if os.path.exists(cachepath + '.params'): + assert os.path.exists(cachepath + '.db') + param = pickle.load(open(cachepath + '.params', 'rb')) if param['seq_no'] < seq_no: - if (bucket.read_after_write_consistent() and - bucket.read_after_delete_consistent()): - print(textwrap.fill(textwrap.dedent('''\ - Up to date metadata is not available. Probably the file system has not - been properly unmounted and you should try to run fsck on the computer - where the file system has been mounted most recently. - '''))) - else: - print(textwrap.fill(textwrap.dedent('''\ - Up to date metadata is not available. Either the file system has not - been unmounted cleanly or the data has not yet propagated through the backend. - In the later case, waiting for a while should fix the problem, in - the former case you should try to run fsck on the computer where - the file system has been mounted most recently - '''))) - - print('Enter "continue" to use the outdated data anyway:', - '> ', sep='\n', end='') - if options.batch: - raise QuietError('(in batch mode, exiting)') - if sys.stdin.readline().strip() != 'continue': - raise QuietError() - - param['seq_no'] = seq_no + log.info('Ignoring locally cached metadata (outdated).') + param = bucket.lookup('s3ql_metadata') + else: + log.info('Using cached metadata.') + db = Connection(cachepath + '.db') + assert not os.path.exists(cachepath + '-cache') or param['needs_fsck'] + + if param_remote['seq_no'] != param['seq_no']: + log.warn('Remote metadata is outdated.') param['needs_fsck'] = True - - - if (not param['needs_fsck'] - and ((time.time() - time.timezone) - param['last_fsck']) - < 60 * 60 * 24 * 31): # last check more than 1 month ago - if options.force: - log.info('File system seems clean, checking anyway.') - else: - log.info('File system is marked as clean. Use --force to force checking.') - return - - # If using local metadata, check consistency - if db: - log.info('Checking DB integrity...') - try: - # get_list may raise CorruptError itself - res = db.get_list('PRAGMA integrity_check(20)') - if res[0][0] != u'ok': - log.error('\n'.join(x[0] for x in res )) - raise apsw.CorruptError() - except apsw.CorruptError: - raise QuietError('Local metadata is corrupted. Remove or repair the following ' - 'files manually and re-run fsck:\n' - + home + '.db (corrupted)\n' - + home + '.param (intact)') + + else: + param = param_remote + assert not os.path.exists(cachepath + '-cache') + # .db might exist if mount.s3ql is killed at exactly the right instant + # and should just be ignored. + + # Check revision + if param['revision'] < CURRENT_FS_REV: + raise QuietError('File system revision too old, please run `s3qladm upgrade` first.') + elif param['revision'] > CURRENT_FS_REV: + raise QuietError('File system revision too new, please update your ' + 'S3QL installation.') + + if param['seq_no'] < seq_no: + if bucket.is_get_consistent(): + print(textwrap.fill(textwrap.dedent('''\ + Up to date metadata is not available. Probably the file system has not + been properly unmounted and you should try to run fsck on the computer + where the file system has been mounted most recently. + '''))) else: - log.info("Downloading & uncompressing metadata...") - fh = tempfile.TemporaryFile() - bucket.fetch_fh("s3ql_metadata", fh) - os.close(os.open(home + '.db.tmp', os.O_RDWR | os.O_CREAT | os.O_TRUNC, - stat.S_IRUSR | stat.S_IWUSR)) - db = Connection(home + '.db.tmp', fast_mode=True) - fh.seek(0) - log.info('Reading metadata...') - restore_metadata(fh, db) - fh.close() - db.close() - os.rename(home + '.db.tmp', home + '.db') - db = Connection(home + '.db') - - # Increase metadata sequence no - param['seq_no'] += 1 + print(textwrap.fill(textwrap.dedent('''\ + Up to date metadata is not available. Either the file system has not + been unmounted cleanly or the data has not yet propagated through the backend. + In the later case, waiting for a while should fix the problem, in + the former case you should try to run fsck on the computer where + the file system has been mounted most recently + '''))) + + print('Enter "continue" to use the outdated data anyway:', + '> ', sep='\n', end='') + if options.batch: + raise QuietError('(in batch mode, exiting)') + if sys.stdin.readline().strip() != 'continue': + raise QuietError() + + param['seq_no'] = seq_no param['needs_fsck'] = True - bucket.store('s3ql_seq_no_%d' % param['seq_no'], 'Empty') - pickle.dump(param, open(home + '.params', 'wb'), 2) - - fsck = Fsck(home + '-cache', bucket, param, db) - fsck.check() - - if fsck.uncorrectable_errors: - raise QuietError("Uncorrectable errors found, aborting.") - - if os.path.exists(home + '-cache'): - os.rmdir(home + '-cache') + + + if (not param['needs_fsck'] + and ((time.time() - time.timezone) - param['last_fsck']) + < 60 * 60 * 24 * 31): # last check more than 1 month ago + if options.force: + log.info('File system seems clean, checking anyway.') + else: + log.info('File system is marked as clean. Use --force to force checking.') + return + + # If using local metadata, check consistency + if db: + log.info('Checking DB integrity...') + try: + # get_list may raise CorruptError itself + res = db.get_list('PRAGMA integrity_check(20)') + if res[0][0] != u'ok': + log.error('\n'.join(x[0] for x in res )) + raise apsw.CorruptError() + except apsw.CorruptError: + raise QuietError('Local metadata is corrupted. Remove or repair the following ' + 'files manually and re-run fsck:\n' + + cachepath + '.db (corrupted)\n' + + cachepath + '.param (intact)') + else: + log.info("Downloading & uncompressing metadata...") + os.close(os.open(cachepath + '.db.tmp', os.O_RDWR | os.O_CREAT | os.O_TRUNC, + stat.S_IRUSR | stat.S_IWUSR)) + db = Connection(cachepath + '.db.tmp', fast_mode=True) + with bucket.open_read("s3ql_metadata") as fh: + restore_metadata(fh, db) + db.close() + os.rename(cachepath + '.db.tmp', cachepath + '.db') + db = Connection(cachepath + '.db') + + # Increase metadata sequence no + param['seq_no'] += 1 + param['needs_fsck'] = True + bucket['s3ql_seq_no_%d' % param['seq_no']] = 'Empty' + pickle.dump(param, open(cachepath + '.params', 'wb'), 2) + + fsck = Fsck(cachepath + '-cache', bucket, param, db) + fsck.check() + + if fsck.uncorrectable_errors: + raise QuietError("Uncorrectable errors found, aborting.") + + if os.path.exists(cachepath + '-cache'): + os.rmdir(cachepath + '-cache') + + log.info('Saving metadata...') + fh = tempfile.TemporaryFile() + dump_metadata(fh, db) - log.info('Saving metadata...') - fh = tempfile.TemporaryFile() - dump_metadata(fh, db) - - log.info("Compressing & uploading metadata..") - cycle_metadata(bucket) + log.info("Compressing & uploading metadata..") + cycle_metadata(bucket) + fh.seek(0) + param['needs_fsck'] = False + param['last_fsck'] = time.time() - time.timezone + param['last-modified'] = time.time() - time.timezone + with bucket.open_write("s3ql_metadata", param) as dst: fh.seek(0) - param['needs_fsck'] = False - param['last_fsck'] = time.time() - time.timezone - param['last-modified'] = time.time() - time.timezone - bucket.store_fh("s3ql_metadata", fh, param) - fh.close() - pickle.dump(param, open(home + '.params', 'wb'), 2) + shutil.copyfileobj(fh, dst) + fh.close() + pickle.dump(param, open(cachepath + '.params', 'wb'), 2) db.execute('ANALYZE') db.execute('VACUUM') diff --git a/src/s3ql/cli/lock.py b/src/s3ql/cli/lock.py index 75c01d1..683d458 100644 --- a/src/s3ql/cli/lock.py +++ b/src/s3ql/cli/lock.py @@ -3,7 +3,7 @@ lock.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import diff --git a/src/s3ql/cli/mkfs.py b/src/s3ql/cli/mkfs.py index 0f4037b..568d4c4 100644 --- a/src/s3ql/cli/mkfs.py +++ b/src/s3ql/cli/mkfs.py @@ -3,27 +3,24 @@ mkfs.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import - -import sys -import os from getpass import getpass -import shutil -import logging -import cPickle as pickle from s3ql import CURRENT_FS_REV -from s3ql.common import (get_backend, get_bucket_home, setup_logging, - QuietError, dump_metadata, create_tables, - init_tables) -from s3ql.parse_args import ArgumentParser +from s3ql.backends.common import get_bucket, BetterBucket +from s3ql.common import (get_bucket_cachedir, setup_logging, QuietError, + dump_metadata, create_tables, init_tables) from s3ql.database import Connection -from s3ql.backends.boto.s3.connection import Location -from s3ql.backends import s3 +from s3ql.parse_args import ArgumentParser +import cPickle as pickle +import logging +import os +import shutil +import sys import time -import tempfile + log = logging.getLogger("mkfs") @@ -32,18 +29,13 @@ def parse_args(args): parser = ArgumentParser( description="Initializes an S3QL file system") - parser.add_homedir() + parser.add_cachedir() + parser.add_authfile() parser.add_debug_modules() parser.add_quiet() parser.add_version() parser.add_storage_url() - parser.add_ssl() - parser.add_argument("--s3-location", default='EU', metavar='<name>', - choices=('EU', 'us-west-1', 'us-standard', 'ap-southeast-1'), - help="Storage location for new S3 buckets. Allowed values: `EU`, " - '`us-west-1`, `ap-southeast-1`, or `us-standard`. ' - '(default: %(default)s)') parser.add_argument("-L", default='', help="Filesystem label", dest="label", metavar='<name>',) parser.add_argument("--blocksize", type=int, default=10240, metavar='<size>', @@ -54,12 +46,6 @@ def parse_args(args): help="Overwrite any existing data.") options = parser.parse_args(args) - - if options.s3_location == 'us-standard': - options.s3_location = Location.DEFAULT - - if not os.path.exists(options.homedir): - os.mkdir(options.homedir, 0700) return options @@ -70,77 +56,74 @@ def main(args=None): options = parse_args(args) setup_logging(options) - - with get_backend(options.storage_url, options.homedir, - options.ssl) as (conn, bucketname): - if conn.bucket_exists(bucketname): - if not options.force: - raise QuietError("Bucket already exists! Use --force to overwrite") + + plain_bucket = get_bucket(options, plain=True) + + if 's3ql_metadata' in plain_bucket: + if not options.force: + raise QuietError("Found existing file system! Use --force to overwrite") - bucket = conn.get_bucket(bucketname) - log.info('Bucket already exists. Purging old file system data..') - if not bucket.read_after_delete_consistent(): - log.info('Please note that the new file system may appear inconsistent\n' - 'for a while until the removals have propagated through the backend.') - bucket.clear() + log.info('Purging existing file system data..') + plain_bucket.clear() + if not plain_bucket.is_get_consistent(): + log.info('Please note that the new file system may appear inconsistent\n' + 'for a while until the removals have propagated through the backend.') - elif isinstance(conn, s3.Connection): - bucket = conn.create_bucket(bucketname, location=options.s3_location) + if not options.plain: + if sys.stdin.isatty(): + wrap_pw = getpass("Enter encryption password: ") + if not wrap_pw == getpass("Confirm encryption password: "): + raise QuietError("Passwords don't match.") else: - bucket = conn.create_bucket(bucketname) - - if not options.plain: - if sys.stdin.isatty(): - wrap_pw = getpass("Enter encryption password: ") - if not wrap_pw == getpass("Confirm encryption password: "): - raise QuietError("Passwords don't match.") - else: - wrap_pw = sys.stdin.readline().rstrip() - - # Generate data encryption passphrase - log.info('Generating random encryption key...') - fh = open('/dev/urandom', "rb", 0) # No buffering - data_pw = fh.read(32) - fh.close() - - bucket.passphrase = wrap_pw - bucket['s3ql_passphrase'] = data_pw - bucket.passphrase = data_pw - - # Setup database - home = get_bucket_home(options.storage_url, options.homedir) - - # There can't be a corresponding bucket, so we can safely delete - # these files. - if os.path.exists(home + '.db'): - os.unlink(home + '.db') - if os.path.exists(home + '-cache'): - shutil.rmtree(home + '-cache') - - log.info('Creating metadata tables...') - db = Connection(home + '.db') - create_tables(db) - init_tables(db) - - param = dict() - param['revision'] = CURRENT_FS_REV - param['seq_no'] = 0 - param['label'] = options.label - param['blocksize'] = options.blocksize * 1024 - param['needs_fsck'] = False - param['last_fsck'] = time.time() - time.timezone - param['last-modified'] = time.time() - time.timezone - bucket.store('s3ql_seq_no_%d' % param['seq_no'], 'Empty') - - log.info('Saving metadata...') - fh = tempfile.TemporaryFile() - dump_metadata(fh, db) - - log.info("Compressing & uploading metadata..") - fh.seek(0) - bucket.store_fh("s3ql_metadata", fh, param) + wrap_pw = sys.stdin.readline().rstrip() + + # Generate data encryption passphrase + log.info('Generating random encryption key...') + fh = open('/dev/urandom', "rb", 0) # No buffering + data_pw = fh.read(32) fh.close() - pickle.dump(param, open(home + '.params', 'wb'), 2) + + bucket = BetterBucket(wrap_pw, 'bzip2', plain_bucket) + bucket['s3ql_passphrase'] = data_pw + else: + data_pw = None + + bucket = BetterBucket(data_pw, 'bzip2', plain_bucket) + + # Setup database + cachepath = get_bucket_cachedir(options.storage_url, options.cachedir) + + # There can't be a corresponding bucket, so we can safely delete + # these files. + if os.path.exists(cachepath + '.db'): + os.unlink(cachepath + '.db') + if os.path.exists(cachepath + '-cache'): + shutil.rmtree(cachepath + '-cache') + + log.info('Creating metadata tables...') + db = Connection(cachepath + '.db') + create_tables(db) + init_tables(db) + + param = dict() + param['revision'] = CURRENT_FS_REV + param['seq_no'] = 1 + param['label'] = options.label + param['blocksize'] = options.blocksize * 1024 + param['needs_fsck'] = False + param['last_fsck'] = time.time() - time.timezone + param['last-modified'] = time.time() - time.timezone + + # This indicates that the convert_legacy_metadata() stuff + # in BetterBucket is not required for this file system. + param['bucket_revision'] = 1 + + bucket.store('s3ql_seq_no_%d' % param['seq_no'], 'Empty') + + log.info('Uploading metadata...') + with bucket.open_write('s3ql_metadata', param) as fh: + dump_metadata(fh, db) + pickle.dump(param, open(cachepath + '.params', 'wb'), 2) if __name__ == '__main__': diff --git a/src/s3ql/cli/mount.py b/src/s3ql/cli/mount.py index 65cd63b..6cb3f9d 100644 --- a/src/s3ql/cli/mount.py +++ b/src/s3ql/cli/mount.py @@ -3,40 +3,62 @@ mount.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import - -# We can't use relative imports because this file may -# be directly executed. -import sys -from s3ql import fs, CURRENT_FS_REV +from s3ql import fs, CURRENT_FS_REV, inode_cache +from s3ql.backends.common import get_bucket_factory, BucketPool +from s3ql.block_cache import BlockCache +from s3ql.common import (setup_logging, get_bucket_cachedir, get_seq_no, + QuietError, cycle_metadata, dump_metadata, + restore_metadata) from s3ql.daemonize import daemonize -from s3ql.backends.common import (ChecksumError) -from s3ql.common import (setup_logging, get_backend, get_bucket_home, get_seq_no, - QuietError, unlock_bucket, ExceptionStoringThread, - cycle_metadata, dump_metadata, restore_metadata) -from s3ql.parse_args import ArgumentParser from s3ql.database import Connection +from s3ql.parse_args import ArgumentParser +from threading import Thread +import cPickle as pickle import llfuse -import tempfile -import textwrap +import logging import os -import stat +import shutil import signal -import time +import stat +import sys +import tempfile +import textwrap +import thread import threading -import logging -import cPickle as pickle - -#import psyco -#psyco.profile() - -__all__ = [ 'main' ] +import time log = logging.getLogger("mount") +def install_thread_excepthook(): + """work around sys.excepthook thread bug + + See http://bugs.python.org/issue1230540. + + Call once from __main__ before creating any threads. If using + psyco, call psyco.cannotcompile(threading.Thread.run) since this + replaces a new-style class method. + """ + + init_old = threading.Thread.__init__ + def init(self, *args, **kwargs): + init_old(self, *args, **kwargs) + run_old = self.run + def run_with_except_hook(*args, **kw): + try: + run_old(*args, **kw) + except SystemExit: + raise + except: + sys.excepthook(*sys.exc_info()) + self.run = run_with_except_hook + + threading.Thread.__init__ = init +install_thread_excepthook() + def main(args=None): '''Mount S3QL file system''' @@ -44,11 +66,13 @@ def main(args=None): args = sys.argv[1:] options = parse_args(args) - fuse_opts = get_fuse_opts(options) # Save handler so that we can remove it when daemonizing - stdout_log_handler = setup_logging(options, 'mount.log') + stdout_log_handler = setup_logging(options) + if options.threads is None: + options.threads = determine_threads(options) + if not os.path.exists(options.mountpoint): raise QuietError('Mountpoint does not exist.') @@ -57,96 +81,138 @@ def main(args=None): import pstats prof = cProfile.Profile() - with get_backend(options.storage_url, options.homedir, - options.ssl) as (conn, bucketname): - - if not bucketname in conn: - raise QuietError("Bucket does not exist.") - bucket = conn.get_bucket(bucketname, compression=options.compress) - - # Unlock bucket - try: - unlock_bucket(options.homedir, options.storage_url, bucket) - except ChecksumError: - raise QuietError('Checksum error - incorrect password?') - - # Get paths - home = get_bucket_home(options.storage_url, options.homedir) + bucket_factory = get_bucket_factory(options) + bucket_pool = BucketPool(bucket_factory) + + # Get paths + cachepath = get_bucket_cachedir(options.storage_url, options.cachedir) - # Retrieve metadata - (param, db) = get_metadata(bucket, home) + # Retrieve metadata + with bucket_pool() as bucket: + (param, db) = get_metadata(bucket, cachepath) + + if options.nfs: + log.info('Creating NFS indices...') + # NFS may try to look up '..', so we have to speed up this kind of query + db.execute('CREATE INDEX IF NOT EXISTS ix_contents_inode ON contents(inode)') - metadata_upload_thread = MetadataUploadThread(bucket, param, db, - options.metadata_upload_interval) - operations = fs.Operations(bucket, db, cachedir=home + '-cache', - blocksize=param['blocksize'], - cache_size=options.cachesize * 1024, - upload_event=metadata_upload_thread.event, - cache_entries=options.max_cache_entries) + # Since we do not support generation numbers, we have to keep the + # likelihood of reusing a just-deleted inode low + inode_cache.RANDOMIZE_INODES = True + else: + db.execute('DROP INDEX IF EXISTS ix_contents_inode') + + metadata_upload_thread = MetadataUploadThread(bucket_pool, param, db, + options.metadata_upload_interval) + block_cache = BlockCache(bucket_pool, db, cachepath + '-cache', + options.cachesize * 1024, options.max_cache_entries) + commit_thread = CommitThread(block_cache) + operations = fs.Operations(block_cache, db, blocksize=param['blocksize'], + upload_event=metadata_upload_thread.event) + + log.info('Mounting filesystem...') + llfuse.init(operations, options.mountpoint, get_fuse_opts(options)) + + if not options.fg: + if stdout_log_handler: + logging.getLogger().removeHandler(stdout_log_handler) + daemonize(options.cachedir) + + exc_info = setup_exchook() + + # After we start threads, we must be sure to terminate them + # or the process will hang + try: + block_cache.init(options.threads) + metadata_upload_thread.start() + commit_thread.start() - log.info('Mounting filesystem...') - llfuse.init(operations, options.mountpoint, fuse_opts) - try: - if not options.fg: - conn.prepare_fork() - me = threading.current_thread() - for t in threading.enumerate(): - if t is me: - continue - log.error('Waiting for thread %s', t) - t.join() - - if stdout_log_handler: - logging.getLogger().removeHandler(stdout_log_handler) - daemonize(options.homedir) - conn.finish_fork() + if options.upstart: + os.kill(os.getpid(), signal.SIGSTOP) + if options.profile: + prof.runcall(llfuse.main, options.single) + else: + llfuse.main(options.single) + + log.info("FUSE main loop terminated.") + + except: + log.info("Caught exception in main loop, unmounting file system...") + + # Tell finally handler that there already is an exception + if not exc_info: + exc_info = sys.exc_info() + + # We do *not* unmount on exception. Why? E.g. if someone is mirroring the + # mountpoint, and it suddenly becomes empty, all the mirrored data will be + # deleted. However, it's crucial to still call llfuse.close, so that + # Operations.destroy() can flush the inode cache. + with llfuse.lock: + llfuse.close(unmount=False) + + raise - metadata_upload_thread.start() - if options.upstart: - os.kill(os.getpid(), signal.SIGSTOP) - if options.profile: - prof.runcall(llfuse.main, options.single) - else: - llfuse.main(options.single) - - finally: - llfuse.close() - metadata_upload_thread.stop() - - db_mtime = metadata_upload_thread.db_mtime + # Terminate threads + finally: + log.debug("Waiting for background threads...") + for (op, with_lock) in ((metadata_upload_thread.stop, False), + (commit_thread.stop, False), + (block_cache.destroy, True), + (metadata_upload_thread.join, False), + (commit_thread.join, False)): + try: + if with_lock: + with llfuse.lock: + op() + else: + op() + except: + # We just live with the race cond here + if not exc_info: + exc_info = sys.exc_info() + else: + log.exception("Exception during cleanup:") + + log.debug("All background threads terminated.") + + # Re-raise if main loop terminated due to exception in other thread + # or during cleanup + if exc_info: + raise exc_info[0], exc_info[1], exc_info[2] - if operations.encountered_errors: - param['needs_fsck'] = True - else: - param['needs_fsck'] = False - - # Do not update .params yet, dump_metadata() may - # fail if the database is corrupted, in which case we - # want to force an fsck. - + # At this point, there should be no other threads left + + # Unmount + log.info("Unmounting file system.") + with llfuse.lock: + llfuse.close() + + # Do not update .params yet, dump_metadata() may fail if the database is + # corrupted, in which case we want to force an fsck. + + with bucket_pool() as bucket: seq_no = get_seq_no(bucket) - if db_mtime == os.stat(home + '.db').st_mtime: + if metadata_upload_thread.db_mtime == os.stat(cachepath + '.db').st_mtime: log.info('File system unchanged, not uploading metadata.') del bucket['s3ql_seq_no_%d' % param['seq_no']] param['seq_no'] -= 1 - pickle.dump(param, open(home + '.params', 'wb'), 2) + pickle.dump(param, open(cachepath + '.params', 'wb'), 2) elif seq_no == param['seq_no']: - log.info('Saving metadata...') - fh = tempfile.TemporaryFile() - dump_metadata(fh, db) - log.info("Compressing & uploading metadata..") + log.info('Uploading metadata...') cycle_metadata(bucket) - fh.seek(0) param['last-modified'] = time.time() - time.timezone - bucket.store_fh("s3ql_metadata", fh, param) - fh.close() - pickle.dump(param, open(home + '.params', 'wb'), 2) + with tempfile.TemporaryFile() as tmp: + dump_metadata(tmp, db) + tmp.seek(0) + with bucket.open_write('s3ql_metadata', param) as fh: + shutil.copyfileobj(tmp, fh) + pickle.dump(param, open(cachepath + '.params', 'wb'), 2) else: log.error('Remote metadata is newer than local (%d vs %d), ' 'refusing to overwrite!', seq_no, param['seq_no']) log.error('The locally cached metadata will be *lost* the next time the file system ' 'is mounted or checked and has therefore been backed up.') - for name in (home + '.params', home + '.db'): + for name in (cachepath + '.params', cachepath + '.db'): for i in reversed(range(4)): if os.path.exists(name + '.%d' % i): os.rename(name + '.%d' % i, name + '.%d' % (i+1)) @@ -168,13 +234,34 @@ def main(args=None): p.sort_stats('time') p.print_stats(50) fh.close() - - if operations.encountered_errors: - raise QuietError('Some errors were encountered while the file system was mounted,\n' - 'you should run fsck.s3ql and examine ~/.s3ql/mount.log.') - - -def get_metadata(bucket, home): + +def determine_threads(options): + '''Return optimum number of upload threads''' + + cores = os.sysconf('SC_NPROCESSORS_ONLN') + memory = os.sysconf('SC_PHYS_PAGES') * os.sysconf('SC_PAGESIZE') + + if options.compress == 'lzma': + # Keep this in sync with compression level in backends/common.py + # Memory usage according to man xz(1) + mem_per_thread = 186 * 1024**2 + else: + # Only check LZMA memory usage + mem_per_thread = 0 + + if cores == -1 or memory == -1: + log.warn("Can't determine number of cores, using 2 upload threads.") + return 1 + elif 2*cores * mem_per_thread > (memory/2): + threads = int((memory/2) // mem_per_thread) + log.info('Using %d upload threads (memory limited).', threads) + return threads + else: + log.info("Using %d upload threads.", 2*cores) + return 2*cores + + +def get_metadata(bucket, cachepath): '''Retrieve metadata Checks: @@ -183,26 +270,25 @@ def get_metadata(bucket, home): Locally cached metadata is used if up-to-date. ''' - + seq_no = get_seq_no(bucket) # Check for cached metadata db = None - if os.path.exists(home + '.params'): - param = pickle.load(open(home + '.params', 'rb')) + if os.path.exists(cachepath + '.params'): + param = pickle.load(open(cachepath + '.params', 'rb')) if param['seq_no'] < seq_no: log.info('Ignoring locally cached metadata (outdated).') param = bucket.lookup('s3ql_metadata') else: log.info('Using cached metadata.') - db = Connection(home + '.db') + db = Connection(cachepath + '.db') else: param = bucket.lookup('s3ql_metadata') - + # Check for unclean shutdown if param['seq_no'] < seq_no: - if (bucket.read_after_write_consistent() and - bucket.read_after_delete_consistent()): + if bucket.is_get_consistent(): raise QuietError(textwrap.fill(textwrap.dedent('''\ It appears that the file system is still mounted somewhere else. If this is not the case, the file system may have not been unmounted cleanly and you should try @@ -234,28 +320,24 @@ def get_metadata(bucket, home): # Download metadata if not db: log.info("Downloading & uncompressing metadata...") - fh = tempfile.TemporaryFile() - bucket.fetch_fh("s3ql_metadata", fh) - os.close(os.open(home + '.db.tmp', os.O_RDWR | os.O_CREAT | os.O_TRUNC, + os.close(os.open(cachepath + '.db.tmp', os.O_RDWR | os.O_CREAT | os.O_TRUNC, stat.S_IRUSR | stat.S_IWUSR)) - db = Connection(home + '.db.tmp', fast_mode=True) - fh.seek(0) - log.info('Reading metadata...') - restore_metadata(fh, db) - fh.close() + db = Connection(cachepath + '.db.tmp', fast_mode=True) + with bucket.open_read("s3ql_metadata") as fh: + restore_metadata(fh, db) db.close() - os.rename(home + '.db.tmp', home + '.db') - db = Connection(home + '.db') + os.rename(cachepath + '.db.tmp', cachepath + '.db') + db = Connection(cachepath + '.db') # Increase metadata sequence no param['seq_no'] += 1 param['needs_fsck'] = True - bucket.store('s3ql_seq_no_%d' % param['seq_no'], 'Empty') - pickle.dump(param, open(home + '.params', 'wb'), 2) + bucket['s3ql_seq_no_%d' % param['seq_no']] = 'Empty' + pickle.dump(param, open(cachepath + '.params', 'wb'), 2) + param['needs_fsck'] = False return (param, db) - def get_fuse_opts(options): '''Return fuse options for given command line options''' @@ -272,7 +354,6 @@ def get_fuse_opts(options): return fuse_opts - def parse_args(args): '''Parse command line''' @@ -304,15 +385,16 @@ def parse_args(args): parser = ArgumentParser( description="Mount an S3QL file system.") - parser.add_homedir() + parser.add_log('~/.s3ql/mount.log') + parser.add_cachedir() + parser.add_authfile() parser.add_debug_modules() parser.add_quiet() parser.add_version() parser.add_storage_url() - parser.add_ssl() parser.add_argument("mountpoint", metavar='<mountpoint>', - type=(lambda x: x.rstrip('/')), + type=(lambda x: os.path.abspath(x)), help='Where to mount the file system') parser.add_argument("--cachesize", type=int, default=102400, metavar='<size>', @@ -353,16 +435,21 @@ def parse_args(args): default=24*60*60, metavar='<seconds>', help='Interval in seconds between complete metadata uploads. ' 'Set to 0 to disable. Default: 24h.') - parser.add_argument("--compression-threads", action="store", type=int, - default=1, metavar='<no>', - help='Number of parallel compression and encryption threads ' - 'to use (default: %(default)s).') - + parser.add_argument("--threads", action="store", type=int, + default=None, metavar='<no>', + help='Number of parallel upload threads to use (default: auto).') + parser.add_argument("--nfs", action="store_true", default=False, + help='Support export of S3QL file systems over NFS ' + '(default: %(default)s)') + options = parser.parse_args(args) if options.allow_other and options.allow_root: parser.error("--allow-other and --allow-root are mutually exclusive.") + if not options.log and not options.fg: + parser.error("Please activate logging to a file or syslog, or use the --fg option.") + if options.profile: options.single = True @@ -374,24 +461,22 @@ def parse_args(args): if options.compress == 'none': options.compress = None - - if not os.path.exists(options.homedir): - os.mkdir(options.homedir, 0700) - - from .. import upload_manager - upload_manager.MAX_COMPRESS_THREADS = options.compression_threads - + return options -class MetadataUploadThread(ExceptionStoringThread): +class MetadataUploadThread(Thread): ''' - Periodically commit dirty inodes. - ''' + Periodically upload metadata. Upload is done every `interval` + seconds, and whenever `event` is set. To terminate thread, + set `quit` attribute as well as `event` event. + This class uses the llfuse global lock. When calling objects + passed in the constructor, the global lock is acquired first. + ''' - def __init__(self, bucket, param, db, interval): + def __init__(self, bucket_pool, param, db, interval): super(MetadataUploadThread, self).__init__() - self.bucket = bucket + self.bucket_pool = bucket_pool self.param = param self.db = db self.interval = interval @@ -401,10 +486,10 @@ class MetadataUploadThread(ExceptionStoringThread): self.quit = False self.name = 'Metadata-Upload-Thread' - def run_protected(self): + def run(self): log.debug('MetadataUploadThread: start') - while True: + while not self.quit: self.event.wait(self.interval) self.event.clear() @@ -412,47 +497,132 @@ class MetadataUploadThread(ExceptionStoringThread): break with llfuse.lock: + if self.quit: + break new_mtime = os.stat(self.db.file).st_mtime if self.db_mtime == new_mtime: log.info('File system unchanged, not uploading metadata.') continue + # We dump to a file first, so that we don't hold the + # lock for quite so long. log.info('Saving metadata...') fh = tempfile.TemporaryFile() dump_metadata(fh, self.db) - seq_no = get_seq_no(self.bucket) - if seq_no != self.param['seq_no']: - log.error('Remote metadata is newer than local (%d vs %d), ' - 'refusing to overwrite!', seq_no, self.param['seq_no']) + with self.bucket_pool() as bucket: + seq_no = get_seq_no(bucket) + if seq_no != self.param['seq_no']: + log.error('Remote metadata is newer than local (%d vs %d), ' + 'refusing to overwrite!', seq_no, self.param['seq_no']) + fh.close() + continue + + log.info("Compressing & uploading metadata..") + cycle_metadata(bucket) + fh.seek(0) + self.param['last-modified'] = time.time() - time.timezone + + # Temporarily decrease sequence no, this is not the final upload + self.param['seq_no'] -= 1 + with bucket.open_write("s3ql_metadata", self.param) as obj_fh: + shutil.copyfileobj(fh, obj_fh) + self.param['seq_no'] += 1 + fh.close() - continue - - log.info("Compressing & uploading metadata..") - cycle_metadata(self.bucket) - fh.seek(0) - self.param['last-modified'] = time.time() - time.timezone - - # Temporarily decrease sequence no, this is not the final upload - self.param['seq_no'] -= 1 - self.bucket.store_fh("s3ql_metadata", fh, self.param) - self.param['seq_no'] += 1 - - fh.close() - self.db_mtime = new_mtime + self.db_mtime = new_mtime log.debug('MetadataUploadThread: end') def stop(self): - '''Wait for thread to finish, raise any occurred exceptions. - - This method releases the global lock. - ''' + '''Signal thread to terminate''' self.quit = True - self.event.set() - self.join_and_raise() + self.event.set() + +def setup_exchook(): + '''Send SIGTERM if any other thread terminates with an exception + + The exc_info will be saved in the list object returned + by this function. + ''' + + this_thread = thread.get_ident() + old_exchook = sys.excepthook + exc_info = [] + + def exchook(type_, val, tb): + if (thread.get_ident() != this_thread + and not exc_info): + os.kill(os.getpid(), signal.SIGTERM) + exc_info.append(type_) + exc_info.append(val) + exc_info.append(tb) + + old_exchook(type_, val, tb) + + # If the main thread re-raised exception, there is no need to call + # excepthook again + elif not (thread.get_ident() == this_thread + and exc_info == [type_, val, tb]): + old_exchook(type_, val, tb) + + sys.excepthook = exchook + + return exc_info + + +class CommitThread(Thread): + ''' + Periodically upload dirty blocks. + + This class uses the llfuse global lock. When calling objects + passed in the constructor, the global lock is acquired first. + ''' + + + def __init__(self, block_cache): + super(CommitThread, self).__init__() + self.block_cache = block_cache + self.stop_event = threading.Event() + self.name = 'CommitThread' + + def run(self): + log.debug('CommitThread: start') + + while not self.stop_event.is_set(): + did_sth = False + stamp = time.time() + for el in self.block_cache.entries.values_rev(): + if stamp - el.last_access < 10: + break + if not (el.dirty and (el.inode, el.blockno) not in self.block_cache.in_transit): + continue + + # Acquire global lock to access UploadManager instance + with llfuse.lock: + if self.stop_event.is_set(): + break + # Object may have been accessed while waiting for lock + if not (el.dirty and (el.inode, el.blockno) not in self.block_cache.in_transit): + continue + self.block_cache.upload(el) + did_sth = True + + if self.stop_event.is_set(): + break + + if not did_sth: + self.stop_event.wait(5) + + log.debug('CommitThread: end') + + def stop(self): + '''Signal thread to terminate''' + + self.stop_event.set() + if __name__ == '__main__': main(sys.argv[1:]) diff --git a/src/s3ql/cli/remove.py b/src/s3ql/cli/remove.py index 492a9a3..d70017b 100644 --- a/src/s3ql/cli/remove.py +++ b/src/s3ql/cli/remove.py @@ -3,7 +3,7 @@ remove.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import diff --git a/src/s3ql/cli/statfs.py b/src/s3ql/cli/statfs.py index 96eac81..ad55b0c 100644 --- a/src/s3ql/cli/statfs.py +++ b/src/s3ql/cli/statfs.py @@ -3,7 +3,7 @@ statfs.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import diff --git a/src/s3ql/cli/umount.py b/src/s3ql/cli/umount.py index a555d6f..57cc792 100644 --- a/src/s3ql/cli/umount.py +++ b/src/s3ql/cli/umount.py @@ -3,7 +3,7 @@ umount.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import @@ -20,7 +20,6 @@ import time import textwrap log = logging.getLogger("umount") -DONTWAIT = False def parse_args(args): '''Parse command line @@ -32,11 +31,7 @@ def parse_args(args): parser = ArgumentParser( description=textwrap.dedent('''\ Unmounts an S3QL file system. The command returns only after all data - has been uploaded to the backend. If any file system errors occurred - while the file system was mounted, a warning message is printed. Note - that errors occuring during the unmount (e.g. a failure to upload the - metadata) can not be detected and appear only in the logging messages of - the mount program.''')) + has been uploaded to the backend.''')) parser.add_debug() parser.add_quiet() @@ -91,25 +86,20 @@ def lazy_umount(mountpoint): This function writes to stdout/stderr and calls `system.exit()`. ''' - found_errors = False - if not warn_if_error(mountpoint): - found_errors = True - umount_cmd = ('fusermount', '-u', '-z', mountpoint) + if os.getuid() == 0: + umount_cmd = ('umount', '-l', mountpoint) + else: + umount_cmd = ('fusermount', '-u', '-z', mountpoint) + if not subprocess.call(umount_cmd) == 0: - found_errors = True - - if found_errors: sys.exit(1) - def blocking_umount(mountpoint): '''Invoke fusermount and wait for daemon to terminate. This function writes to stdout/stderr and calls `system.exit()`. ''' - found_errors = False - devnull = open('/dev/null', 'wb') if subprocess.call(['fuser', '-m', mountpoint], stdout=devnull, stderr=devnull) == 0: @@ -123,9 +113,6 @@ def blocking_umount(mountpoint): log.debug('Flushing cache...') llfuse.setxattr(ctrlfile, b's3ql_flushcache!', b'dummy') - if not warn_if_error(mountpoint): - found_errors = True - # Get pid log.debug('Trying to get pid') pid = int(llfuse.getxattr(ctrlfile, b's3ql_pid?')) @@ -140,7 +127,13 @@ def blocking_umount(mountpoint): log.debug('Unmounting...') # This seems to be necessary to prevent weird busy errors time.sleep(3) - if subprocess.call(['fusermount', '-u', mountpoint]) != 0: + + if os.getuid() == 0: + umount_cmd = ['umount', mountpoint] + else: + umount_cmd = ['fusermount', '-u', mountpoint] + + if subprocess.call(umount_cmd) != 0: sys.exit(1) # Wait for daemon @@ -168,37 +161,11 @@ def blocking_umount(mountpoint): log.debug('Reading cmdline failed, assuming daemon has quit.') break - if DONTWAIT: # for testing - break - # Process still exists, we wait log.debug('Daemon seems to be alive, waiting...') time.sleep(step) if step < 10: step *= 2 - if found_errors: - sys.exit(1) - - -def warn_if_error(mountpoint): - '''Check if file system encountered any errors - - If there were errors, a warning is printed to stdout and the - function returns False. - ''' - - log.debug('Trying to get error status') - ctrlfile = os.path.join(mountpoint, CTRL_NAME) - status = llfuse.getxattr(ctrlfile, 's3ql_errors?') - - if status != 'no errors': - print('Some errors occurred while the file system was mounted.\n' - 'You should examine the log files and run fsck before mounting the\n' - 'file system again.', file=sys.stderr) - return False - else: - return True - if __name__ == '__main__': main(sys.argv[1:]) diff --git a/src/s3ql/common.py b/src/s3ql/common.py index 13b7ef5..4401501 100644 --- a/src/s3ql/common.py +++ b/src/s3ql/common.py @@ -3,51 +3,33 @@ common.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import - -from getpass import getpass -from time import sleep +from llfuse import ROOT_INODE +import cPickle as pickle import hashlib +import logging import os +import shutil import stat import sys -import threading -import logging.handlers -import traceback +import tempfile import time -import re -import cPickle as pickle -from contextlib import contextmanager -from llfuse import ROOT_INODE -from .backends.common import NoSuchObject - -__all__ = ["get_bucket_home", 'sha256_fh', 'add_stdout_logging', - "get_credentials", "get_dbfile", "inode_for_path", "get_path", - "ROOT_INODE", "ExceptionStoringThread", 'retry', 'LoggerFilter', - "EmbeddedException", 'CTRL_NAME', 'CTRL_INODE', 'unlock_bucket', - 'QuietError', 'get_backend', 'add_file_logging', 'setup_excepthook', - 'cycle_metadata', 'restore_metadata', 'dump_metadata', - 'setup_logging', 'AsyncFn', 'init_tables', 'create_indices', - 'create_tables', 'get_seq_no' ] - - -AUTHINFO_BACKEND_PATTERN = r'^backend\s+(\S+)\s+machine\s+(\S+)\s+login\s+(\S+)\s+password\s+(.+)$' -AUTHINFO_BUCKET_PATTERN = r'^storage-url\s+(\S+)\s+password\s+(.+)$' log = logging.getLogger('common') -def setup_logging(options, logfile=None): +def setup_logging(options): root_logger = logging.getLogger() if root_logger.handlers: log.debug("Logging already initialized.") return stdout_handler = add_stdout_logging(options.quiet) - if logfile: - debug_handler = add_file_logging(os.path.join(options.homedir, logfile)) + if hasattr(options, 'log') and options.log: + root_logger.addHandler(options.log) + debug_handler = options.log else: debug_handler = stdout_handler setup_excepthook() @@ -105,152 +87,60 @@ def add_stdout_logging(quiet=False): root_logger.addHandler(handler) return handler - -def add_file_logging(logfile): - - root_logger = logging.getLogger() - formatter = logging.Formatter('%(asctime)s.%(msecs)03d [%(process)s] %(threadName)s: ' - '[%(name)s] %(message)s', datefmt="%Y-%m-%d %H:%M:%S") - handler = logging.handlers.RotatingFileHandler(logfile, maxBytes=1024**2, - backupCount=5) - handler.setFormatter(formatter) - root_logger.addHandler(handler) - return handler - - -@contextmanager -def get_backend(storage_url, homedir, use_ssl): - '''Return backend connection and bucket name - - This is a context manager, since some connections need to be cleaned - up properly. - ''' - - from .backends import s3, local, ftp - - if storage_url.startswith('local://'): - conn = local.Connection() - bucketname = storage_url[len('local://'):] - - elif storage_url.startswith('s3://'): - (login, password) = get_backend_credentials(homedir, 's3', None) - conn = s3.Connection(login, password, use_ssl) - bucketname = storage_url[len('s3://'):] - - elif storage_url.startswith('s3rr://'): - log.warn('Warning: Using S3 reduced redundancy storage (S3) is *not* recommended!') - (login, password) = get_backend_credentials(homedir, 's3', None) - conn = s3.Connection(login, password, use_ssl, reduced_redundancy=True) - bucketname = storage_url[len('s3rr://'):] - - else: - pat = r'^([a-z]+)://([a-zA-Z0-9.-]+)(?::([0-9]+))?(/[a-zA-Z0-9./_-]+)$' - match = re.match(pat, storage_url) - if not match: - raise QuietError('Invalid storage url: %r' % storage_url) - (backend, host, port, bucketname) = match.groups() - (login, password) = get_backend_credentials(homedir, backend, host) - - if backend == 'ftp' and not use_ssl: - conn = ftp.Connection(host, port, login, password) - elif backend == 'ftps': - conn = ftp.TLSConnection(host, port, login, password) - elif backend == 'sftp': - from .backends import sftp - conn = sftp.Connection(host, port, login, password) - else: - raise QuietError('Unknown backend: %s' % backend) - - try: - yield (conn, bucketname) - finally: - conn.close() - def get_seq_no(bucket): - '''Get current metadata sequence number''' - - seq_nos = [ int(x[len('s3ql_seq_no_'):]) for x in bucket.list('s3ql_seq_no_') ] + '''Get current metadata sequence number''' + from .backends.common import NoSuchObject + + seq_nos = list(bucket.list('s3ql_seq_no_')) if not seq_nos: + # Maybe list result is outdated + seq_nos = [ 's3ql_seq_no_1' ] + + if (seq_nos[0].endswith('.meta') + or seq_nos[0].endswith('.dat')): raise QuietError('Old file system revision, please run `s3qladm upgrade` first.') + + seq_nos = [ int(x[len('s3ql_seq_no_'):]) for x in seq_nos ] seq_no = max(seq_nos) + + # Make sure that object really exists + while ('s3ql_seq_no_%d' % seq_no) not in bucket: + seq_no -= 1 + if seq_no == 0: + raise QuietError('No S3QL file system found in bucket.') + while ('s3ql_seq_no_%d' % seq_no) in bucket: + seq_no += 1 + seq_no -= 1 + + # Delete old seq nos for i in [ x for x in seq_nos if x < seq_no - 10 ]: try: - del bucket['s3ql_seq_no_%d' % i ] + del bucket['s3ql_seq_no_%d' % i] except NoSuchObject: pass # Key list may not be up to date return seq_no def cycle_metadata(bucket): - from .backends.common import UnsupportedError - + from .backends.common import NoSuchObject + for i in reversed(range(10)): - if "s3ql_metadata_bak_%d" % i in bucket: - try: - bucket.rename("s3ql_metadata_bak_%d" % i, "s3ql_metadata_bak_%d" % (i + 1)) - except UnsupportedError: - bucket.copy("s3ql_metadata_bak_%d" % i, "s3ql_metadata_bak_%d" % (i + 1)) + try: + bucket.copy("s3ql_metadata_bak_%d" % i, "s3ql_metadata_bak_%d" % (i + 1)) + except NoSuchObject: + pass - try: - bucket.rename("s3ql_metadata", "s3ql_metadata_bak_0") - except UnsupportedError: - bucket.copy("s3ql_metadata", "s3ql_metadata_bak_0") - - -def unlock_bucket(homedir, storage_url, bucket): - '''Ask for passphrase if bucket requires one''' - - if 's3ql_passphrase' not in bucket: - return - - # Try to read from file - keyfile = os.path.join(homedir, 'authinfo') - wrap_pw = None - - if os.path.isfile(keyfile): - mode = os.stat(keyfile).st_mode - if mode & (stat.S_IRGRP | stat.S_IROTH): - raise QuietError("%s has insecure permissions, aborting." % keyfile) - - fh = open(keyfile, "r") - for line in fh: - line = line.strip() - if not line or line.startswith('#'): - continue - if re.match(AUTHINFO_BACKEND_PATTERN, line): - continue - res = re.match(AUTHINFO_BUCKET_PATTERN, line) - if not res: - log.warn('Cannot parse line in %s:\n %s', keyfile, line) - continue - - if storage_url == res.group(1): - wrap_pw = res.group(2) - log.info('Using encryption password from %s', keyfile) - break - - # Otherwise from stdin - if wrap_pw is None: - if sys.stdin.isatty(): - wrap_pw = getpass("Enter bucket encryption passphrase: ") - else: - wrap_pw = sys.stdin.readline().rstrip() - - bucket.passphrase = wrap_pw - data_pw = bucket['s3ql_passphrase'] - bucket.passphrase = data_pw - + bucket.copy("s3ql_metadata", "s3ql_metadata_bak_0") def dump_metadata(ofh, conn): pickler = pickle.Pickler(ofh, 2) - data_start = 2048 bufsize = 256 buf = range(bufsize) - tables_to_dump = [('inodes', 'id'), - ('contents', 'name, parent_inode'), - ('ext_attributes', 'inode, name'), - ('objects', 'id'), - ('blocks', 'inode, blockno')] + tables_to_dump = [('objects', 'id'), ('blocks', 'id'), + ('inode_blocks', 'inode, blockno'), + ('inodes', 'id'), ('symlink_targets', 'inode'), + ('names', 'id'), ('contents', 'parent_inode, name_id'), + ('ext_attributes', 'inode, name')] columns = dict() for (table, _) in tables_to_dump: @@ -258,47 +148,53 @@ def dump_metadata(ofh, conn): for row in conn.query('PRAGMA table_info(%s)' % table): columns[table].append(row[1]) - ofh.seek(data_start) - sizes = dict() + pickler.dump((tables_to_dump, columns)) + for (table, order) in tables_to_dump: log.info('Saving %s' % table) pickler.clear_memo() - sizes[table] = 0 i = 0 - for row in conn.query('SELECT * FROM %s ORDER BY %s' % (table, order)): + for row in conn.query('SELECT %s FROM %s ORDER BY %s' + % (','.join(columns[table]), table, order)): buf[i] = row i += 1 if i == bufsize: pickler.dump(buf) pickler.clear_memo() - sizes[table] += 1 i = 0 if i != 0: pickler.dump(buf[:i]) - sizes[table] += 1 + + pickler.dump(None) - ofh.seek(0) - pickler.dump((data_start, tables_to_dump, sizes, columns)) - assert ofh.tell() < data_start def restore_metadata(ifh, conn): + # Unpickling is terribly slow if fh is not a real file + # object. + if not hasattr(ifh, 'fileno'): + log.info('Caching tables...') + with tempfile.TemporaryFile() as tmp: + shutil.copyfileobj(ifh, tmp) + tmp.seek(0) + return restore_metadata(tmp, conn) + unpickler = pickle.Unpickler(ifh) - (data_start, to_dump, sizes, columns) = unpickler.load() - ifh.seek(data_start) + (to_dump, columns) = unpickler.load() create_tables(conn) for (table, _) in to_dump: log.info('Loading %s', table) col_str = ', '.join(columns[table]) val_str = ', '.join('?' for _ in columns[table]) sql_str = 'INSERT INTO %s (%s) VALUES(%s)' % (table, col_str, val_str) - for _ in xrange(sizes[table]): + while True: buf = unpickler.load() + if not buf: + break for row in buf: conn.execute(sql_str, row) - create_indices(conn) conn.execute('ANALYZE') class QuietError(Exception): @@ -352,14 +248,13 @@ def inode_for_path(path, conn): inode = ROOT_INODE for el in path.split(b'/'): try: - inode = conn.get_val("SELECT inode FROM contents WHERE name=? AND parent_inode=?", - (el, inode)) + inode = conn.get_val("SELECT inode FROM contents_v WHERE name=? AND parent_inode=?", + (el, inode)) except NoSuchRowError: raise KeyError('Path %s does not exist' % path) return inode - def get_path(id_, conn, name=None): """Return a full path for inode `id_`. @@ -377,9 +272,9 @@ def get_path(id_, conn, name=None): maxdepth = 255 while id_ != ROOT_INODE: - # This can be ambigious if directories are hardlinked - (name2, id_) = conn.get_row("SELECT name, parent_inode FROM contents WHERE inode=? LIMIT 1", - (id_,)) + # This can be ambiguous if directories are hardlinked + (name2, id_) = conn.get_row("SELECT name, parent_inode FROM contents_v " + "WHERE inode=? LIMIT 1", (id_,)) path.append(name2) maxdepth -= 1 if maxdepth == 0: @@ -401,158 +296,20 @@ def _escape(s): return s -def get_bucket_home(storage_url, homedir): - if not os.path.exists(homedir): - os.mkdir(homedir) - return os.path.join(homedir, _escape(storage_url)) - - -def get_backend_credentials(homedir, backend, host): - """Get credentials for given backend and host""" - - # Try to read from file - keyfile = os.path.join(homedir, 'authinfo') - - if os.path.isfile(keyfile): - mode = os.stat(keyfile).st_mode - if mode & (stat.S_IRGRP | stat.S_IROTH): - raise QuietError("%s has insecure permissions, aborting." % keyfile) - - fh = open(keyfile, "r") - for line in fh: - line = line.strip() - if not line or line.startswith('#'): - continue - if re.match(AUTHINFO_BUCKET_PATTERN, line): - continue - res = re.match(AUTHINFO_BACKEND_PATTERN, line) - if not res: - log.warn('Cannot parse line in %s:\n %s', keyfile, line) - continue - - if backend == res.group(1) and (host is None or host == res.group(2)): - log.info('Using backend credentials from %s', keyfile) - return res.group(3, 4) - - # Otherwise from stdin - if sys.stdin.isatty(): - if host: - print("Enter backend login for %s: " % host, end='') - else: - print("Enter backend login: ", end='') - key = sys.stdin.readline().rstrip() - - if sys.stdin.isatty(): - if host: - pw = getpass("Enter backend password for %s: " % host) - else: - pw = getpass("Enter backend password: ") - else: - pw = sys.stdin.readline().rstrip() - - return (key, pw) - -def retry(timeout, fn, *a, **kw): - """Wait for fn(*a, **kw) to return True. - - If the return value of fn() returns something True, this value - is returned. Otherwise, the function is called repeatedly for - `timeout` seconds. If the timeout is reached, `TimeoutError` is - raised. - """ - - step = 0.2 - waited = 0 - while waited < timeout: - ret = fn(*a, **kw) - if ret: - return ret - sleep(step) - waited += step - if step < waited / 30: - step *= 2 - - raise TimeoutError() - -class TimeoutError(Exception): - '''Raised by `retry()` when a timeout is reached.''' - - pass +def get_bucket_cachedir(storage_url, cachedir): + if not os.path.exists(cachedir): + os.mkdir(cachedir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + return os.path.join(cachedir, _escape(storage_url)) # Name and inode of the special s3ql control file CTRL_NAME = b'.__s3ql__ctrl__' CTRL_INODE = 2 -class ExceptionStoringThread(threading.Thread): - def __init__(self): - super(ExceptionStoringThread, self).__init__() - self._exc_info = None - self._joined = False - - def run_protected(self): - pass - - def run(self): - try: - self.run_protected() - except: - # This creates a circular reference chain - self._exc_info = sys.exc_info() - - def join_get_exc(self): - self._joined = True - self.join() - return self._exc_info - - def join_and_raise(self): - '''Wait for the thread to finish, raise any occurred exceptions''' - - self._joined = True - if self.is_alive(): - self.join() - - if self._exc_info is not None: - # Break reference chain - exc_info = self._exc_info - self._exc_info = None - raise EmbeddedException(exc_info, self.name) - - def __del__(self): - if not self._joined: - raise RuntimeError("ExceptionStoringThread instance was destroyed " - "without calling join_and_raise()!") - - -class AsyncFn(ExceptionStoringThread): - def __init__(self, fn, *args, **kwargs): - super(AsyncFn, self).__init__() - self.target = fn - self.args = args - self.kwargs = kwargs - - def run_protected(self): - self.target(*self.args, **self.kwargs) - -class EmbeddedException(Exception): - '''Encapsulates an exception that happened in a different thread - ''' - - def __init__(self, exc_info, threadname): - super(EmbeddedException, self).__init__() - self.exc_info = exc_info - self.threadname = threadname - - log.error('Thread %s terminated with exception:\n%s', - self.threadname, ''.join(traceback.format_exception(*self.exc_info))) - - def __str__(self): - return ''.join(['caused by an exception in thread %s.\n' % self.threadname, - 'Original/inner traceback (most recent call last): \n' ] + - traceback.format_exception(*self.exc_info)) - - def sha256_fh(fh): fh.seek(0) + + # Bogus error about hashlib not having a sha256 member + #pylint: disable=E1101 sha = hashlib.sha256() while True: @@ -567,7 +324,7 @@ def init_tables(conn): # Insert root directory timestamp = time.time() - time.timezone conn.execute("INSERT INTO inodes (id,mode,uid,gid,mtime,atime,ctime,refcount) " - "VALUES (?,?,?,?,?,?,?,?)", + "VALUES (?,?,?,?,?,?,?,?)", (ROOT_INODE, stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, os.getuid(), os.getgid(), timestamp, timestamp, timestamp, 1)) @@ -583,10 +340,32 @@ def init_tables(conn): "VALUES (?,?,?,?,?,?,?)", (stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR, os.getuid(), os.getgid(), timestamp, timestamp, timestamp, 1)) - conn.execute("INSERT INTO contents (name, inode, parent_inode) VALUES(?,?,?)", - (b"lost+found", inode, ROOT_INODE)) + name_id = conn.rowid('INSERT INTO names (name, refcount) VALUES(?,?)', + (b'lost+found', 1)) + conn.execute("INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)", + (name_id, inode, ROOT_INODE)) -def create_tables(conn): +def create_tables(conn): + # Table of storage objects + # Refcount is included for performance reasons + conn.execute(""" + CREATE TABLE objects ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + refcount INT NOT NULL, + compr_size INT + )""") + + # Table of known data blocks + # Refcount is included for performance reasons + conn.execute(""" + CREATE TABLE blocks ( + id INTEGER PRIMARY KEY, + hash BLOB(16) UNIQUE, + refcount INT NOT NULL, + size INT NOT NULL, + obj_id INTEGER NOT NULL REFERENCES objects(id) + )""") + # Table with filesystem metadata # The number of links `refcount` to an inode can in theory # be determined from the `contents` table. However, managing @@ -604,24 +383,52 @@ def create_tables(conn): atime REAL NOT NULL, ctime REAL NOT NULL, refcount INT NOT NULL, - target BLOB(256) , size INT NOT NULL DEFAULT 0, rdev INT NOT NULL DEFAULT 0, - locked BOOLEAN NOT NULL DEFAULT 0 - ) - """) + locked BOOLEAN NOT NULL DEFAULT 0, + + -- id of first block (blockno == 0) + -- since most inodes have only one block, we can make the db 20% + -- smaller by not requiring a separate inode_blocks row for these + -- cases. + block_id INT REFERENCES blocks(id) + )""") + + # Further Blocks used by inode (blockno >= 1) + conn.execute(""" + CREATE TABLE inode_blocks ( + inode INTEGER NOT NULL REFERENCES inodes(id), + blockno INT NOT NULL, + block_id INTEGER NOT NULL REFERENCES blocks(id), + PRIMARY KEY (inode, blockno) + )""") + + # Symlinks + conn.execute(""" + CREATE TABLE symlink_targets ( + inode INTEGER PRIMARY KEY REFERENCES inodes(id), + target BLOB NOT NULL + )""") + + # Names of file system objects + conn.execute(""" + CREATE TABLE names ( + id INTEGER PRIMARY KEY, + name BLOB NOT NULL, + refcount INT NOT NULL, + UNIQUE (name) + )""") # Table of filesystem objects - # id is used by readdir() to restart at the correct - # position + # rowid is used by readdir() to restart at the correct position conn.execute(""" CREATE TABLE contents ( rowid INTEGER PRIMARY KEY AUTOINCREMENT, - name BLOB(256) NOT NULL, + name_id INT NOT NULL REFERENCES names(id), inode INT NOT NULL REFERENCES inodes(id), parent_inode INT NOT NULL REFERENCES inodes(id), - UNIQUE (name, parent_inode) + UNIQUE (parent_inode, name_id) )""") # Extended attributes @@ -634,31 +441,18 @@ def create_tables(conn): PRIMARY KEY (inode, name) )""") - # Refcount is included for performance reasons - conn.execute(""" - CREATE TABLE objects ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - refcount INT NOT NULL, - hash BLOB(16) UNIQUE, - size INT NOT NULL, - compr_size INT - )""") - - - # Maps blocks to objects + # Shortcurts conn.execute(""" - CREATE TABLE blocks ( - inode INTEGER NOT NULL REFERENCES inodes(id), - blockno INT NOT NULL, - obj_id INTEGER NOT NULL REFERENCES objects(id), - - PRIMARY KEY (inode, blockno) - )""") + CREATE VIEW contents_v AS + SELECT * FROM contents JOIN names ON names.id = name_id + """) -def create_indices(conn): - conn.execute('CREATE INDEX IF NOT EXISTS ix_contents_parent_inode ON contents(parent_inode)') - conn.execute('CREATE INDEX IF NOT EXISTS ix_contents_inode ON contents(inode)') - conn.execute('CREATE INDEX IF NOT EXISTS ix_ext_attributes_inode ON ext_attributes(inode)') - conn.execute('CREATE INDEX IF NOT EXISTS ix_objects_hash ON objects(hash)') - conn.execute('CREATE INDEX IF NOT EXISTS ix_blocks_obj_id ON blocks(obj_id)') - conn.execute('CREATE INDEX IF NOT EXISTS ix_blocks_inode ON blocks(inode)') + # We have to be very careful when using the following view, because it + # confuses the SQLite optimizer a lot, so it often stops using indices and + # always scans both instnead tables. + conn.execute(""" + CREATE VIEW inode_blocks_v AS + SELECT * FROM inode_blocks + UNION + SELECT id as inode, 0 as blockno, block_id FROM inodes WHERE block_id IS NOT NULL + """) diff --git a/src/s3ql/daemonize.py b/src/s3ql/daemonize.py index fbbee88..a43bf81 100644 --- a/src/s3ql/daemonize.py +++ b/src/s3ql/daemonize.py @@ -5,7 +5,7 @@ daemonize.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. The functions in this file are based on the python-daemon module by Ben Finney <ben+python@benfinney.id.au>. diff --git a/src/s3ql/database.py b/src/s3ql/database.py index 012e030..aedfd45 100644 --- a/src/s3ql/database.py +++ b/src/s3ql/database.py @@ -3,7 +3,7 @@ database.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. Module Attributes: diff --git a/src/s3ql/fs.py b/src/s3ql/fs.py index 3cf55f9..78664cf 100644 --- a/src/s3ql/fs.py +++ b/src/s3ql/fs.py @@ -3,32 +3,26 @@ fs.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import - -import os -import errno -import stat -import llfuse -import collections -import logging +from .backends.common import NoSuchObject, ChecksumError +from .common import (get_path, CTRL_NAME, CTRL_INODE, LoggerFilter) +from .database import NoSuchRowError from .inode_cache import InodeCache, OutOfInodesError -from .common import (get_path, CTRL_NAME, CTRL_INODE, LoggerFilter, - EmbeddedException, ExceptionStoringThread) -import time -from .block_cache import BlockCache from cStringIO import StringIO -from .database import NoSuchRowError -from .backends.common import NoSuchObject, ChecksumError -import struct +from llfuse import FUSEError import cPickle as pickle +import collections +import errno +import llfuse +import logging import math -import threading -from llfuse import FUSEError, lock, lock_released - -__all__ = [ "Server" ] +import os +import stat +import struct +import time # standard logger for this module log = logging.getLogger("fs") @@ -52,7 +46,6 @@ class Operations(llfuse.Operations): ----------- :cache: Holds information about cached blocks - :encountered_errors: Is set to true if a request handler raised an exception :inode_cache: A cache for the attributes of the currently opened inodes. :open_inodes: dict of currently opened inodes. This is used to not remove the blocks of unlinked inodes that are still open. @@ -74,52 +67,19 @@ class Operations(llfuse.Operations): users relying on unlink()/rmdir() to fail for a directory/file. For that, it explicitly checks the st_mode attribute. """ - - def handle_exc(self, fn, exc): - '''Handle exceptions that occurred during request processing. - - This method marks the file system as needing fsck and logs the - error. - ''' - # Unused arguments - #pylint: disable=W0613 - log.error("Unexpected internal filesystem error.\n" - "Filesystem may be corrupted, run fsck.s3ql as soon as possible!\n" - "Please report this bug on http://code.google.com/p/s3ql/.") - self.encountered_errors = True - - - def __init__(self, bucket, db, cachedir, blocksize, cache_size, - cache_entries=768, upload_event=None): + def __init__(self, block_cache, db, blocksize, upload_event=None): super(Operations, self).__init__() - self.encountered_errors = False self.inodes = InodeCache(db) self.db = db self.upload_event = upload_event - self.inode_flush_thread = None self.open_inodes = collections.defaultdict(lambda: 0) self.blocksize = blocksize - self.cache = BlockCache(bucket, db, cachedir, cache_size, cache_entries) - - def init(self): - self.cache.init() - self.inode_flush_thread = InodeFlushThread(self.inodes) - self.inode_flush_thread.start() + self.cache = block_cache def destroy(self): - try: - self.inode_flush_thread.stop() - except EmbeddedException: - log.error('FlushThread terminated with exception.') - self.encountered_errors = True - self.inodes.destroy() - self.cache.destroy() - - if self.cache.encountered_errors: - self.encountered_errors = True def lookup(self, id_p, name): if name == CTRL_NAME: @@ -141,7 +101,7 @@ class Operations(llfuse.Operations): return self.inodes[id_] try: - id_ = self.db.get_val("SELECT inode FROM contents WHERE name=? AND parent_inode=?", + id_ = self.db.get_val("SELECT inode FROM contents_v WHERE name=? AND parent_inode=?", (name, id_p)) except NoSuchRowError: raise(llfuse.FUSEError(errno.ENOENT)) @@ -168,7 +128,11 @@ class Operations(llfuse.Operations): inode = self.inodes[id_] if inode.atime < inode.ctime or inode.atime < inode.mtime: inode.atime = timestamp - return inode.target + try: + return self.db.get_val("SELECT target FROM symlink_targets WHERE inode=?", (id_,)) + except NoSuchRowError: + log.warn('Inode does not have symlink target: %d', id_) + raise FUSEError(errno.EINVAL) def opendir(self, id_): return id_ @@ -190,20 +154,15 @@ class Operations(llfuse.Operations): # The ResultSet is automatically deleted # when yield raises GeneratorExit. - res = self.db.query("SELECT rowid, name, inode FROM contents WHERE parent_inode=? " - 'AND rowid > ? ORDER BY rowid', (id_, off)) + res = self.db.query("SELECT rowid, name, inode FROM contents_v " + 'WHERE parent_inode=? AND contents_v.rowid > ? ORDER BY rowid', (id_, off)) for (next_, name, cid_) in res: yield (name, self.inodes[cid_], next_) def getxattr(self, id_, name): # Handle S3QL commands if id_ == CTRL_INODE: - if name == b's3ql_errors?': - if self.encountered_errors: - return b'errors encountered' - else: - return b'no errors' - elif name == b's3ql_pid?': + if name == b's3ql_pid?': return bytes(os.getpid()) elif name == b's3qlstat': @@ -231,7 +190,6 @@ class Operations(llfuse.Operations): if id_ == CTRL_INODE: if name == b's3ql_flushcache!': self.cache.clear() - self.cache.upload_manager.join_all() elif name == 'copy': self.copy_tree(*struct.unpack('II', value)) elif name == 'upload-meta': @@ -317,17 +275,17 @@ class Operations(llfuse.Operations): while True: found_subdirs = False id_p = queue.pop() - for (name, id_) in self.db.query('SELECT name, inode FROM contents WHERE ' - 'parent_inode=?', (id_p,)): + for (name_id, id_) in self.db.query('SELECT name_id, inode FROM contents WHERE ' + 'parent_inode=?', (id_p,)): - if self.db.has_val('SELECT 1 FROM contents WHERE parent_inode=?', - (id_,)): + if self.db.has_val('SELECT 1 FROM contents WHERE parent_inode=?', (id_,)): if not found_subdirs: found_subdirs = True queue.append(id_p) queue.append(id_) else: + name = self.db.get_val("SELECT name FROM names WHERE id=?", (name_id,)) llfuse.invalidate_entry(id_p, name) self._remove(id_p, name, id_, force=True) @@ -374,7 +332,8 @@ class Operations(llfuse.Operations): for attr in ('atime', 'ctime', 'mtime', 'mode', 'uid', 'gid'): setattr(target_inode, attr, getattr(src_inode, attr)) - # We first replicate into a dummy inode + # We first replicate into a dummy inode, so that we + # need to invalidate only once. timestamp = time.time() tmp = make_inode(mtime=timestamp, ctime=timestamp, atime=timestamp, uid=0, gid=0, mode=0, refcount=0) @@ -384,14 +343,13 @@ class Operations(llfuse.Operations): processed = 0 # Number of steps since last GIL release stamp = time.time() # Time of last GIL release gil_step = 100 # Approx. number of steps between GIL releases - in_transit = set() while queue: (src_id, target_id, rowid) = queue.pop() log.debug('copy_tree(%d, %d): Processing directory (%d, %d, %d)', src_inode.id, target_inode.id, src_id, target_id, rowid) - for (name, id_, rowid) in db.query('SELECT name, inode, rowid FROM contents ' - 'WHERE parent_inode=? AND rowid > ? ' - 'ORDER BY rowid', (src_id, rowid)): + for (name_id, id_, rowid) in db.query('SELECT name_id, inode, rowid FROM contents ' + 'WHERE parent_inode=? AND rowid > ? ' + 'ORDER BY rowid', (src_id, rowid)): if id_ not in id_cache: inode = self.inodes[id_] @@ -400,8 +358,7 @@ class Operations(llfuse.Operations): inode_new = make_inode(refcount=1, mode=inode.mode, size=inode.size, uid=inode.uid, gid=inode.gid, mtime=inode.mtime, atime=inode.atime, - ctime=inode.ctime, target=inode.target, - rdev=inode.rdev) + ctime=inode.ctime, rdev=inode.rdev) except OutOfInodesError: log.warn('Could not find a free inode') raise FUSEError(errno.ENOSPC) @@ -411,24 +368,37 @@ class Operations(llfuse.Operations): if inode.refcount != 1: id_cache[id_] = id_new - for (obj_id, blockno) in db.query('SELECT obj_id, blockno FROM blocks ' - 'WHERE inode=?', (id_,)): - processed += 1 - db.execute('INSERT INTO blocks (inode, blockno, obj_id) VALUES(?, ?, ?)', - (id_new, blockno, obj_id)) - db.execute('UPDATE objects SET refcount=refcount+1 WHERE id=?', (obj_id,)) + db.execute('INSERT INTO symlink_targets (inode, target) ' + 'SELECT ?, target FROM symlink_targets WHERE inode=?', + (id_new, id_)) - if (id_, blockno) in self.cache.upload_manager.in_transit: - in_transit.add((id_, blockno)) - + db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) ' + 'SELECT ?, blockno, block_id FROM inode_blocks ' + 'WHERE inode=?', (id_new, id_)) + + db.execute('UPDATE inodes SET block_id=' + '(SELECT block_id FROM inodes WHERE id=?) ' + 'WHERE id=?', (id_, id_new)) + + processed += db.execute('REPLACE INTO blocks (id, hash, refcount, size, obj_id) ' + 'SELECT blocks.id, hash, blocks.refcount+1, blocks.size, obj_id ' + 'FROM inodes JOIN blocks ON block_id = blocks.id ' + 'WHERE inodes.id = ? AND block_id IS NOT NULL', (id_new,)) + + processed += db.execute('REPLACE INTO blocks (id, hash, refcount, size, obj_id) ' + 'SELECT id, hash, refcount+1, size, obj_id ' + 'FROM inode_blocks JOIN blocks ON block_id = id ' + 'WHERE inode = ?', (id_new,)) + if db.has_val('SELECT 1 FROM contents WHERE parent_inode=?', (id_,)): queue.append((id_, id_new, 0)) else: id_new = id_cache[id_] self.inodes[id_new].refcount += 1 - db.execute('INSERT INTO contents (name, inode, parent_inode) VALUES(?, ?, ?)', - (name, id_new, target_id)) + db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?, ?, ?)', + (name_id, id_new, target_id)) + db.execute('UPDATE names SET refcount=refcount+1 WHERE id=?', (name_id,)) processed += 1 @@ -446,29 +416,15 @@ class Operations(llfuse.Operations): processed = 0 llfuse.lock.yield_() stamp = time.time() - - # If we replicated blocks whose associated objects where still in - # transit, we have to wait for the transit to complete before we make - # the replicated tree visible to the user. Otherwise access to the newly - # created blocks will raise a NoSuchObject exception. - while in_transit: - log.debug('copy_tree(%d, %d): in_transit: %s', - src_inode.id, target_inode.id, in_transit) - in_transit = [ x for x in in_transit - if x in self.cache.upload_manager.in_transit ] - if in_transit: - self.cache.upload_manager.join_one() - # Make replication visible self.db.execute('UPDATE contents SET parent_inode=? WHERE parent_inode=?', - (target_inode.id, tmp.id)) + (target_inode.id, tmp.id)) del self.inodes[tmp.id] llfuse.invalidate_inode(target_inode.id) log.debug('copy_tree(%d, %d): end', src_inode.id, target_inode.id) - def unlink(self, id_p, name): inode = self.lookup(id_p, name) @@ -509,8 +465,10 @@ class Operations(llfuse.Operations): if self.inodes[id_p].locked and not force: raise FUSEError(errno.EPERM) - self.db.execute("DELETE FROM contents WHERE name=? AND parent_inode=?", - (name, id_p)) + name_id = self._del_name(name) + self.db.execute("DELETE FROM contents WHERE name_id=? AND parent_inode=?", + (name_id, id_p)) + inode = self.inodes[id_] inode.refcount -= 1 inode.ctime = timestamp @@ -524,6 +482,7 @@ class Operations(llfuse.Operations): # Since the inode is not open, it's not possible that new blocks # get created at this point and we can safely delete the inode self.db.execute('DELETE FROM ext_attributes WHERE inode=?', (id_,)) + self.db.execute('DELETE FROM symlink_targets WHERE inode=?', (id_,)) del self.inodes[id_] def symlink(self, id_p, name, target, ctx): @@ -536,7 +495,11 @@ class Operations(llfuse.Operations): # with this size. If the kernel ever learns to open and read # symlinks directly, it will read the corresponding number of \0 # bytes. - return self._create(id_p, name, mode, ctx, target=target, size=len(target)) + inode = self._create(id_p, name, mode, ctx, size=len(target)) + self.db.execute('INSERT INTO symlink_targets (inode, target) VALUES(?,?)', + (inode.id, target)) + + return inode def rename(self, id_p_old, name_old, id_p_new, name_new): if name_new == CTRL_NAME or name_old == CTRL_NAME: @@ -569,12 +532,46 @@ class Operations(llfuse.Operations): self._rename(id_p_old, name_old, id_p_new, name_new) + def _add_name(self, name): + '''Get id for *name* and increase refcount + + Name is inserted in table if it does not yet exist. + ''' + + try: + name_id = self.db.get_val('SELECT id FROM names WHERE name=?', (name,)) + except NoSuchRowError: + name_id = self.db.rowid('INSERT INTO names (name, refcount) VALUES(?,?)', + (name, 1)) + else: + self.db.execute('UPDATE names SET refcount=refcount+1 WHERE id=?', (name_id,)) + return name_id + + def _del_name(self, name): + '''Decrease refcount for *name* + + Name is removed from table if refcount drops to zero. Returns the + (possibly former) id of the name. + ''' + + (name_id, refcount) = self.db.get_row('SELECT id, refcount FROM names WHERE name=?', (name,)) + + if refcount > 1: + self.db.execute('UPDATE names SET refcount=refcount-1 WHERE id=?', (name_id,)) + else: + self.db.execute('DELETE FROM names WHERE id=?', (name_id,)) + + return name_id + def _rename(self, id_p_old, name_old, id_p_new, name_new): timestamp = time.time() - self.db.execute("UPDATE contents SET name=?, parent_inode=? WHERE name=? " - "AND parent_inode=?", (name_new, id_p_new, - name_old, id_p_old)) + name_id_new = self._add_name(name_new) + name_id_old = self._del_name(name_old) + + self.db.execute("UPDATE contents SET name_id=?, parent_inode=? WHERE name_id=? " + "AND parent_inode=?", (name_id_new, id_p_new, + name_id_old, id_p_old)) inode_p_old = self.inodes[id_p_old] inode_p_new = self.inodes[id_p_new] @@ -594,13 +591,15 @@ class Operations(llfuse.Operations): raise llfuse.FUSEError(errno.EINVAL) # Replace target - self.db.execute("UPDATE contents SET inode=? WHERE name=? AND parent_inode=?", - (id_old, name_new, id_p_new)) + name_id_new = self.db.get_val('SELECT id FROM names WHERE name=?', (name_new,)) + self.db.execute("UPDATE contents SET inode=? WHERE name_id=? AND parent_inode=?", + (id_old, name_id_new, id_p_new)) # Delete old name - self.db.execute('DELETE FROM contents WHERE name=? AND parent_inode=?', - (name_old, id_p_old)) + name_id_old = self._del_name(name_old) + self.db.execute('DELETE FROM contents WHERE name_id=? AND parent_inode=?', + (name_id_old, id_p_old)) inode_new = self.inodes[id_new] inode_new.refcount -= 1 @@ -620,6 +619,7 @@ class Operations(llfuse.Operations): # Since the inode is not open, it's not possible that new blocks # get created at this point and we can safely delete the inode self.db.execute('DELETE FROM ext_attributes WHERE inode=?', (id_new,)) + self.db.execute('DELETE FROM symlink_targets WHERE inode=?', (id_new,)) del self.inodes[id_new] @@ -643,8 +643,8 @@ class Operations(llfuse.Operations): inode_p.ctime = timestamp inode_p.mtime = timestamp - self.db.execute("INSERT INTO contents (name, inode, parent_inode) VALUES(?,?,?)", - (new_name, id_, new_id_p)) + self.db.execute("INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)", + (self._add_name(new_name), id_, new_id_p)) inode = self.inodes[id_] inode.refcount += 1 inode.ctime = timestamp @@ -684,7 +684,6 @@ class Operations(llfuse.Operations): except NoSuchObject as exc: log.warn('Backend lost block %d of inode %d (id %s)!', last_block, id_, exc.key) - self.encountered_errors = True raise FUSEError(errno.EIO) except ChecksumError as exc: @@ -727,11 +726,14 @@ class Operations(llfuse.Operations): def extstat(self): '''Return extended file system statistics''' + # Flush inode cache to get better estimate of total fs size + self.inodes.flush() + entries = self.db.get_val("SELECT COUNT(rowid) FROM contents") blocks = self.db.get_val("SELECT COUNT(id) FROM objects") inodes = self.db.get_val("SELECT COUNT(id) FROM inodes") fs_size = self.db.get_val('SELECT SUM(size) FROM inodes') or 0 - dedup_size = self.db.get_val('SELECT SUM(size) FROM objects') or 0 + dedup_size = self.db.get_val('SELECT SUM(size) FROM blocks') or 0 compr_size = self.db.get_val('SELECT SUM(compr_size) FROM objects') or 0 return struct.pack('QQQQQQQ', entries, blocks, inodes, fs_size, dedup_size, @@ -744,7 +746,7 @@ class Operations(llfuse.Operations): # Get number of blocks & inodes blocks = self.db.get_val("SELECT COUNT(id) FROM objects") inodes = self.db.get_val("SELECT COUNT(id) FROM inodes") - size = self.db.get_val('SELECT SUM(size) FROM objects') + size = self.db.get_val('SELECT SUM(size) FROM blocks') if size is None: size = 0 @@ -759,10 +761,7 @@ class Operations(llfuse.Operations): # size of fs in f_frsize units # (since backend is supposed to be unlimited, always return a half-full filesystem, # but at least 50 GB) - if stat_.f_bsize != 0: - total_blocks = int(max(2 * blocks, 50 * 1024 ** 3 // stat_.f_bsize)) - else: - total_blocks = 2 * blocks + total_blocks = int(max(2 * blocks, 50 * 1024 ** 3 // stat_.f_frsize)) stat_.f_blocks = total_blocks stat_.f_bfree = total_blocks - blocks @@ -800,7 +799,7 @@ class Operations(llfuse.Operations): self.open_inodes[inode.id] += 1 return (inode.id, inode) - def _create(self, id_p, name, mode, ctx, rdev=0, target=None, size=0): + def _create(self, id_p, name, mode, ctx, rdev=0, size=0): if name == CTRL_NAME: log.warn('Attempted to create s3ql control file at %s', get_path(id_p, self.db, name)) @@ -822,13 +821,13 @@ class Operations(llfuse.Operations): try: inode = self.inodes.create_inode(mtime=timestamp, ctime=timestamp, atime=timestamp, uid=ctx.uid, gid=ctx.gid, mode=mode, refcount=1, - rdev=rdev, target=target, size=size) + rdev=rdev, size=size) except OutOfInodesError: log.warn('Could not find a free inode') raise FUSEError(errno.ENOSPC) - self.db.execute("INSERT INTO contents(name, inode, parent_inode) VALUES(?,?,?)", - (name, inode.id, id_p)) + self.db.execute("INSERT INTO contents(name_id, inode, parent_inode) VALUES(?,?,?)", + (self._add_name(name), inode.id, id_p)) return inode @@ -889,7 +888,6 @@ class Operations(llfuse.Operations): except NoSuchObject as exc: log.warn('Backend lost block %d of inode %d (id %s)!', blockno, id_, exc.key) - self.encountered_errors = True raise FUSEError(errno.EIO) except ChecksumError as exc: @@ -957,7 +955,6 @@ class Operations(llfuse.Operations): except NoSuchObject as exc: log.warn('Backend lost block %d of inode %d (id %s)!', blockno, id_, exc.key) - self.encountered_errors = True raise FUSEError(errno.EIO) except ChecksumError as exc: @@ -987,7 +984,7 @@ class Operations(llfuse.Operations): inode = self.inodes[fh] if inode.refcount == 0: self.cache.remove(inode.id, 0, - int(math.ceil(inode.size // self.blocksize))) + int(math.ceil(inode.size / self.blocksize))) # Since the inode is not open, it's not possible that new blocks # get created at this point and we can safely delete the in del self.inodes[fh] @@ -1016,39 +1013,4 @@ def update_logging(level, modules): else: logging.disable(logging.DEBUG) root_logger.setLevel(level) - - -class InodeFlushThread(ExceptionStoringThread): - ''' - Periodically commit dirty inodes. - - This class uses the llfuse global lock. When calling objects - passed in the constructor, the global lock is acquired first. - ''' - - def __init__(self, cache): - super(InodeFlushThread, self).__init__() - self.cache = cache - self.stop_event = threading.Event() - self.name = 'Inode Flush Thread' - self.daemon = True - - def run_protected(self): - log.debug('FlushThread: start') - - while not self.stop_event.is_set(): - with lock: - self.cache.flush() - self.stop_event.wait(5) - log.debug('FlushThread: end') - - def stop(self): - '''Wait for thread to finish, raise any occurred exceptions. - - This method releases the global lock. - ''' - - self.stop_event.set() - with lock_released: - self.join_and_raise() - +
\ No newline at end of file diff --git a/src/s3ql/fsck.py b/src/s3ql/fsck.py index 3b020ae..6200759 100644 --- a/src/s3ql/fsck.py +++ b/src/s3ql/fsck.py @@ -3,22 +3,21 @@ fsck.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import - -import os +from .backends.common import NoSuchObject +from .common import ROOT_INODE, CTRL_INODE, inode_for_path, sha256_fh, get_path +from .database import NoSuchRowError from os.path import basename -import stat -import time +from s3ql.backends.common import CompressFilter import logging +import os import re -from .database import NoSuchRowError -from .backends.common import NoSuchObject -from .common import (ROOT_INODE, CTRL_INODE, inode_for_path, sha256_fh, get_path) - -__all__ = [ "Fsck" ] +import shutil +import stat +import time log = logging.getLogger("fsck") @@ -35,35 +34,94 @@ class Fsck(object): self.found_errors = False self.uncorrectable_errors = False self.blocksize = param['blocksize'] - self.conn = conn + self.conn = conn + + # Set of blocks that have been unlinked by check_cache. + # check_block_refcounts() will not report errors if these blocks still + # exist even though they have refcount=0 + self.unlinked_blocks = set() + + # Similarly for objects + self.unlinked_objects = set() def check(self): """Check file system - Sets module variable `found_errors`. + Sets instance variable `found_errors`. """ - - self.check_cache() - self.check_lof() - self.check_contents() - self.check_loops() - self.check_inode_refcount() - self.check_inode_sizes() - self.check_inode_unix() - self.check_obj_refcounts() - self.check_keylist() - + + # Create indices required for reference checking + log.info('Creating temporary extra indices...') + self.conn.execute('CREATE INDEX tmp1 ON blocks(obj_id)') + self.conn.execute('CREATE INDEX tmp2 ON inode_blocks(block_id)') + self.conn.execute('CREATE INDEX tmp3 ON inodes(block_id)') + self.conn.execute('CREATE INDEX tmp4 ON contents(inode)') + try: + self.check_foreign_keys() + self.check_cache() + self.check_lof() + self.check_name_refcount() + self.check_contents() + self.check_inode_refcount() + self.check_loops() + self.check_inode_sizes() + self.check_inode_unix() + self.check_block_refcount() + self.check_obj_refcounts() + self.check_keylist() + finally: + log.info('Dropping temporary indices...') + for idx in ('tmp1', 'tmp2', 'tmp3', 'tmp4'): + self.conn.execute('DROP INDEX %s' % idx) def log_error(self, *a, **kw): '''Log file system error if not expected''' if not self.expect_errors: return log.warn(*a, **kw) - + + def check_foreign_keys(self): + '''Check for referential integrity + + Checks that all foreign keys in the SQLite tables actually resolve. + This is necessary, because we disable runtime checking by SQLite + for performance reasons. + ''' + + log.info("Checking referential integrity...") + + errors_found = True + while errors_found: + errors_found = False + + for (table,) in self.conn.query("SELECT name FROM sqlite_master WHERE type='table'"): + for row in self.conn.query('PRAGMA foreign_key_list(%s)' % table): + sql_objs = { 'src_table': table, + 'dst_table': row[2], + 'src_col': row[3], + 'dst_col': row[4] } + to_delete = [] + for (val,) in self.conn.query('SELECT %(src_table)s.%(src_col)s ' + 'FROM %(src_table)s LEFT JOIN %(dst_table)s ' + 'ON %(src_table)s.%(src_col)s = %(dst_table)s.%(dst_col)s ' + 'WHERE %(dst_table)s.%(dst_col)s IS NULL ' + 'AND %(src_table)s.%(src_col)s IS NOT NULL' + % sql_objs): + self.found_errors = True + sql_objs['val'] = val + self.log_error('%(src_table)s.%(src_col)s refers to non-existing key %(val)s ' + 'in %(dst_table)s.%(dst_col)s, deleting.', sql_objs) + to_delete.append(val) + + for val in to_delete: + self.conn.execute('DELETE FROM %(src_table)s WHERE %(src_col)s = ?' + % sql_objs, (val,)) + if to_delete: + errors_found = True + def check_cache(self): """Commit uncommitted cache files""" - log.info("Checking cached objects...") if not os.path.exists(self.cachedir): return @@ -71,71 +129,70 @@ class Fsck(object): for filename in os.listdir(self.cachedir): self.found_errors = True - match = re.match('^inode_(\\d+)_block_(\\d+)(\\.d)?$', filename) + match = re.match('^(\\d+)-(\\d+)$', filename) if match: inode = int(match.group(1)) blockno = int(match.group(2)) - dirty = match.group(3) == '.d' else: raise RuntimeError('Strange file in cache directory: %s' % filename) - - if not dirty: - self.log_error('Removing cached block %d of inode %d', blockno, inode) - os.unlink(os.path.join(self.cachedir, filename)) - continue - self.log_error("Committing changed block %d of inode %d to backend", + self.log_error("Committing block %d of inode %d to backend", blockno, inode) fh = open(os.path.join(self.cachedir, filename), "rb") - fh.seek(0, 2) - size = fh.tell() - fh.seek(0) + size = os.fstat(fh.fileno()).st_size hash_ = sha256_fh(fh) try: - obj_id = self.conn.get_val('SELECT id FROM objects WHERE hash=?', (hash_,)) + (block_id, obj_id) = self.conn.get_row('SELECT id, obj_id FROM blocks WHERE hash=?', (hash_,)) except NoSuchRowError: - obj_id = self.conn.rowid('INSERT INTO objects (refcount, hash, size) VALUES(?, ?, ?)', - (1, hash_, size)) - self.bucket.store_fh('s3ql_data_%d' % obj_id, fh) + obj_id = self.conn.rowid('INSERT INTO objects (refcount) VALUES(1)') + block_id = self.conn.rowid('INSERT INTO blocks (refcount, hash, obj_id, size) ' + 'VALUES(?, ?, ?, ?)', (1, hash_, obj_id, size)) + with self.bucket.open_write('s3ql_data_%d' % obj_id) as dest: + fh.seek(0) + shutil.copyfileobj(fh, dest) + + if isinstance(dest, CompressFilter): + obj_size = dest.compr_size + else: + obj_size = fh.tell() + self.conn.execute('UPDATE objects SET compr_size=? WHERE id=?', + (obj_size, obj_id)) else: - # Verify that this object actually exists - if self.bucket.contains('s3ql_data_%d' % obj_id): - self.conn.execute('UPDATE objects SET refcount=refcount+1 WHERE id=?', - (obj_id,)) - else: - # We don't want to delete the vanished object here, because - # check_keylist will do a better job and print all affected - # inodes. However, we need to reset the hash so that we can - # insert the new object. - self.conn.execute('UPDATE objects SET hash=NULL WHERE id=?', (obj_id,)) - obj_id = self.conn.rowid('INSERT INTO objects (refcount, hash, size) VALUES(?, ?, ?)', - (1, hash_, size)) - self.bucket.store_fh('s3ql_data_%d' % obj_id, fh) + self.conn.execute('UPDATE blocks SET refcount=refcount+1 WHERE id=?', (block_id,)) + try: - old_obj_id = self.conn.get_val('SELECT obj_id FROM blocks WHERE inode=? AND blockno=?', - (inode, blockno)) + if blockno == 0: + old_block_id = self.conn.get_val('SELECT block_id FROM inodes ' + 'WHERE id=? AND block_id IS NOT NULL', (inode,)) + else: + old_block_id = self.conn.get_val('SELECT block_id FROM inode_blocks ' + 'WHERE inode=? AND blockno=?', (inode, blockno)) except NoSuchRowError: - self.conn.execute('INSERT INTO blocks (obj_id, inode, blockno) VALUES(?,?,?)', - (obj_id, inode, blockno)) + if blockno == 0: + self.conn.execute('UPDATE inodes SET block_id=? WHERE id=?', + (block_id, inode)) + else: + self.conn.execute('INSERT INTO inode_blocks (block_id, inode, blockno) VALUES(?,?,?)', + (block_id, inode, blockno)) else: - self.conn.execute('UPDATE blocks SET obj_id=? WHERE inode=? AND blockno=?', - (obj_id, inode, blockno)) - - refcount = self.conn.get_val('SELECT refcount FROM objects WHERE id=?', - (old_obj_id,)) - if refcount > 1: - self.conn.execute('UPDATE objects SET refcount=refcount-1 WHERE id=?', - (old_obj_id,)) + if blockno == 0: + self.conn.execute('UPDATE inodes SET block_id=? WHERE id=?', + (block_id, inode)) else: - # Don't delete yet, maybe it's still referenced - pass - - + self.conn.execute('UPDATE inode_blocks SET block_id=? WHERE inode=? AND blockno=?', + (block_id, inode, blockno)) + + # We just decrease the refcount, but don't take any action + # because the reference count might be wrong + self.conn.execute('UPDATE blocks SET refcount=refcount-1 WHERE id=?', + (old_block_id,)) + self.unlinked_blocks.add(old_block_id) + fh.close() os.unlink(os.path.join(self.cachedir, filename)) @@ -147,45 +204,44 @@ class Fsck(object): timestamp = time.time() - time.timezone try: - inode_l = self.conn.get_val("SELECT inode FROM contents WHERE name=? AND parent_inode=?", - (b"lost+found", ROOT_INODE)) + (inode_l, name_id) = self.conn.get_row("SELECT inode, name_id FROM contents_v " + "WHERE name=? AND parent_inode=?", (b"lost+found", ROOT_INODE)) except NoSuchRowError: self.found_errors = True self.log_error("Recreating missing lost+found directory") inode_l = self.conn.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount) " - "VALUES (?,?,?,?,?,?,?)", - (stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR, - os.getuid(), os.getgid(), timestamp, timestamp, timestamp, 1)) - self.conn.execute("INSERT INTO contents (name, inode, parent_inode) VALUES(?,?,?)", - (b"lost+found", inode_l, ROOT_INODE)) + "VALUES (?,?,?,?,?,?,?)", + (stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR, + os.getuid(), os.getgid(), timestamp, timestamp, timestamp, 1)) + self.conn.execute("INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)", + (self._add_name(b"lost+found"), inode_l, ROOT_INODE)) mode = self.conn.get_val('SELECT mode FROM inodes WHERE id=?', (inode_l,)) if not stat.S_ISDIR(mode): self.found_errors = True self.log_error('/lost+found is not a directory! Old entry will be saved as ' - '/lost+found/inode-%s*', inode_l) + '/lost+found/inode-%s*', inode_l) # We leave the old inode unassociated, so that it will be added # to lost+found later on. inode_l = self.conn.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount) " - "VALUES (?,?,?,?,?,?,?)", - (stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR, - os.getuid(), os.getgid(), timestamp, timestamp, timestamp, 2)) - self.conn.execute('UPDATE contents SET inode=? WHERE name=? AND parent_inode=?', - (inode_l, b"lost+found", ROOT_INODE)) + "VALUES (?,?,?,?,?,?,?)", + (stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR, + os.getuid(), os.getgid(), timestamp, timestamp, timestamp, 1)) + self.conn.execute('UPDATE contents SET inode=? WHERE name_id=? AND parent_inode=?', + (inode_l, name_id, ROOT_INODE)) def check_contents(self): """Check direntry names""" log.info('Checking directory entry names...') - for (name, id_p) in self.conn.query('SELECT name, parent_inode FROM contents ' + for (name, id_p) in self.conn.query('SELECT name, parent_inode FROM contents_v ' 'WHERE LENGTH(name) > 255'): path = get_path(id_p, self.conn, name) self.log_error('Entry name %s... in %s has more than 255 characters, ' - 'this could cause problems', - name[:40], path[:-len(name)]) + 'this could cause problems', name[:40], path[:-len(name)]) self.found_errors = True @@ -226,41 +282,62 @@ class Fsck(object): log.info('Checking inodes (sizes)...') self.conn.execute('CREATE TEMPORARY TABLE min_sizes ' - '(id INTEGER PRIMARY KEY, min_size INTEGER NOT NULL)') + '(id INTEGER PRIMARY KEY, min_size INTEGER NOT NULL)') try: - self.conn.execute('''INSERT INTO min_sizes (id, min_size) - SELECT inode, MAX(blockno * ? + size) - FROM blocks JOIN objects ON obj_id == id - GROUP BY inode''', - (self.blocksize,)) + self.conn.execute(''' + INSERT INTO min_sizes (id, min_size) + SELECT inode, MAX(blockno * ? + size) + FROM inode_blocks_v JOIN blocks ON block_id == blocks.id + GROUP BY inode''', (self.blocksize,)) self.conn.execute(''' CREATE TEMPORARY TABLE wrong_sizes AS SELECT id, size, min_size - FROM inodes JOIN min_sizes USING (id) + FROM inodes JOIN min_sizes USING (id) WHERE size < min_size''') for (id_, size_old, size) in self.conn.query('SELECT * FROM wrong_sizes'): self.found_errors = True self.log_error("Size of inode %d (%s) does not agree with number of blocks, " - "setting from %d to %d", - id_, get_path(id_, self.conn), size_old, size) + "setting from %d to %d", + id_, get_path(id_, self.conn), size_old, size) self.conn.execute("UPDATE inodes SET size=? WHERE id=?", (size, id_)) finally: self.conn.execute('DROP TABLE min_sizes') self.conn.execute('DROP TABLE IF EXISTS wrong_sizes') + def _add_name(self, name): + '''Get id for *name* and increase refcount + + Name is inserted in table if it does not yet exist. + ''' + + try: + name_id = self.conn.get_val('SELECT id FROM names WHERE name=?', (name,)) + except NoSuchRowError: + name_id = self.conn.rowid('INSERT INTO names (name, refcount) VALUES(?,?)', + (name, 1)) + else: + self.conn.execute('UPDATE names SET refcount=refcount+1 WHERE id=?', (name_id,)) + return name_id + + def _del_name(self, name_id): + '''Decrease refcount for name_id, remove if it reaches 0''' + + self.conn.execute('UPDATE names SET refcount=refcount-1 WHERE id=?', (name_id,)) + self.conn.execute('DELETE FROM names WHERE refcount=0 AND id=?', (name_id,)) + def check_inode_refcount(self): """Check inode reference counters""" log.info('Checking inodes (refcounts)...') - + self.conn.execute('CREATE TEMPORARY TABLE refcounts ' - '(id INTEGER PRIMARY KEY, refcount INTEGER NOT NULL)') + '(id INTEGER PRIMARY KEY, refcount INTEGER NOT NULL)') try: self.conn.execute('INSERT INTO refcounts (id, refcount) ' - 'SELECT inode, COUNT(name) FROM contents GROUP BY inode') + 'SELECT inode, COUNT(name_id) FROM contents GROUP BY inode') self.conn.execute(''' CREATE TEMPORARY TABLE wrong_refcounts AS @@ -278,8 +355,8 @@ class Fsck(object): if cnt is None: (id_p, name) = self.resolve_free(b"/lost+found", b"inode-%d" % id_) self.log_error("Inode %d not referenced, adding as /lost+found/%s", id_, name) - self.conn.execute("INSERT INTO contents (name, inode, parent_inode) " - "VALUES (?,?,?)", (basename(name), id_, id_p)) + self.conn.execute("INSERT INTO contents (name_id, inode, parent_inode) " + "VALUES (?,?,?)", (self._add_name(basename(name)), id_, id_p)) self.conn.execute("UPDATE inodes SET refcount=? WHERE id=?", (1, id_)) else: @@ -289,8 +366,99 @@ class Fsck(object): finally: self.conn.execute('DROP TABLE refcounts') self.conn.execute('DROP TABLE IF EXISTS wrong_refcounts') + + + def check_block_refcount(self): + """Check block reference counters""" + log.info('Checking blocks (refcounts)...') + self.conn.execute('CREATE TEMPORARY TABLE refcounts ' + '(id INTEGER PRIMARY KEY, refcount INTEGER NOT NULL)') + try: + self.conn.execute(''' + INSERT INTO refcounts (id, refcount) + SELECT block_id, COUNT(blockno) + FROM inode_blocks_v + GROUP BY block_id + ''') + + self.conn.execute(''' + CREATE TEMPORARY TABLE wrong_refcounts AS + SELECT id, refcounts.refcount, blocks.refcount, obj_id + FROM blocks LEFT JOIN refcounts USING (id) + WHERE blocks.refcount != refcounts.refcount + OR refcounts.refcount IS NULL''') + + for (id_, cnt, cnt_old, obj_id) in self.conn.query('SELECT * FROM wrong_refcounts'): + if cnt is None and id_ in self.unlinked_blocks and cnt_old == 0: + # Block was unlinked by check_cache and can now really be + # removed (since we have checked that there are truly no + # other references) + self.conn.execute('DELETE FROM blocks WHERE id=?', (id_,)) + + # We can't remove associated objects yet, because their refcounts + # might be wrong, too. + self.conn.execute('UPDATE objects SET refcount=refcount-1 WHERE id=?', (obj_id,)) + self.unlinked_objects.add(obj_id) + + elif cnt is None: + self.found_errors = True + (id_p, name) = self.resolve_free(b"/lost+found", b"block-%d" % id_) + self.log_error("Block %d not referenced, adding as /lost+found/%s", id_, name) + timestamp = time.time() - time.timezone + inode = self.conn.rowid(""" + INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,block_id) + VALUES (?,?,?,?,?,?,?,?)""", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR, + os.getuid(), os.getgid(), timestamp, timestamp, timestamp, 1, id_)) + self.conn.execute("INSERT INTO contents (name_id, inode, parent_inode) VALUES (?,?,?)", + (self._add_name(basename(name)), inode, id_p)) + self.conn.execute("UPDATE blocks SET refcount=? WHERE id=?", (1, id_)) + + else: + self.found_errors = True + self.log_error("Block %d has wrong reference count, setting from %d to %d", + id_, cnt_old, cnt) + self.conn.execute("UPDATE blocks SET refcount=? WHERE id=?", (cnt, id_)) + finally: + self.conn.execute('DROP TABLE refcounts') + self.conn.execute('DROP TABLE IF EXISTS wrong_refcounts') + + + def check_name_refcount(self): + """Check name reference counters""" + + log.info('Checking names (refcounts)...') + + self.conn.execute('CREATE TEMPORARY TABLE refcounts ' + '(id INTEGER PRIMARY KEY, refcount INTEGER NOT NULL)') + try: + self.conn.execute(''' + INSERT INTO refcounts (id, refcount) + SELECT name_id, COUNT(name_id) FROM contents GROUP BY name_id''') + + self.conn.execute(''' + CREATE TEMPORARY TABLE wrong_refcounts AS + SELECT id, refcounts.refcount, names.refcount + FROM names LEFT JOIN refcounts USING (id) + WHERE names.refcount != refcounts.refcount + OR refcounts.refcount IS NULL''') + + for (id_, cnt, cnt_old) in self.conn.query('SELECT * FROM wrong_refcounts'): + self.found_errors = True + if cnt is None: + self.log_error("Name %d not referenced, removing (old refcount: %d)", + id_, cnt_old) + self.conn.execute('DELETE FROM names WHERE id=?', (id_,)) + else: + self.log_error("Name %d has wrong reference count, setting from %d to %d", + id_, cnt_old, cnt) + self.conn.execute("UPDATE names SET refcount=? WHERE id=?", (cnt, id_)) + finally: + self.conn.execute('DROP TABLE refcounts') + self.conn.execute('DROP TABLE IF EXISTS wrong_refcounts') + def check_inode_unix(self): """Check inode attributes for agreement with UNIX conventions @@ -310,9 +478,16 @@ class Fsck(object): log.info('Checking inodes (types)...') for (inode, mode, size, target, rdev) \ - in self.conn.query("SELECT id, mode, size, target, rdev FROM inodes"): + in self.conn.query("SELECT id, mode, size, target, rdev " + "FROM inodes LEFT JOIN symlink_targets ON id = inode"): - if stat.S_ISLNK(mode) and size != len(target): + if stat.S_ISLNK(mode) and target is None: + self.found_errors = True + self.log_error('Inode %d (%s): symlink does not have target. ' + 'This is probably going to confuse your system!', + inode, get_path(inode, self.conn)) + + if stat.S_ISLNK(mode) and target is not None and size != len(target): self.found_errors = True self.log_error('Inode %d (%s): symlink size (%d) does not agree with target ' 'length (%d). This is probably going to confuse your system!', @@ -346,9 +521,10 @@ class Fsck(object): 'This is probably going to confuse your system!', inode, get_path(inode, self.conn)) - has_blocks = self.conn.has_val('SELECT 1 FROM blocks WHERE inode=? LIMIT 1', - (inode,)) - if has_blocks and not stat.S_ISREG(mode): + if (not stat.S_ISREG(mode) and + self.conn.has_val('SELECT 1 FROM inode_blocks WHERE inode=? ' + 'UNION SELECT 1 FROM inodes WHERE id=? ' + 'AND block_id IS NOT NULL', (inode, inode))): self.found_errors = True self.log_error('Inode %d (%s) is not a regular file but has data blocks. ' 'This is probably going to confuse your system!', @@ -361,10 +537,10 @@ class Fsck(object): log.info('Checking object reference counts...') self.conn.execute('CREATE TEMPORARY TABLE refcounts ' - '(id INTEGER PRIMARY KEY, refcount INTEGER NOT NULL)') + '(id INTEGER PRIMARY KEY, refcount INTEGER NOT NULL)') try: self.conn.execute('INSERT INTO refcounts (id, refcount) ' - 'SELECT obj_id, COUNT(inode) FROM blocks GROUP BY obj_id') + 'SELECT obj_id, COUNT(obj_id) FROM blocks GROUP BY obj_id') self.conn.execute(''' CREATE TEMPORARY TABLE wrong_refcounts AS @@ -374,19 +550,26 @@ class Fsck(object): OR refcounts.refcount IS NULL''') for (id_, cnt, cnt_old) in self.conn.query('SELECT * FROM wrong_refcounts'): - self.log_error("Object %s has invalid refcount, setting from %d to %d", - id_, cnt_old, cnt or 0) - self.found_errors = True - if cnt is not None: - self.conn.execute("UPDATE objects SET refcount=? WHERE id=?", - (cnt, id_)) - else: - # Orphaned object will be picked up by check_keylist + if cnt is None and id_ in self.unlinked_objects and cnt_old == 0: + # Object was unlinked by check_block_refcounts self.conn.execute('DELETE FROM objects WHERE id=?', (id_,)) + + else: + self.found_errors = True + self.log_error("Object %s has invalid refcount, setting from %d to %d", + id_, cnt_old, cnt or 0) + + if cnt is not None: + self.conn.execute("UPDATE objects SET refcount=? WHERE id=?", + (cnt, id_)) + else: + # Orphaned object will be picked up by check_keylist + self.conn.execute('DELETE FROM objects WHERE id=?', (id_,)) finally: self.conn.execute('DROP TABLE refcounts') self.conn.execute('DROP TABLE IF EXISTS wrong_refcounts') - + + def check_keylist(self): """Check the list of objects. @@ -398,11 +581,10 @@ class Fsck(object): log.info('Checking object list...') - lof_id = self.conn.get_val("SELECT inode FROM contents WHERE name=? AND parent_inode=?", - (b"lost+found", ROOT_INODE)) + lof_id = self.conn.get_val("SELECT inode FROM contents_v " + "WHERE name=? AND parent_inode=?", (b"lost+found", ROOT_INODE)) - # We use this table to keep track of the objects that we have - # seen + # We use this table to keep track of the objects that we have seen self.conn.execute("CREATE TEMP TABLE obj_ids (id INTEGER PRIMARY KEY)") try: for (i, obj_name) in enumerate(self.bucket.list('s3ql_data_')): @@ -411,38 +593,69 @@ class Fsck(object): log.info('..processed %d objects so far..', i) # We only bother with data objects - obj_id = int(obj_name[10:]) + try: + obj_id = int(obj_name[10:]) + except ValueError: + log.warn("Ignoring unexpected object %r", obj_name) + continue self.conn.execute('INSERT INTO obj_ids VALUES(?)', (obj_id,)) for (obj_id,) in self.conn.query('SELECT id FROM obj_ids ' - 'EXCEPT SELECT id FROM objects'): - self.found_errors = True - self.log_error("Deleting spurious object %d", obj_id) + 'EXCEPT SELECT id FROM objects'): try: - del self.bucket['s3ql_data_%d' % obj_id] + if obj_id in self.unlinked_objects: + del self.bucket['s3ql_data_%d' % obj_id] + else: + # TODO: Save the data in lost+found instead + del self.bucket['s3ql_data_%d' % obj_id] + self.found_errors = True + self.log_error("Deleted spurious object %d", obj_id) except NoSuchObject: - if self.bucket.read_after_write_consistent(): - raise + pass self.conn.execute('CREATE TEMPORARY TABLE missing AS ' - 'SELECT id FROM objects EXCEPT SELECT id FROM obj_ids') + 'SELECT id FROM objects EXCEPT SELECT id FROM obj_ids') + moved_inodes = set() for (obj_id,) in self.conn.query('SELECT * FROM missing'): + if (not self.bucket.is_list_create_consistent() + and ('s3ql_data_%d' % obj_id) in self.bucket): + # Object was just not in list yet + continue + self.found_errors = True self.log_error("object %s only exists in table but not in bucket, deleting", obj_id) - self.log_error("The following files may lack data and have been moved to /lost+found:") - for (id_,) in self.conn.query('SELECT inode FROM blocks WHERE obj_id=?', (obj_id,)): - for (name, id_p) in self.conn.query('SELECT name, parent_inode FROM contents ' - 'WHERE inode=?', (id_,)): + + for (id_,) in self.conn.query('SELECT inode FROM inode_blocks JOIN blocks ON block_id = id ' + 'WHERE obj_id=? ' + 'UNION ' + 'SELECT inodes.id FROM inodes JOIN blocks ON block_id = blocks.id ' + 'WHERE obj_id=? AND block_id IS NOT NULL', + (obj_id, obj_id)): + + # Same file may lack several blocks, but we want to move it + # only once + if id_ in moved_inodes: + continue + moved_inodes.add(id_) + + for (name, name_id, id_p) in self.conn.query('SELECT name, name_id, parent_inode ' + 'FROM contents_v WHERE inode=?', (id_,)): path = get_path(id_p, self.conn, name) - self.log_error(path) + self.log_error("File may lack data, moved to /lost+found: %s", path) (_, newname) = self.resolve_free(b"/lost+found", path[1:].replace('_', '__').replace('/', '_')) - self.conn.execute('UPDATE contents SET name=?, parent_inode=? ' - 'WHERE name=? AND parent_inode=?', - (newname, lof_id, name, id_p)) + self.conn.execute('UPDATE contents SET name_id=?, parent_inode=? ' + 'WHERE name_id=? AND parent_inode=?', + (self._add_name(newname), lof_id, name_id, id_p)) + self._del_name(name_id) + # Unlink missing blocks + for (block_id,) in self.conn.query('SELECT id FROM blocks WHERE obj_id=?', (obj_id,)): + self.conn.execute('DELETE FROM inode_blocks WHERE block_id=?', (block_id,)) + self.conn.execute('UPDATE inodes SET block_id = NULL WHERE block_id=?', (block_id,)) + self.conn.execute("DELETE FROM blocks WHERE obj_id=?", (obj_id,)) self.conn.execute("DELETE FROM objects WHERE id=?", (obj_id,)) finally: @@ -462,20 +675,22 @@ class Fsck(object): inode_p = inode_for_path(path, self.conn) + # Debugging http://code.google.com/p/s3ql/issues/detail?id=217 + # and http://code.google.com/p/s3ql/issues/detail?id=261 + if len(name) > 255-4: + name = '%s ... %s' % (name[0:120], name[-120:]) + i = 0 newname = name name += b'-' try: while True: - self.conn.get_val("SELECT inode FROM contents WHERE name=? AND parent_inode=?", - (newname, inode_p)) + self.conn.get_val("SELECT inode FROM contents_v " + "WHERE name=? AND parent_inode=?", (newname, inode_p)) i += 1 newname = name + bytes(i) except NoSuchRowError: pass - # Debugging http://code.google.com/p/s3ql/issues/detail?id=217 - assert len(newname) < 256 - return (inode_p, newname) diff --git a/src/s3ql/inode_cache.py b/src/s3ql/inode_cache.py index 149947f..7f8b090 100644 --- a/src/s3ql/inode_cache.py +++ b/src/s3ql/inode_cache.py @@ -3,7 +3,7 @@ inode_cache.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2010 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function, absolute_import @@ -19,13 +19,17 @@ log = logging.getLogger('inode_cache') CACHE_SIZE = 100 ATTRIBUTES = ('mode', 'refcount', 'uid', 'gid', 'size', 'locked', - 'rdev', 'target', 'atime', 'mtime', 'ctime', 'id') + 'rdev', 'atime', 'mtime', 'ctime', 'id') ATTRIBUTE_STR = ', '.join(ATTRIBUTES) UPDATE_ATTRS = ('mode', 'refcount', 'uid', 'gid', 'size', 'locked', - 'rdev', 'target', 'atime', 'mtime', 'ctime') + 'rdev', 'atime', 'mtime', 'ctime') UPDATE_STR = ', '.join('%s=?' % x for x in UPDATE_ATTRS) TIMEZONE = time.timezone + +# If True, new inodes are assigned randomly rather than sequentially +RANDOMIZE_INODES = False + class _Inode(object): '''An inode with its attributes''' @@ -192,37 +196,33 @@ class InodeCache(object): def create_inode(self, **kw): - inode = _Inode() - - for (key, val) in kw.iteritems(): - setattr(inode, key, val) - for i in ('atime', 'ctime', 'mtime'): kw[i] -= TIMEZONE - init_attrs = [ x for x in ATTRIBUTES if x in kw ] - - # We want to restrict inodes to 2^32, and we do not want to immediately - # reuse deleted inodes (so that the lack of generation numbers isn't too - # likely to cause problems with NFS) - sql = ('INSERT INTO inodes (id, %s) VALUES(?, %s)' - % (', '.join(init_attrs), ','.join('?' for _ in init_attrs))) - bindings = [ kw[x] for x in init_attrs ] - for _ in range(100): - # _Inode.id is not explicitly defined - #pylint: disable-msg=W0201 - inode.id = randint(0, 2 ** 32 - 1) - try: - self.db.execute(sql, [inode.id] + bindings) - except apsw.ConstraintError: - pass + bindings = [ kw[x] for x in ATTRIBUTES if x in kw ] + columns = ', '.join([ x for x in ATTRIBUTES if x in kw]) + values = ', '.join('?' * len(kw)) + + if RANDOMIZE_INODES: + # We want to restrict inodes to 2^32, and we do not want to immediately + # reuse deleted inodes (so that the lack of generation numbers isn't too + # likely to cause problems with NFS) + sql = 'INSERT INTO inodes (id, %s) VALUES(?, %s)' % (columns, values) + for _ in range(100): + id_ = randint(0, 2 ** 32 - 1) + try: + self.db.execute(sql, [id_] + bindings) + except apsw.ConstraintError: + pass + else: + break else: - break + raise OutOfInodesError() else: - raise OutOfInodesError() + id_ = self.db.rowid('INSERT INTO inodes (%s) VALUES(%s)' % (columns, values), + bindings) - - return self[inode.id] + return self[id_] def setattr(self, inode): @@ -236,7 +236,7 @@ class InodeCache(object): inode.ctime -= TIMEZONE self.db.execute("UPDATE inodes SET %s WHERE id=?" % UPDATE_STR, - [ getattr(inode, x) for x in UPDATE_ATTRS ] + [inode.id]) + [ getattr(inode, x) for x in UPDATE_ATTRS ] + [inode.id]) def flush_id(self, id_): if id_ in self.attrs: @@ -257,6 +257,9 @@ class InodeCache(object): else: del self.attrs[id_] self.setattr(inode) + + self.cached_rows = None + self.attrs = None def flush(self): '''Flush all entries to database''' diff --git a/src/s3ql/multi_lock.py b/src/s3ql/multi_lock.py deleted file mode 100644 index 865c16c..0000000 --- a/src/s3ql/multi_lock.py +++ /dev/null @@ -1,85 +0,0 @@ -''' -multi_lock.py - this file is part of S3QL (http://s3ql.googlecode.com) - -Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> - -This program can be distributed under the terms of the GNU LGPL. -''' - -from __future__ import division, print_function - -import threading -import logging -from contextlib import contextmanager -import time - -__all__ = [ "MultiLock" ] - -log = logging.getLogger("MultiLock") - - -# For debugging, can be set to impose an artifical delay when -# obtaining the lock. Introduced to debug a very -# timing-critical bug. -FAKEDELAY = False - -class MultiLock(object): - """Provides locking for multiple objects. - - This class provides locking for a dynamically changing set of objects: - The `acquire` and `release` methods have an additional argument, the - locking key. Only locks with the same key can actually see each other, - so that several threads can hold locks with different locking keys - at the same time. - - MultiLock instances can be used with `with` statements as - - lock = MultiLock() - with lock(key): - pass - - Note that it is actually possible for one thread to release a lock - that has been obtained by a different thread. This is not a bug, - but a feature used in `BlockCache._expire_parallel`. - """ - - def __init__(self): - self.locked_keys = set() - self.cond = threading.Condition() - - - @contextmanager - def __call__(self, *key): - self.acquire(*key) - try: - yield - finally: - self.release(*key) - - def acquire(self, *key): - '''Acquire lock for given key''' - - if FAKEDELAY: - time.sleep(FAKEDELAY) - - # Lock set of lockedkeys (global lock) - with self.cond: - - # Wait for given key becoming unused - while key in self.locked_keys: - self.cond.wait() - - # Mark it as used (local lock) - self.locked_keys.add(key) - - def release(self, *key): - """Release lock on given key""" - - # Lock set of locked keys (global lock) - with self.cond: - - # Mark key as free (release local lock) - self.locked_keys.remove(key) - - # Notify other threads - self.cond.notifyAll() diff --git a/src/s3ql/ordered_dict.py b/src/s3ql/ordered_dict.py index e52cb69..70e0333 100644 --- a/src/s3ql/ordered_dict.py +++ b/src/s3ql/ordered_dict.py @@ -3,7 +3,7 @@ ordered_dict.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function @@ -71,13 +71,9 @@ class OrderedDict(collections.MutableMapping): beginning of the list. If the key already exists, the position of the element does not change. - All methods are threadsafe and may be called concurrently - from several threads. - Attributes: ----------- :data: Backend dict object that holds OrderedDictElement instances - :lock: Global lock, required when rearranging the order :head: First element in list :tail: Last element in list @@ -85,26 +81,23 @@ class OrderedDict(collections.MutableMapping): def __init__(self): self.data = dict() - self.lock = threading.Lock() self.head = HeadSentinel() self.tail = TailSentinel(self.head) self.head.next = self.tail def __setitem__(self, key, value): - with self.lock: - if key in self.data: - self.data[key].value = value - else: - el = OrderedDictElement(key, value, next_=self.head.next, prev=self.head) - self.head.next.prev = el - self.head.next = el - self.data[key] = el + if key in self.data: + self.data[key].value = value + else: + el = OrderedDictElement(key, value, next_=self.head.next, prev=self.head) + self.head.next.prev = el + self.head.next = el + self.data[key] = el def __delitem__(self, key): - with self.lock: - el = self.data.pop(key) # exception can be passed on - el.prev.next = el.next - el.next.prev = el.prev + el = self.data.pop(key) # exception can be passed on + el.prev.next = el.next + el.next.prev = el.prev def __getitem__(self, key): return self.data[key].value @@ -137,83 +130,79 @@ class OrderedDict(collections.MutableMapping): def to_head(self, key): """Moves `key` to the head in the ordering """ - with self.lock: - el = self.data[key] - # Splice out - el.prev.next = el.next - el.next.prev = el.prev + el = self.data[key] + # Splice out + el.prev.next = el.next + el.next.prev = el.prev - # Insert back at front - el.next = self.head.next - el.prev = self.head + # Insert back at front + el.next = self.head.next + el.prev = self.head - self.head.next.prev = el - self.head.next = el + self.head.next.prev = el + self.head.next = el def to_tail(self, key): """Moves `key` to the end in the ordering """ - with self.lock: - el = self.data[key] - # Splice out - el.prev.next = el.next - el.next.prev = el.prev + el = self.data[key] + # Splice out + el.prev.next = el.next + el.next.prev = el.prev - # Insert back at end - el.next = self.tail - el.prev = self.tail.prev + # Insert back at end + el.next = self.tail + el.prev = self.tail.prev - self.tail.prev.next = el - self.tail.prev = el + self.tail.prev.next = el + self.tail.prev = el def pop_last(self): """Fetch and remove last element """ - with self.lock: - el = self.tail.prev - if el is self.head: - raise IndexError() - del self.data[el.key] - self.tail.prev = el.prev - el.prev.next = self.tail + el = self.tail.prev + if el is self.head: + raise IndexError() + + del self.data[el.key] + self.tail.prev = el.prev + el.prev.next = self.tail return el.value def get_last(self): """Fetch last element""" - with self.lock: - if self.tail.prev is self.head: - raise IndexError() - - return self.tail.prev.value + + if self.tail.prev is self.head: + raise IndexError() + + return self.tail.prev.value def pop_first(self): """Fetch and remove first element""" - with self.lock: - el = self.head.next - if el is self.tail: - raise IndexError - del self.data[el.key] - self.head.next = el.next - el.next.prev = self.head - return el.value + el = self.head.next + if el is self.tail: + raise IndexError + del self.data[el.key] + self.head.next = el.next + el.next.prev = self.head + + return el.value def get_first(self): """Fetch first element""" - with self.lock: - if self.head.next is self.tail: - raise IndexError() - - return self.head.next.value + if self.head.next is self.tail: + raise IndexError() + + return self.head.next.value def clear(self): '''Delete all elements''' - with self.lock: - self.data.clear() - self.head = HeadSentinel() - self.tail = TailSentinel(self.head) - self.head.next = self.tail + self.data.clear() + self.head = HeadSentinel() + self.tail = TailSentinel(self.head) + self.head.next = self.tail diff --git a/src/s3ql/parse_args.py b/src/s3ql/parse_args.py index 22a0d26..bc18d44 100644 --- a/src/s3ql/parse_args.py +++ b/src/s3ql/parse_args.py @@ -4,7 +4,7 @@ # # Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> # -# This program can be distributed under the terms of the GNU LGPL. +# This program can be distributed under the terms of the GNU GPLv3. # ''' This module provides a customized ArgumentParser class. Differences @@ -13,7 +13,7 @@ are: * a --version argument is added by default * convenience functions are available for adding --quiet, - --debug and --homedir options. + --debug, --cachedir, --log and --authfile options. * instead of the usage string one can pass a usage list. The first element will be prefixed with ``usage: `` as usual. Additional @@ -41,7 +41,8 @@ import s3ql import argparse import re import os -import textwrap +import logging.handlers +import sys __all__ = [ 'ArgumentParser', 'DEFAULT_USAGE'] @@ -152,28 +153,49 @@ class ArgumentParser(argparse.ArgumentParser): self.add_argument("--debug", action="store_const", const=['all'], help="activate debugging output") - def add_homedir(self): - self.add_argument("--homedir", type=str, metavar='<path>', + def add_authfile(self): + self.add_argument("--authfile", type=str, metavar='<path>', + default=os.path.expanduser("~/.s3ql/authinfo2"), + help='Read authentication credentials from this file ' + '(default: `~/.s3ql/authinfo2)`') + def add_cachedir(self): + self.add_argument("--cachedir", type=str, metavar='<path>', default=os.path.expanduser("~/.s3ql"), - help='Directory for log files, cache and authentication info. ' + help='Store cached data in this directory ' '(default: `~/.s3ql)`') - + + def add_log(self, default='none'): + def log_handler(s): + if s.lower() == 'none': + return None + elif s.lower() == 'syslog': + handler = logging.handlers.SysLogHandler('/dev/log') + formatter = logging.Formatter(os.path.basename(sys.argv[0]) + + '[%(process)s] %(threadName)s: ' + + '[%(name)s] %(message)s') + else: + fullpath = os.path.expanduser(s) + dirname = os.path.dirname(fullpath) + if dirname and not os.path.exists(dirname): + os.makedirs(dirname) + handler = logging.handlers.RotatingFileHandler(fullpath, + maxBytes=1024**2, backupCount=5) + formatter = logging.Formatter('%(asctime)s.%(msecs)03d [%(process)s] %(threadName)s: ' + '[%(name)s] %(message)s', datefmt="%Y-%m-%d %H:%M:%S") + + handler.setFormatter(formatter) + return handler + + self.add_argument("--log", type=log_handler, metavar='<target>', default=default, + help='Write logging info into this file. File will be rotated when ' + 'it reaches 1 MB, and at most 5 old log files will be kept. ' + 'Specify ``none`` to disable logging. Default: ``%(default)s``') + def add_storage_url(self): self.add_argument("storage_url", metavar='<storage-url>', type=storage_url_type, help='Storage URL of the backend that contains the file system') - - def add_ssl(self): - self.add_argument("--ssl", action="store_true", default=False, - help=textwrap.dedent('''\ - Use SSL when connecting to remote servers. This option - is not enabled by default, because for encrypted file - systems, all data is already encrypted anyway, and - authentication data is never transmitted in plaintext - even for unencrypted file systems. - ''')) - - + def add_subparsers(self, **kw): '''Pass parent and set prog to default usage message''' kw.setdefault('parser_class', argparse.ArgumentParser) @@ -204,4 +226,4 @@ def storage_url_type(s): return s else: msg = '%s is not a valid storage url.' % s - raise argparse.ArgumentTypeError(msg)
\ No newline at end of file + raise argparse.ArgumentTypeError(msg) diff --git a/src/s3ql/thread_group.py b/src/s3ql/thread_group.py deleted file mode 100644 index db077b4..0000000 --- a/src/s3ql/thread_group.py +++ /dev/null @@ -1,171 +0,0 @@ -''' -thread_group.py - this file is part of S3QL (http://s3ql.googlecode.com) - -Copyright (C) 2010 Nikolaus Rath <Nikolaus@rath.org> - -This program can be distributed under the terms of the GNU LGPL. -''' - -from __future__ import division, print_function, absolute_import - -import threading -import sys -import logging -from .common import ExceptionStoringThread -from llfuse import lock, lock_released - -log = logging.getLogger("thread_group") - -__all__ = [ 'Thread', 'ThreadGroup' ] - -class Thread(ExceptionStoringThread): - ''' - A thread that can be executed in a `ThreadGroup`. - ''' - - def __init__(self): - super(Thread, self).__init__() - self._group = None - - def run_protected(self): - '''Perform task asynchronously - - This method must be overridden in derived classes. It will be - called in a separate thread. Exceptions will be encapsulated in - `EmbeddedException` and re-raised when the thread is joined. - ''' - pass - - def start(self): - if not self._group: - raise ValueError('Must set _group attribute first') - - super(Thread, self).start() - - def run(self): - try: - try: - self.run_protected() - finally: - log.debug('thread: waiting for lock') - with self._group.lock: - log.debug('thread: calling notify()') - self._group.active_threads.remove(self) - self._group.finished_threads.append(self) - self._group.lock.notifyAll() - except: - self._exc_info = sys.exc_info() # This creates a circular reference chain - -class ThreadGroup(object): - '''Represents a group of threads. - - This class uses the llfuse global lock. Methods which release the - global lock have are marked as such in their docstring. - - Implementation Note: - -------------------- - - ThreadGroup instances have an internal lock object that is used to - communicate with the started threads. These threads do not hold the global - lock when they start and finish. To prevent deadlocks, the instance-level - lock must therefore only be acquired when the global lock is not held. - ''' - - def __init__(self, max_threads): - '''Initialize thread group - - `max_active` specifies the maximum number of running threads - in the group. - ''' - - self.max_threads = max_threads - self.active_threads = list() - self.finished_threads = list() - self.lock = threading.Condition(threading.RLock()) - - def add_thread(self, t, max_threads=None): - '''Add new thread - - `t` must be a `Thread` instance that has overridden - the `run_protected` and possibly `finalize` methods. - - This method waits until there are less than `max_threads` active - threads, then it starts a new thread executing `fn`. - - If `max_threads` is `None`, the `max_threads` value passed - to the constructor is used instead. - - This method may release the global lock. - ''' - - if not isinstance(t, Thread): - raise TypeError('Parameter must be `Thread` instance') - t._group = self - - if max_threads is None: - max_threads = self.max_threads - - lock.release() - with self.lock: - lock.acquire() - while len(self) >= max_threads: - self.join_one() - - self.active_threads.append(t) - - t.start() - - def join_one(self): - '''Wait for any one thread to finish - - If the thread terminated with an exception, the exception - is encapsulated in `EmbeddedException` and raised again. - - If there are no active threads, the call returns without doing - anything. - - If more than one thread has called `join_one`, a single thread - that finishes execution will cause all pending `join_one` calls - to return. - - This method may release the global lock. - ''' - - lock.release() - with self.lock: - lock.acquire() - - # Make sure that at least 1 thread is joined - if len(self) == 0: - return - - try: - t = self.finished_threads.pop() - except IndexError: - # Wait for thread to terminate - log.debug('join_one: wait()') - with lock_released: - self.lock.wait() - try: - t = self.finished_threads.pop() - except IndexError: - # Already joined by other waiting thread - return - - t.join_and_raise() - - def join_all(self): - '''Call join_one() until all threads have terminated - - This method may release the global lock. - ''' - - with self.lock: - while len(self) > 0: - self.join_one() - - def __len__(self): - lock.release() - with self.lock: - lock.acquire() - return len(self.active_threads) + len(self.finished_threads) diff --git a/src/s3ql/upload_manager.py b/src/s3ql/upload_manager.py deleted file mode 100644 index 1cf374d..0000000 --- a/src/s3ql/upload_manager.py +++ /dev/null @@ -1,387 +0,0 @@ -''' -upload_manager.py - this file is part of S3QL (http://s3ql.googlecode.com) - -Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> - -This program can be distributed under the terms of the GNU LGPL. -''' - -from __future__ import division, print_function, absolute_import - -from .backends.common import NoSuchObject -from .common import sha256_fh, TimeoutError -from .thread_group import ThreadGroup, Thread -from .database import NoSuchRowError -import logging -import threading -import os -import errno -from llfuse import lock -import time -from s3ql.common import EmbeddedException - -__all__ = [ "UploadManager", 'retry_exc', 'RemoveThread' ] - -# standard logger for this module -log = logging.getLogger("UploadManager") - - -MAX_UPLOAD_THREADS = 10 -MAX_COMPRESS_THREADS = 1 -MIN_TRANSIT_SIZE = 1024 * 1024 -class UploadManager(object): - ''' - Schedules and executes object uploads to make optimum usage - network bandwidth and CPU time. - - Methods which release the - global lock have are marked as such in their docstring. - - Attributes: - ----------- - - :encountered_errors: This attribute is set if some non-fatal errors - were encountered during asynchronous operations (for - example, an object that was supposed to be deleted did - not exist). - ''' - - def __init__(self, bucket, db, removal_queue): - self.upload_threads = ThreadGroup(MAX_UPLOAD_THREADS) - self.compress_threads = ThreadGroup(MAX_COMPRESS_THREADS) - self.removal_queue = removal_queue - self.bucket = bucket - self.db = db - self.transit_size = 0 - self.transit_size_lock = threading.Lock() - self.in_transit = set() - self.encountered_errors = False - - def add(self, el): - '''Upload cache entry `el` asynchronously - - Return (uncompressed) size of cache entry. - - This method releases the global lock. - ''' - - log.debug('UploadManager.add(%s): start', el) - - if (el.inode, el.blockno) in self.in_transit: - raise ValueError('Block already in transit') - - old_obj_id = el.obj_id - size = os.fstat(el.fileno()).st_size - el.seek(0) - if log.isEnabledFor(logging.DEBUG): - time_ = time.time() - hash_ = sha256_fh(el) - time_ = time.time() - time_ - if time_ != 0: - rate = size / (1024**2 * time_) - else: - rate = 0 - log.debug('UploadManager(inode=%d, blockno=%d): ' - 'hashed %d bytes in %.3f seconds, %.2f MB/s', - el.inode, el.blockno, size, time_, rate) - else: - hash_ = sha256_fh(el) - - try: - el.obj_id = self.db.get_val('SELECT id FROM objects WHERE hash=?', (hash_,)) - - except NoSuchRowError: - need_upload = True - el.obj_id = self.db.rowid('INSERT INTO objects (refcount, hash, size) VALUES(?, ?, ?)', - (1, hash_, size)) - log.debug('add(inode=%d, blockno=%d): created new object %d', - el.inode, el.blockno, el.obj_id) - - else: - need_upload = False - if old_obj_id == el.obj_id: - log.debug('add(inode=%d, blockno=%d): unchanged, obj_id=%d', - el.inode, el.blockno, el.obj_id) - el.dirty = False - el.modified_after_upload = False - os.rename(el.name + '.d', el.name) - return size - - log.debug('add(inode=%d, blockno=%d): (re)linking to %d', - el.inode, el.blockno, el.obj_id) - self.db.execute('UPDATE objects SET refcount=refcount+1 WHERE id=?', - (el.obj_id,)) - - to_delete = False - if old_obj_id is None: - log.debug('add(inode=%d, blockno=%d): no previous object', - el.inode, el.blockno) - self.db.execute('INSERT INTO blocks (obj_id, inode, blockno) VALUES(?,?,?)', - (el.obj_id, el.inode, el.blockno)) - else: - self.db.execute('UPDATE blocks SET obj_id=? WHERE inode=? AND blockno=?', - (el.obj_id, el.inode, el.blockno)) - refcount = self.db.get_val('SELECT refcount FROM objects WHERE id=?', - (old_obj_id,)) - if refcount > 1: - log.debug('add(inode=%d, blockno=%d): ' - 'decreased refcount for prev. obj: %d', - el.inode, el.blockno, old_obj_id) - self.db.execute('UPDATE objects SET refcount=refcount-1 WHERE id=?', - (old_obj_id,)) - else: - log.debug('add(inode=%d, blockno=%d): ' - 'prev. obj %d marked for removal', - el.inode, el.blockno, old_obj_id) - self.db.execute('DELETE FROM objects WHERE id=?', (old_obj_id,)) - to_delete = True - - if need_upload: - log.debug('add(inode=%d, blockno=%d): starting compression thread', - el.inode, el.blockno) - el.modified_after_upload = False - self.in_transit.add((el.inode, el.blockno)) - - # Create a new fd so that we don't get confused if another - # thread repositions the cursor (and do so before unlocking) - fh = open(el.name + '.d', 'rb') - self.compress_threads.add_thread(CompressThread(el, fh, self, size)) # Releases global lock - - else: - el.dirty = False - el.modified_after_upload = False - os.rename(el.name + '.d', el.name) - - if to_delete: - log.debug('add(inode=%d, blockno=%d): removing object %d', - el.inode, el.blockno, old_obj_id) - - try: - # Note: Old object can not be in transit - # Releases global lock - self.removal_queue.add_thread(RemoveThread(old_obj_id, self.bucket)) - except EmbeddedException as exc: - exc = exc.exc - if isinstance(exc, NoSuchObject): - log.warn('Backend seems to have lost object %s', exc.key) - self.encountered_errors = True - else: - raise - - log.debug('add(inode=%d, blockno=%d): end', el.inode, el.blockno) - return size - - def join_all(self): - '''Wait until all blocks in transit have been uploaded - - This method releases the global lock. - ''' - - self.compress_threads.join_all() - self.upload_threads.join_all() - - def join_one(self): - '''Wait until one block has been uploaded - - If there are no blocks in transit, return immediately. - This method releases the global lock. - ''' - - if len(self.upload_threads) == 0: - self.compress_threads.join_one() - - self.upload_threads.join_one() - - def upload_in_progress(self): - '''Return True if there are any blocks in transit''' - - return len(self.compress_threads) + len(self.upload_threads) > 0 - - -class CompressThread(Thread): - ''' - Compress a block and then pass it on for uploading. - - This class uses the llfuse global lock. When calling objects - passed in the constructor, the global lock is acquired first. - - The `size` attribute will be updated to the compressed size. - ''' - - def __init__(self, el, fh, um, size): - super(CompressThread, self).__init__() - self.el = el - self.fh = fh - self.um = um - self.size = size - - def run_protected(self): - '''Compress block - - After compression: - - the file handle is closed - - the compressed block size is updated in the database - - an UploadThread instance started for uploading the data. - - In case of an exception, the block is removed from the in_transit - set. - ''' - - try: - if log.isEnabledFor(logging.DEBUG): - oldsize = self.size - time_ = time.time() - (self.size, fn) = self.um.bucket.prep_store_fh('s3ql_data_%d' % self.el.obj_id, - self.fh) - time_ = time.time() - time_ - if time_ != 0: - rate = oldsize / (1024**2 * time_) - else: - rate = 0 - log.debug('CompressionThread(inode=%d, blockno=%d): ' - 'compressed %d bytes in %.3f seconds, %.2f MB/s', - self.el.inode, self.el.blockno, oldsize, - time_, rate) - else: - (self.size, fn) = self.um.bucket.prep_store_fh('s3ql_data_%d' % self.el.obj_id, - self.fh) - - self.fh.close() - - with lock: - # If we already have the minimum transit size, do not start more - # than two threads - log.debug('CompressThread(%s): starting upload thread', self.el) - - if self.um.transit_size > MIN_TRANSIT_SIZE: - max_threads = 2 - else: - max_threads = None - - self.um.transit_size += self.size - self.um.db.execute('UPDATE objects SET compr_size=? WHERE id=?', - (self.size, self.el.obj_id)) - self.um.upload_threads.add_thread(UploadThread(fn, self.el, self.size, self.um), - max_threads) - - except EmbeddedException: - raise - except: - with lock: - self.um.in_transit.remove((self.el.inode, self.el.blockno)) - self.um.transit_size -= self.size - raise - - -class UploadThread(Thread): - ''' - Uploads a cache entry with the function passed in the constructor. - - This class uses the llfuse global lock. When calling objects - passed in the constructor, the global lock is acquired first. - ''' - - def __init__(self, fn, el, size, um): - super(UploadThread, self).__init__() - self.fn = fn - self.el = el - self.size = size - self.um = um - - def run_protected(self): - '''Upload block by calling self.fn() - - The upload duration is timed. After the upload (or if an exception - occurs), the block is removed from in_transit. - ''' - try: - if log.isEnabledFor(logging.DEBUG): - time_ = time.time() - self.fn() - time_ = time.time() - time_ - if time_ != 0: - rate = self.size / (1024**2 * time_) - else: - rate = 0 - log.debug('CompressionThread(inode=%d, blockno=%d): ' - 'compressed %d bytes in %.3f seconds, %.2f MB/s', - self.el.inode, self.el.blockno, self.size, - time_, rate) - else: - self.fn() - - except: - with lock: - self.um.in_transit.remove((self.el.inode, self.el.blockno)) - self.um.transit_size -= self.size - raise - - with lock: - self.um.in_transit.remove((self.el.inode, self.el.blockno)) - self.um.transit_size -= self.size - - if not self.el.modified_after_upload: - self.el.dirty = False - try: - os.rename(self.el.name + '.d', self.el.name) - except OSError as exc: - # Entry may have been removed while being uploaded - if exc.errno != errno.ENOENT: - raise - - -def retry_exc(timeout, exc_types, fn, *a, **kw): - """Wait for fn(*a, **kw) to succeed - - If `fn(*a, **kw)` raises an exception in `exc_types`, the function is called again. - If the timeout is reached, `TimeoutError` is raised. - """ - - step = 0.2 - waited = 0 - while waited < timeout: - try: - return fn(*a, **kw) - except BaseException as exc: - for exc_type in exc_types: - if isinstance(exc, exc_type): - log.warn('Encountered %s error when calling %s, retrying...', - exc.__class__.__name__, fn.__name__) - break - else: - raise exc - - time.sleep(step) - waited += step - if step < timeout / 30: - step *= 2 - - raise TimeoutError() - -class RemoveThread(Thread): - ''' - Remove an object from backend. If a transit key is specified, the - thread first waits until the object is no longer in transit. - - TThis class uses the llfuse global lock. When calling objects - passed in the constructor, the global lock is acquired first. - ''' - - def __init__(self, id_, bucket, transit_key=None, upload_manager=None): - super(RemoveThread, self).__init__() - self.id = id_ - self.bucket = bucket - self.transit_key = transit_key - self.um = upload_manager - - def run_protected(self): - if self.transit_key: - while self.transit_key in self.um.in_transit: - with lock: - self.um.join_one() - - if self.bucket.read_after_create_consistent(): - self.bucket.delete('s3ql_data_%d' % self.id) - else: - retry_exc(300, [ NoSuchObject ], self.bucket.delete, - 's3ql_data_%d' % self.id)
\ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py index 06e210b..41711fa 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -3,7 +3,7 @@ __init__.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function diff --git a/tests/_common.py b/tests/_common.py index 66ff40f..5f251f1 100644 --- a/tests/_common.py +++ b/tests/_common.py @@ -3,7 +3,7 @@ _common.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. This module defines a new TestCase that aborts the test run as soon as a test fails. The module also servers as a storage container @@ -28,13 +28,10 @@ file has been written correctly. from __future__ import division, print_function import unittest2 as unittest -import os import logging from s3ql.common import add_stdout_logging, setup_excepthook #from s3ql.common import LoggerFilter -__all__ = [ 'TestCase' ] - log = logging.getLogger() class TestCase(unittest.TestCase): @@ -64,23 +61,3 @@ class TestCase(unittest.TestCase): # Abort if any test failed if result.errors or result.failures: result.stop() - -# Try to read credentials from file. Meant for developer use only, -# so that we can run individual tests without the setup.py -# initialization. -def init_credentials(): - keyfile = os.path.expanduser("~/.awssecret") - - if not os.path.isfile(keyfile): - return None - - with open(keyfile, "r") as fh: - key = fh.readline().rstrip() - pw = fh.readline().rstrip() - - return (key, pw) - -aws_credentials = init_credentials() - - - diff --git a/tests/t1_backends.py b/tests/t1_backends.py index d629466..a6c885b 100644 --- a/tests/t1_backends.py +++ b/tests/t1_backends.py @@ -3,59 +3,70 @@ t1_backends.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function - -import unittest2 as unittest -from s3ql.backends import local, s3 -from s3ql.backends.common import ChecksumError, ObjectNotEncrypted, NoSuchObject -import tempfile +from _common import TestCase +from s3ql.backends import local, s3, s3s, gs, gss, s3c +from s3ql.backends.common import (ChecksumError, ObjectNotEncrypted, NoSuchObject, + BetterBucket) +import ConfigParser import os +import stat +import tempfile import time -from _common import TestCase -import _common -from random import randrange - -class BackendTests(object): - +import unittest2 as unittest + +class BackendTestsMixin(object): + def newname(self): self.name_cnt += 1 # Include special characters return "s3ql_=/_%d" % self.name_cnt - def test_store(self): + def test_write(self): key = self.newname() value = self.newname() metadata = { 'jimmy': 'jups@42' } self.assertRaises(NoSuchObject, self.bucket.lookup, key) - self.bucket.store(key, value, metadata) - time.sleep(self.delay) - self.assertEquals(self.bucket.fetch(key), (value, metadata)) - self.assertEquals(self.bucket[key], value) - - def test_fetch(self): - key = self.newname() - value = self.newname() - metadata = { 'jimmy': 'jups@42' } - self.assertRaises(NoSuchObject, self.bucket.fetch, key) - self.bucket.store(key, value, metadata) + + with self.bucket.open_write(key, metadata) as fh: + fh.write(value) + time.sleep(self.delay) - self.assertEquals(self.bucket.fetch(key), (value, metadata)) + + with self.bucket.open_read(key) as fh: + value2 = fh.read() + + self.assertEquals(value, value2) + self.assertEquals(metadata, fh.metadata) + self.assertEquals(self.bucket[key], value) + self.assertEquals(self.bucket.lookup(key), metadata) - def test_lookup(self): + def test_setitem(self): key = self.newname() value = self.newname() metadata = { 'jimmy': 'jups@42' } self.assertRaises(NoSuchObject, self.bucket.lookup, key) - self.bucket.store(key, value, metadata) + self.assertRaises(NoSuchObject, self.bucket.__getitem__, key) + + with self.bucket.open_write(key, metadata) as fh: + fh.write(self.newname()) time.sleep(self.delay) - self.assertEquals(self.bucket.lookup(key), metadata) - + self.bucket[key] = value + time.sleep(self.delay) + + with self.bucket.open_read(key) as fh: + value2 = fh.read() + + self.assertEquals(value, value2) + self.assertEquals(fh.metadata, dict()) + self.assertEquals(self.bucket.lookup(key), dict()) + def test_contains(self): key = self.newname() value = self.newname() @@ -77,13 +88,17 @@ class BackendTests(object): self.assertFalse(key in self.bucket) def test_clear(self): - self.bucket[self.newname()] = self.newname() - self.bucket[self.newname()] = self.newname() + key1 = self.newname() + key2 = self.newname() + self.bucket[key1] = self.newname() + self.bucket[key2] = self.newname() time.sleep(self.delay) self.assertEquals(len(list(self.bucket)), 2) self.bucket.clear() time.sleep(self.delay) + self.assertTrue(key1 not in self.bucket) + self.assertTrue(key2 not in self.bucket) self.assertEquals(len(list(self.bucket)), 0) def test_list(self): @@ -96,28 +111,6 @@ class BackendTests(object): time.sleep(self.delay) self.assertEquals(sorted(self.bucket.list()), sorted(keys)) - def test_encryption(self): - bucket = self.bucket - bucket.passphrase = None - bucket['plain'] = b'foobar452' - - bucket.passphrase = 'schlurp' - bucket.store('encrypted', 'testdata', { 'tag': True }) - time.sleep(self.delay) - self.assertEquals(bucket['encrypted'], b'testdata') - self.assertRaises(ObjectNotEncrypted, bucket.fetch, 'plain') - self.assertRaises(ObjectNotEncrypted, bucket.lookup, 'plain') - - bucket.passphrase = None - self.assertRaises(ChecksumError, bucket.fetch, 'encrypted') - self.assertRaises(ChecksumError, bucket.lookup, 'encrypted') - - bucket.passphrase = self.passphrase - self.assertRaises(ChecksumError, bucket.fetch, 'encrypted') - self.assertRaises(ChecksumError, bucket.lookup, 'encrypted') - self.assertRaises(ObjectNotEncrypted, bucket.fetch, 'plain') - self.assertRaises(ObjectNotEncrypted, bucket.lookup, 'plain') - def test_copy(self): key1 = self.newname() @@ -133,55 +126,152 @@ class BackendTests(object): time.sleep(self.delay) self.assertEquals(self.bucket[key2], value) + def test_rename(self): + + key1 = self.newname() + key2 = self.newname() + value = self.newname() + self.assertRaises(NoSuchObject, self.bucket.lookup, key1) + self.assertRaises(NoSuchObject, self.bucket.lookup, key2) + + self.bucket.store(key1, value) + time.sleep(self.delay) + self.bucket.rename(key1, key2) + time.sleep(self.delay) + self.assertEquals(self.bucket[key2], value) + self.assertRaises(NoSuchObject, self.bucket.lookup, key1) + # This test just takes too long (because we have to wait really long so that we don't # get false errors due to propagation delays) -@unittest.skip('takes too long') -@unittest.skipUnless(_common.aws_credentials, 'no AWS credentials available') -class S3Tests(BackendTests, TestCase): - @staticmethod - def random_name(prefix=""): - return "s3ql-" + prefix + str(randrange(1000, 9999, 1)) - +#@unittest.skip('takes too long') +class S3Tests(BackendTestsMixin, TestCase): def setUp(self): - self.name_cnt = 0 - self.conn = s3.Connection(*_common.aws_credentials) + self.name_cnt = 0 + # This is the time in which we expect S3 changes to propagate. It may + # be much longer for larger objects, but for tests this is usually enough. + self.delay = 15 - self.bucketname = self.random_name() - tries = 10 - while self.conn.bucket_exists(self.bucketname) and tries > 10: - self.bucketname = self.random_name() - tries -= 1 + self.bucket = s3.Bucket(*self.get_credentials('s3-test')) - if tries == 0: - raise RuntimeError("Failed to find an unused bucket name.") + def tearDown(self): + self.bucket.clear() + + def get_credentials(self, name): + + authfile = os.path.expanduser('~/.s3ql/authinfo2') + if not os.path.exists(authfile): + self.skipTest('No authentication file found.') + + mode = os.stat(authfile).st_mode + if mode & (stat.S_IRGRP | stat.S_IROTH): + self.skipTest("Authentication file has insecure permissions") + + config = ConfigParser.SafeConfigParser() + config.read(authfile) + + try: + bucket_name = config.get(name, 'test-bucket') + backend_login = config.get(name, 'backend-login') + backend_password = config.get(name, 'backend-password') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.skipTest("Authentication file does not have test section") + + return (bucket_name, backend_login, backend_password) + + +class S3STests(S3Tests): + def setUp(self): + self.name_cnt = 0 + # This is the time in which we expect S3 changes to propagate. It may + # be much longer for larger objects, but for tests this is usually enough. + self.delay = 15 + + self.bucket = s3s.Bucket(*self.get_credentials('s3-test')) - self.passphrase = 'flurp' - self.bucket = self.conn.create_bucket(self.bucketname, self.passphrase) +class GSTests(S3Tests): + def setUp(self): + self.name_cnt = 0 + # This is the time in which we expect S3 changes to propagate. It may + # be much longer for larger objects, but for tests this is usually enough. + self.delay = 15 + + self.bucket = gs.Bucket(*self.get_credentials('gs-test')) + +class GSSTests(S3Tests): + def setUp(self): + self.name_cnt = 0 + # This is the time in which we expect S3 changes to propagate. It may + # be much longer for larger objects, but for tests this is usually enough. + self.delay = 15 + self.bucket = gss.Bucket(*self.get_credentials('gs-test')) +class S3CTests(S3Tests): + def setUp(self): + self.name_cnt = 0 # This is the time in which we expect S3 changes to propagate. It may # be much longer for larger objects, but for tests this is usually enough. - self.delay = 8 - time.sleep(self.delay) + self.delay = 0 + self.bucket = s3c.Bucket(*self.get_credentials('s3c-test')) + +class LocalTests(BackendTestsMixin, TestCase): + def setUp(self): + self.name_cnt = 0 + self.bucket_dir = tempfile.mkdtemp() + self.bucket = local.Bucket(self.bucket_dir, None, None) + self.delay = 0 + def tearDown(self): - self.conn.delete_bucket(self.bucketname, recursive=True) - -class LocalTests(BackendTests, TestCase): + self.bucket.clear() + os.rmdir(self.bucket_dir) +class CompressionTests(BackendTestsMixin, TestCase): + def setUp(self): - self.name_cnt = 0 - self.conn = local.Connection() + self.name_cnt = 0 self.bucket_dir = tempfile.mkdtemp() - self.bucketname = os.path.join(self.bucket_dir, 'mybucket') - self.passphrase = 'flurp' - self.bucket = self.conn.create_bucket(self.bucketname, self.passphrase) + self.plain_bucket = local.Bucket(self.bucket_dir, None, None) + self.bucket = self._wrap_bucket() self.delay = 0 + def _wrap_bucket(self): + return BetterBucket(None, 'zlib', self.plain_bucket) + def tearDown(self): - self.conn.delete_bucket(self.bucketname, recursive=True) + self.bucket.clear() os.rmdir(self.bucket_dir) + +class EncryptionTests(CompressionTests): + + def _wrap_bucket(self): + return BetterBucket('schlurz', None, self.plain_bucket) + + def test_encryption(self): + + self.plain_bucket['plain'] = b'foobar452' + self.bucket.store('encrypted', 'testdata', { 'tag': True }) + time.sleep(self.delay) + self.assertEquals(self.bucket['encrypted'], b'testdata') + self.assertNotEquals(self.plain_bucket['encrypted'], b'testdata') + self.assertRaises(ObjectNotEncrypted, self.bucket.fetch, 'plain') + self.assertRaises(ObjectNotEncrypted, self.bucket.lookup, 'plain') + + self.bucket.passphrase = None + self.assertRaises(ChecksumError, self.bucket.fetch, 'encrypted') + self.assertRaises(ChecksumError, self.bucket.lookup, 'encrypted') + + self.bucket.passphrase = 'jobzrul' + self.assertRaises(ChecksumError, self.bucket.fetch, 'encrypted') + self.assertRaises(ChecksumError, self.bucket.lookup, 'encrypted') + + +class EncryptionCompressionTests(EncryptionTests): + def _wrap_bucket(self): + return BetterBucket('schlurz', 'zlib', self.plain_bucket) + + # Somehow important according to pyunit documentation def suite(): return unittest.makeSuite(LocalTests) diff --git a/tests/t1_multi_lock.py b/tests/t1_multi_lock.py deleted file mode 100644 index eeaf0ee..0000000 --- a/tests/t1_multi_lock.py +++ /dev/null @@ -1,93 +0,0 @@ -''' -t1_ordered_dict.py - this file is part of S3QL (http://s3ql.googlecode.com) - -Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> - -This program can be distributed under the terms of the GNU LGPL. -''' - -from __future__ import division, print_function - -import unittest2 as unittest -from s3ql.multi_lock import MultiLock -import time -from s3ql.common import AsyncFn -from _common import TestCase - -BASE_DELAY = 1 - -@unittest.skip('takes too long') -class MultiLockTests(TestCase): - - def test_lock(self): - mlock = MultiLock() - key = (22, 'bar') - - def hold(): - mlock.acquire(key) - time.sleep(2 * BASE_DELAY) - mlock.release(key) - - t = AsyncFn(hold) - t.start() - time.sleep(BASE_DELAY) - - stamp = time.time() - with mlock(key): - pass - self.assertTrue(time.time() - stamp > BASE_DELAY) - - t.join_and_raise() - - def test_nolock(self): - mlock = MultiLock() - key1 = (22, 'bar') - key2 = (23, 'bar') - - def hold(): - mlock.acquire(key1) - time.sleep(2 * BASE_DELAY) - mlock.release(key1) - - t = AsyncFn(hold) - t.start() - time.sleep(BASE_DELAY) - - stamp = time.time() - with mlock(key2): - pass - self.assertTrue(time.time() - stamp < BASE_DELAY) - - t.join_and_raise() - - def test_multi(self): - mlock = MultiLock() - key = (22, 'bar') - - def lock(): - mlock.acquire(key) - - def unlock(): - time.sleep(2 * BASE_DELAY) - mlock.release(key) - - t1 = AsyncFn(lock) - t1.start() - t1.join_and_raise() - - t2 = AsyncFn(unlock) - t2.start() - - stamp = time.time() - with mlock(key): - pass - self.assertTrue(time.time() - stamp > BASE_DELAY) - - t2.join_and_raise() - -def suite(): - return unittest.makeSuite(MultiLockTests) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/t1_ordered_dict.py b/tests/t1_ordered_dict.py index b25bda8..60d9351 100644 --- a/tests/t1_ordered_dict.py +++ b/tests/t1_ordered_dict.py @@ -3,7 +3,7 @@ t1_ordered_dict.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function diff --git a/tests/t2_block_cache.py b/tests/t2_block_cache.py index f80883a..0631956 100644 --- a/tests/t2_block_cache.py +++ b/tests/t2_block_cache.py @@ -3,31 +3,33 @@ t2_block_cache.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2010 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function - -from s3ql.block_cache import BlockCache +from _common import TestCase +from contextlib import contextmanager from s3ql.backends import local -from s3ql.backends.common import NoSuchObject +from s3ql.backends.common import BucketPool, AbstractBucket +from s3ql.block_cache import BlockCache from s3ql.common import create_tables, init_tables from s3ql.database import Connection +import llfuse import os -import tempfile -from _common import TestCase -import unittest2 as unittest +import shutil import stat +import tempfile +import threading import time -import llfuse -import shutil +import unittest2 as unittest + class cache_tests(TestCase): def setUp(self): self.bucket_dir = tempfile.mkdtemp() - self.bucket = local.Connection().get_bucket(self.bucket_dir) + self.bucket_pool = BucketPool(lambda: local.Bucket(self.bucket_dir, None, None)) self.cachedir = tempfile.mkdtemp() + "/" self.blocksize = 1024 @@ -40,26 +42,21 @@ class cache_tests(TestCase): # Create an inode we can work with self.inode = 42 self.db.execute("INSERT INTO inodes (id,mode,uid,gid,mtime,atime,ctime,refcount,size) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (self.inode, stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR - | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, - os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1, 32)) - - self.cache = BlockCache(self.bucket, self.db, self.cachedir, - 100 * self.blocksize) - self.cache.init() + "VALUES (?,?,?,?,?,?,?,?,?)", + (self.inode, stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR + | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, + os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1, 32)) + + self.cache = BlockCache(self.bucket_pool, self.db, self.cachedir, + self.blocksize * 100) # Tested methods assume that they are called from # file system request handler llfuse.lock.acquire() - # We do not want background threads - self.cache.commit_thread.stop() - - def tearDown(self): - self.cache.upload_manager.bucket = self.bucket - self.cache.destroy() + self.cache.bucket_pool = self.bucket_pool + self.cache.destroy() if os.path.exists(self.cachedir): shutil.rmtree(self.cachedir) shutil.rmtree(self.bucket_dir) @@ -88,7 +85,6 @@ class cache_tests(TestCase): # Case 3: Object needs to be downloaded self.cache.clear() - self.cache.upload_manager.join_all() with self.cache.get(inode, blockno) as fh: fh.seek(0) self.assertEqual(data, fh.read(len(data))) @@ -117,17 +113,16 @@ class cache_tests(TestCase): # We want to expire 4 entries, 2 of which are already flushed self.cache.max_entries = 16 - self.cache.upload_manager.bucket = TestBucket(self.bucket, no_store=2) + self.cache.bucket_pool = TestBucketPool(self.bucket_pool, no_write=2) self.cache.expire() - self.cache.upload_manager.join_all() - self.cache.upload_manager.bucket.verify() - self.assertEqual(len(self.cache.cache), 16) + self.cache.bucket_pool.verify() + self.assertEqual(len(self.cache.entries), 16) for i in range(20): if i in most_recent: - self.assertTrue((inode, i) not in self.cache.cache) + self.assertTrue((inode, i) not in self.cache.entries) else: - self.assertTrue((inode, i) in self.cache.cache) + self.assertTrue((inode, i) in self.cache.entries) def test_upload(self): inode = self.inode @@ -139,85 +134,66 @@ class cache_tests(TestCase): data1 = self.random_data(datalen) data2 = self.random_data(datalen) data3 = self.random_data(datalen) - - mngr = self.cache.upload_manager # Case 1: create new object - self.cache.upload_manager.bucket = TestBucket(self.bucket, no_store=1) + self.cache.bucket_pool = TestBucketPool(self.bucket_pool, no_write=1) with self.cache.get(inode, blockno1) as fh: fh.seek(0) fh.write(data1) el1 = fh - mngr.add(el1) - mngr.join_all() - self.cache.removal_queue.join_all() - self.cache.upload_manager.bucket.verify() + self.cache.upload(el1) + self.cache.bucket_pool.verify() # Case 2: Link new object - self.cache.upload_manager.bucket = TestBucket(self.bucket) + self.cache.bucket_pool = TestBucketPool(self.bucket_pool) with self.cache.get(inode, blockno2) as fh: fh.seek(0) fh.write(data1) el2 = fh - mngr.add(el2) - mngr.join_all() - self.cache.removal_queue.join_all() - self.cache.upload_manager.bucket.verify() + self.cache.upload(el2) + self.cache.bucket_pool.verify() # Case 3: Upload old object, still has references - self.cache.upload_manager.bucket = TestBucket(self.bucket, no_store=1) + self.cache.bucket_pool = TestBucketPool(self.bucket_pool, no_write=1) with self.cache.get(inode, blockno1) as fh: fh.seek(0) fh.write(data2) - mngr.add(el1) - mngr.join_all() - self.cache.removal_queue.join_all() - self.cache.upload_manager.bucket.verify() - + self.cache.upload(el1) + self.cache.bucket_pool.verify() # Case 4: Upload old object, no references left - self.cache.upload_manager.bucket = TestBucket(self.bucket, no_del=1, no_store=1) + self.cache.bucket_pool = TestBucketPool(self.bucket_pool, no_del=1, no_write=1) with self.cache.get(inode, blockno2) as fh: fh.seek(0) fh.write(data3) - mngr.add(el2) - mngr.join_all() - self.cache.removal_queue.join_all() - self.cache.upload_manager.bucket.verify() + self.cache.upload(el2) + self.cache.bucket_pool.verify() # Case 5: Link old object, no references left - self.cache.upload_manager.bucket = TestBucket(self.bucket, no_del=1) + self.cache.bucket_pool = TestBucketPool(self.bucket_pool, no_del=1) with self.cache.get(inode, blockno2) as fh: fh.seek(0) fh.write(data2) - mngr.add(el2) - mngr.join_all() - self.cache.removal_queue.join_all() - self.cache.upload_manager.bucket.verify() - + self.cache.upload(el2) + self.cache.bucket_pool.verify() # Case 6: Link old object, still has references # (Need to create another object first) - self.cache.upload_manager.bucket = TestBucket(self.bucket, no_store=1) + self.cache.bucket_pool = TestBucketPool(self.bucket_pool, no_write=1) with self.cache.get(inode, blockno3) as fh: fh.seek(0) fh.write(data1) el3 = fh - mngr.add(el3) - mngr.join_all() - self.cache.removal_queue.join_all() - self.cache.upload_manager.bucket.verify() + self.cache.upload(el3) + self.cache.bucket_pool.verify() - self.cache.upload_manager.bucket = TestBucket(self.bucket) + self.cache.bucket_pool = TestBucketPool(self.bucket_pool) with self.cache.get(inode, blockno1) as fh: fh.seek(0) fh.write(data1) - mngr.add(el1) - mngr.join_all() - self.cache.removal_queue.join_all() - self.cache.upload_manager.bucket.verify() - - + self.cache.upload(el1) + self.cache.clear() + self.cache.bucket_pool.verify() def test_remove_referenced(self): inode = self.inode @@ -226,7 +202,7 @@ class cache_tests(TestCase): blockno2 = 24 data = self.random_data(datalen) - self.cache.upload_manager.bucket = TestBucket(self.bucket, no_store=1) + self.cache.bucket_pool = TestBucketPool(self.bucket_pool, no_write=1) with self.cache.get(inode, blockno1) as fh: fh.seek(0) fh.write(data) @@ -234,12 +210,11 @@ class cache_tests(TestCase): fh.seek(0) fh.write(data) self.cache.clear() - self.cache.upload_manager.join_all() - self.cache.upload_manager.bucket.verify() + self.cache.bucket_pool.verify() - self.cache.upload_manager.bucket = TestBucket(self.bucket) + self.cache.bucket_pool = TestBucketPool(self.bucket_pool) self.cache.remove(inode, blockno1) - self.cache.upload_manager.bucket.verify() + self.cache.bucket_pool.verify() def test_remove_cache(self): inode = self.inode @@ -262,15 +237,18 @@ class cache_tests(TestCase): with self.cache.get(inode, 1) as fh: fh.seek(0) fh.write(data1) - self.cache.upload_manager.bucket = TestBucket(self.bucket, no_store=1) + self.cache.bucket_pool = TestBucketPool(self.bucket_pool, no_write=1) commit(self.cache, inode) - self.cache.upload_manager.bucket.verify() - self.cache.upload_manager.bucket = TestBucket(self.bucket, no_del=1) + self.cache.bucket_pool.verify() + self.cache.bucket_pool = TestBucketPool(self.bucket_pool, no_del=1) self.cache.remove(inode, 1) + self.cache.bucket_pool.verify() + with self.cache.get(inode, 1) as fh: fh.seek(0) self.assertTrue(fh.read(42) == '') - + + def test_remove_db(self): inode = self.inode data1 = self.random_data(int(0.4 * self.blocksize)) @@ -279,96 +257,112 @@ class cache_tests(TestCase): with self.cache.get(inode, 1) as fh: fh.seek(0) fh.write(data1) - self.cache.upload_manager.bucket = TestBucket(self.bucket, no_store=1) + self.cache.bucket_pool = TestBucketPool(self.bucket_pool, no_write=1) self.cache.clear() - self.cache.upload_manager.join_all() - self.cache.upload_manager.bucket.verify() - self.cache.upload_manager.bucket = TestBucket(self.bucket, no_del=1) + self.cache.bucket_pool.verify() + self.cache.bucket_pool = TestBucketPool(self.bucket_pool, no_del=1) self.cache.remove(inode, 1) + self.cache.bucket_pool.verify() with self.cache.get(inode, 1) as fh: fh.seek(0) - self.assertTrue(fh.read(42) == '') - + self.assertTrue(fh.read(42) == '') -class TestBucket(object): - def __init__(self, bucket, no_fetch=0, no_store=0, no_del=0): - self.no_fetch = no_fetch - self.no_store = no_store +class TestBucketPool(AbstractBucket): + def __init__(self, bucket_pool, no_read=0, no_write=0, no_del=0): + super(TestBucketPool, self).__init__() + self.no_read = no_read + self.no_write = no_write self.no_del = no_del - self.bucket = bucket + self.bucket_pool = bucket_pool + self.bucket = bucket_pool.pop_conn() + self.lock = threading.Lock() - def read_after_create_consistent(self): - return self.bucket.read_after_create_consistent() - - def read_after_write_consistent(self): - return self.bucket.read_after_write_consistent() - + def __del__(self): + self.bucket_pool.push_conn(self.bucket) + def verify(self): - if self.no_fetch != 0: - raise RuntimeError('Got too few fetch calls') - if self.no_store != 0: - raise RuntimeError('Got too few store calls') + if self.no_read != 0: + raise RuntimeError('Got too few open_read calls') + if self.no_write != 0: + raise RuntimeError('Got too few open_write calls') if self.no_del != 0: raise RuntimeError('Got too few delete calls') - def prep_store_fh(self, *a, **kw): - (size, fn) = self.bucket.prep_store_fh(*a, **kw) - def fn2(): - self.no_store -= 1 - if self.no_store < 0: - raise RuntimeError('Got too many store calls') - return fn() + @contextmanager + def __call__(self): + '''Provide connection from pool (context manager)''' - return (size, fn2) - - def store_fh(self, *a, **kw): - self.no_store -= 1 - - if self.no_store < 0: - raise RuntimeError('Got too many store calls') - - return self.bucket.store_fh(*a, **kw) + with self.lock: + yield self + + def lookup(self, key): + return self.bucket.lookup(key) + + def open_read(self, key): + self.no_read -= 1 + if self.no_read < 0: + raise RuntimeError('Got too many open_read calls') + + return self.bucket.open_read(key) - def fetch_fh(self, *a, **kw): - self.no_fetch -= 1 + def open_write(self, key, metadata=None): + self.no_write -= 1 + if self.no_write < 0: + raise RuntimeError('Got too many open_write calls') - if self.no_fetch < 0: - raise RuntimeError('Got too many fetch calls') + return self.bucket.open_write(key, metadata) + + def is_get_consistent(self): + return self.bucket.is_get_consistent() + + def is_list_create_consistent(self): + return self.bucket.is_get_consistent() + + def clear(self): + return self.bucket.clear() - return self.bucket.fetch_fh(*a, **kw) + def contains(self, key): + return self.bucket.contains(key) - def delete(self, *a, **kw): + def delete(self, key, force=False): self.no_del -= 1 - if self.no_del < 0: raise RuntimeError('Got too many delete calls') + + return self.bucket.delete(key, force) - try: - return self.bucket.delete(*a, **kw) - except NoSuchObject: - # Don't count key errors - self.no_del += 1 - raise - + def list(self, prefix=''): + '''List keys in bucket - def __delitem__(self, key): - self.delete(key) + Returns an iterator over all keys in the bucket. + ''' + return self.bucket.list(prefix) - def __iter__(self): - return self.bucket.list() + def copy(self, src, dest): + """Copy data stored under key `src` to key `dest` + + If `dest` already exists, it will be overwritten. The copying + is done on the remote side. + """ + return self.bucket.copy(src, dest) - def __contains__(self, key): - return self.bucket.contains(key) + def rename(self, src, dest): + """Rename key `src` to `dest` + + If `dest` already exists, it will be overwritten. The rename + is done on the remote side. + """ + return self.bucket.rename(src, dest) -def commit(self, inode, block=None): +def commit(cache, inode, block=None): """Upload data for `inode` - This is only for testing purposes, since the method blocks - until all current uploads have been completed. + This is only for testing purposes, since the method blocks until all current + uploads have been completed. """ - for el in self.cache.itervalues(): + for el in cache.entries.itervalues(): if el.inode != inode: continue if not el.dirty: @@ -377,9 +371,7 @@ def commit(self, inode, block=None): if block is not None and el.blockno != block: continue - self.upload_manager.add(el) - - self.upload_manager.join_all() + cache.upload(el) def suite(): diff --git a/tests/t3_fs_api.py b/tests/t3_fs_api.py index 04a2249..2a45af3 100644 --- a/tests/t3_fs_api.py +++ b/tests/t3_fs_api.py @@ -3,27 +3,29 @@ t3_fs_api.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function - +from _common import TestCase +from llfuse import FUSEError from random import randint -from s3ql.fsck import Fsck from s3ql import fs from s3ql.backends import local +from s3ql.backends.common import BucketPool +from s3ql.block_cache import BlockCache from s3ql.common import ROOT_INODE, create_tables, init_tables -from llfuse import FUSEError from s3ql.database import Connection -from _common import TestCase +from s3ql.fsck import Fsck +import errno +import llfuse import os +import shutil import stat +import tempfile import time -import llfuse import unittest2 as unittest -import errno -import shutil -import tempfile + # We need to access to protected members #pylint: disable=W0212 @@ -46,7 +48,8 @@ class fs_api_tests(TestCase): def setUp(self): self.bucket_dir = tempfile.mkdtemp() - self.bucket = local.Connection().get_bucket(self.bucket_dir) + self.bucket_pool = BucketPool(lambda: local.Bucket(self.bucket_dir, None, None)) + self.bucket = self.bucket_pool.pop_conn() self.cachedir = tempfile.mkdtemp() + "/" self.blocksize = 1024 @@ -55,24 +58,23 @@ class fs_api_tests(TestCase): create_tables(self.db) init_tables(self.db) - self.server = fs.Operations(self.bucket, self.db, self.cachedir, - self.blocksize, cache_size=self.blocksize * 5) - # Tested methods assume that they are called from # file system request handler llfuse.lock.acquire() - + + self.block_cache = BlockCache(self.bucket_pool, self.db, self.cachedir, + self.blocksize * 5) + self.server = fs.Operations(self.block_cache, self.db, self.blocksize) + self.server.init() - # We don't want background flushing - self.server.cache.commit_thread.stop() - self.server.inode_flush_thread.stop() - # Keep track of unused filenames self.name_cnt = 0 def tearDown(self): self.server.destroy() + self.block_cache.destroy() + if os.path.exists(self.cachedir): shutil.rmtree(self.cachedir) shutil.rmtree(self.bucket_dir) @@ -84,8 +86,7 @@ class fs_api_tests(TestCase): return fd.read(len_) def fsck(self): - self.server.cache.clear() - self.server.cache.upload_manager.join_all() + self.block_cache.clear() self.server.inodes.flush() fsck = Fsck(self.cachedir, self.bucket, { 'blocksize': self.blocksize }, self.db) @@ -109,8 +110,8 @@ class fs_api_tests(TestCase): time.sleep(CLOCK_GRANULARITY) self.server._create(ROOT_INODE, name, mode, ctx) - id_ = self.db.get_val('SELECT inode FROM contents WHERE name=? AND ' - 'parent_inode = ?', (name, ROOT_INODE)) + id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON name_id = names.id ' + 'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)) inode = self.server.getattr(id_) @@ -184,8 +185,8 @@ class fs_api_tests(TestCase): inode_after = self.server.lookup(inode_p_new.id, name) inode_p_new_after = self.server.getattr(inode_p_new.id) - id_ = self.db.get_val('SELECT inode FROM contents WHERE name=? AND ' - 'parent_inode = ?', (name, inode_p_new.id)) + id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (name, inode_p_new.id)) self.assertEqual(inode_before.id, id_) self.assertEqual(inode_after.refcount, 2) @@ -280,8 +281,8 @@ class fs_api_tests(TestCase): self.file_mode(), Ctx()) self.server.write(fh, 0, 'foobar') self.server.unlink(ROOT_INODE, name) - self.assertFalse(self.db.has_val('SELECT 1 FROM contents WHERE name=? AND ' - 'parent_inode = ?', (name, ROOT_INODE))) + self.assertFalse(self.db.has_val('SELECT 1 FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE))) self.assertTrue(self.server.getattr(inode.id).id) self.server.release(fh) @@ -317,10 +318,10 @@ class fs_api_tests(TestCase): inode_p_old_after = self.server.getattr(ROOT_INODE) inode_p_new_after = self.server.getattr(inode_p_new.id) - self.assertFalse(self.db.has_val('SELECT inode FROM contents WHERE name=? AND ' - 'parent_inode = ?', (oldname, ROOT_INODE))) - id_ = self.db.get_val('SELECT inode FROM contents WHERE name=? AND ' - 'parent_inode = ?', (newname, inode_p_new.id)) + self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (oldname, ROOT_INODE))) + id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id == name_id ' + 'WHERE name=? AND parent_inode = ?', (newname, inode_p_new.id)) self.assertEqual(inode.id, id_) self.assertLess(inode_p_new_before.mtime, inode_p_new_after.mtime) @@ -355,10 +356,10 @@ class fs_api_tests(TestCase): inode_p_old_after = self.server.getattr(ROOT_INODE) inode_p_new_after = self.server.getattr(inode_p_new.id) - self.assertFalse(self.db.has_val('SELECT inode FROM contents WHERE name=? AND ' - 'parent_inode = ?', (oldname, ROOT_INODE))) - id_ = self.db.get_val('SELECT inode FROM contents WHERE name=? AND ' - 'parent_inode = ?', (newname, inode_p_new.id)) + self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (oldname, ROOT_INODE))) + id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (newname, inode_p_new.id)) self.assertEqual(inode.id, id_) self.assertLess(inode_p_new_before.mtime, inode_p_new_after.mtime) @@ -388,10 +389,10 @@ class fs_api_tests(TestCase): inode_p_old_after = self.server.getattr(ROOT_INODE) inode_p_new_after = self.server.getattr(inode_p_new.id) - self.assertFalse(self.db.has_val('SELECT inode FROM contents WHERE name=? AND ' - 'parent_inode = ?', (oldname, ROOT_INODE))) - id_ = self.db.get_val('SELECT inode FROM contents WHERE name=? AND ' - 'parent_inode = ?', (newname, inode_p_new.id)) + self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (oldname, ROOT_INODE))) + id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (newname, inode_p_new.id)) self.assertEqual(inode.id, id_) self.assertLess(inode_p_new_before.mtime, inode_p_new_after.mtime) @@ -503,8 +504,8 @@ class fs_api_tests(TestCase): self.assertEqual(target, self.server.readlink(inode.id)) - id_ = self.db.get_val('SELECT inode FROM contents WHERE name=? AND ' - 'parent_inode = ?', (name, ROOT_INODE)) + id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)) self.assertEqual(inode.id, id_) self.assertLess(inode_p_before.mtime, inode_p_after.mtime) @@ -529,8 +530,8 @@ class fs_api_tests(TestCase): self.assertLess(inode_p_before.mtime, inode_p_after.mtime) self.assertLess(inode_p_before.ctime, inode_p_after.ctime) - self.assertFalse(self.db.has_val('SELECT inode FROM contents WHERE name=? AND ' - 'parent_inode = ?', (name, ROOT_INODE))) + self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE))) self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode.id,))) self.fsck() @@ -545,8 +546,8 @@ class fs_api_tests(TestCase): self.assertLess(inode_p_before.mtime, inode_p_after.mtime) self.assertLess(inode_p_before.ctime, inode_p_after.ctime) - self.assertFalse(self.db.has_val('SELECT inode FROM contents WHERE name=? AND ' - 'parent_inode = ?', (name, ROOT_INODE))) + self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE))) self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode.id,))) self.fsck() @@ -559,8 +560,8 @@ class fs_api_tests(TestCase): (fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(), Ctx()) self.server.write(fh, 0, data) self.server.unlink(ROOT_INODE, name) - self.assertFalse(self.db.has_val('SELECT inode FROM contents WHERE name=? AND ' - 'parent_inode = ?', (name, ROOT_INODE))) + self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE))) self.assertTrue(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode.id,))) self.server.link(inode.id, ROOT_INODE, name2) @@ -594,6 +595,25 @@ class fs_api_tests(TestCase): self.fsck() + def test_edit(self): + len_ = self.blocksize + data = self.random_data(len_) + (fh, inode) = self.server.create(ROOT_INODE, self.newname(), + self.file_mode(), Ctx()) + self.server.write(fh, 0, data) + self.server.release(fh) + + self.block_cache.clear() + + fh = self.server.open(inode.id, os.O_RDWR) + attr = llfuse.EntryAttributes() + attr.st_size = 0 + self.server.setattr(inode.id, attr) + self.server.write(fh, 0, data[50:]) + self.server.release(fh) + + self.fsck() + def test_copy_tree(self): src_inode = self.server.mkdir(ROOT_INODE, 'source', self.dir_mode(), Ctx()) @@ -761,8 +781,8 @@ class fs_api_tests(TestCase): (inode1.id, 'file1'), (inode1.id, 'dir1'), (inode2.id, 'file2')): - self.assertFalse(self.db.has_val('SELECT inode FROM contents WHERE name=? AND ' - 'parent_inode = ?', (name, id_p))) + self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (name, id_p))) for id_ in (inode1.id, inode1a.id, inode2.id, inode2a.id): self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (id_,))) diff --git a/tests/t3_fsck.py b/tests/t3_fsck.py index e111320..4a31019 100644 --- a/tests/t3_fsck.py +++ b/tests/t3_fsck.py @@ -3,29 +3,28 @@ t3_fsck.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function - -import unittest2 as unittest -from s3ql.fsck import Fsck +from _common import TestCase from s3ql.backends import local -from s3ql.database import Connection from s3ql.common import ROOT_INODE, create_tables, init_tables -from _common import TestCase +from s3ql.database import Connection, NoSuchRowError +from s3ql.fsck import Fsck import os +import shutil import stat import tempfile import time -import shutil +import unittest2 as unittest + class fsck_tests(TestCase): def setUp(self): self.bucket_dir = tempfile.mkdtemp() - self.passphrase = 'schnupp' - self.bucket = local.Connection().get_bucket(self.bucket_dir, self.passphrase) + self.bucket = local.Bucket(self.bucket_dir, None, None) self.cachedir = tempfile.mkdtemp() + "/" self.blocksize = 1024 @@ -45,97 +44,175 @@ class fsck_tests(TestCase): def assert_fsck(self, fn): '''Check that fn detects and corrects an error''' - self.fsck.found_errors = False fn() self.assertTrue(self.fsck.found_errors) self.fsck.found_errors = False - fn() + self.fsck.check() self.assertFalse(self.fsck.found_errors) def test_cache(self): - inode = 6 - self.db.execute("INSERT INTO inodes (id, mode,uid,gid,mtime,atime,ctime,refcount) " - "VALUES (?,?,?,?,?,?,?,?)", - (inode, stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR - | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, - os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1)) - - fh = open(self.cachedir + 'inode_%d_block_1.d' % inode, 'wb') + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR + | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, + os.getuid(), os.getgid(), time.time(), time.time(), time.time(), + 1, 8)) + self._link('test-entry', inode) + + # Create new block + fh = open(self.cachedir + '%d-0' % inode, 'wb') fh.write('somedata') fh.close() - self.assert_fsck(self.fsck.check_cache) self.assertEquals(self.bucket['s3ql_data_1'], 'somedata') - fh = open(self.cachedir + 'inode_%d_block_1' % inode, 'wb') - fh.write('otherdata') - fh.close() - + # Existing block + self.db.execute('UPDATE inodes SET size=? WHERE id=?', + (self.blocksize + 8, inode)) + with open(self.cachedir + '%d-1' % inode, 'wb') as fh: + fh.write('somedata') self.assert_fsck(self.fsck.check_cache) - self.assertEquals(self.bucket['s3ql_data_1'], 'somedata') - - + + # Old block preserved + with open(self.cachedir + '%d-0' % inode, 'wb') as fh: + fh.write('somedat2') + self.assert_fsck(self.fsck.check_cache) + + # Old block removed + with open(self.cachedir + '%d-1' % inode, 'wb') as fh: + fh.write('somedat3') + self.assert_fsck(self.fsck.check_cache) + + def test_lof1(self): # Make lost+found a file - inode = self.db.get_val("SELECT inode FROM contents WHERE name=? AND parent_inode=?", + inode = self.db.get_val("SELECT inode FROM contents_v WHERE name=? AND parent_inode=?", (b"lost+found", ROOT_INODE)) self.db.execute('DELETE FROM contents WHERE parent_inode=?', (inode,)) self.db.execute('UPDATE inodes SET mode=?, size=? WHERE id=?', (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, 0, inode)) - self.assert_fsck(self.fsck.check_lof) + def check(): + self.fsck.check_lof() + self.fsck.check_inode_refcount() + + self.assert_fsck(check) def test_lof2(self): # Remove lost+found - self.db.execute('DELETE FROM contents WHERE name=? and parent_inode=?', - (b'lost+found', ROOT_INODE)) - + name_id = self.db.get_val('SELECT id FROM names WHERE name=?', (b'lost+found',)) + inode = self.db.get_val('SELECT inode FROM contents WHERE name_id=? AND ' + 'parent_inode=?', (name_id, ROOT_INODE)) + self.db.execute('DELETE FROM inodes WHERE id=?', (inode,)) + self.db.execute('DELETE FROM contents WHERE name_id=? and parent_inode=?', + (name_id, ROOT_INODE)) + self.db.execute('UPDATE names SET refcount = refcount-1 WHERE id=?', (name_id,)) + self.assert_fsck(self.fsck.check_lof) - def test_inode_refcount(self): - - # Create an orphaned inode - self.db.execute("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) " - "VALUES (?,?,?,?,?,?,?,?)", - (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, - 0, 0, time.time(), time.time(), time.time(), 2, 0)) + def test_wrong_inode_refcount(self): + + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time.time(), time.time(), time.time(), 1, 0)) + self._link('name1', inode) + self._link('name2', inode) + self.assert_fsck(self.fsck.check_inode_refcount) + def test_orphaned_inode(self): + + self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time.time(), time.time(), time.time(), 1, 0)) self.assert_fsck(self.fsck.check_inode_refcount) + + def test_name_refcount(self): - # Create an inode with wrong refcount inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) " - "VALUES (?,?,?,?,?,?,?,?)", - (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, - 0, 0, time.time(), time.time(), time.time(), 1, 0)) - self.db.execute('INSERT INTO contents (name, inode, parent_inode) VALUES(?, ?, ?)', - (b'name1', inode, ROOT_INODE)) - self.db.execute('INSERT INTO contents (name, inode, parent_inode) VALUES(?, ?, ?)', - (b'name2', inode, ROOT_INODE)) + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time.time(), time.time(), time.time(), 2, 0)) + self._link('name1', inode) + self._link('name2', inode) + + self.db.execute('UPDATE names SET refcount=refcount+1 WHERE name=?', ('name1',)) + + self.assert_fsck(self.fsck.check_name_refcount) - self.assert_fsck(self.fsck.check_inode_refcount) + def test_orphaned_name(self): + + self._add_name('zupbrazl') + self.assert_fsck(self.fsck.check_name_refcount) + + def test_ref_integrity(self): + self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)', + (self._add_name('foobar'), 124, ROOT_INODE)) + + self.fsck.found_errors = False + self.fsck.check_foreign_keys() + self.assertTrue(self.fsck.found_errors) + + def _add_name(self, name): + '''Get id for *name* and increase refcount + + Name is inserted in table if it does not yet exist. + ''' + + try: + name_id = self.db.get_val('SELECT id FROM names WHERE name=?', (name,)) + except NoSuchRowError: + name_id = self.db.rowid('INSERT INTO names (name, refcount) VALUES(?,?)', + (name, 1)) + else: + self.db.execute('UPDATE names SET refcount=refcount+1 WHERE id=?', (name_id,)) + return name_id + + def _link(self, name, inode): + '''Link /*name* to *inode*''' + + self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)', + (self._add_name(name), inode, ROOT_INODE)) + def test_inode_sizes(self): id_ = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) " - "VALUES (?,?,?,?,?,?,?,?)", - (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, - 0, 0, time.time(), time.time(), time.time(), 2, 0)) - - self.db.execute('INSERT INTO contents (name, inode, parent_inode) VALUES(?,?,?)', - ('test-entry', id_, ROOT_INODE)) - - # Create a block - obj_id = self.db.rowid('INSERT INTO objects (refcount, size) VALUES(?, ?)', - (1, 500)) - self.db.execute('INSERT INTO blocks (inode, blockno, obj_id) VALUES(?, ?, ?)', - (id_, 0, obj_id)) - - + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time.time(), time.time(), time.time(), 1, 128)) + self._link('test-entry', id_) + + obj_id = self.db.rowid('INSERT INTO objects (refcount) VALUES(1)') + block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)', + (1, obj_id, 512)) + self.bucket['s3ql_data_%d' % obj_id] = 'foo' + + # Case 1 + self.db.execute('UPDATE inodes SET block_id=?, size=? WHERE id=?', + (None, self.blocksize + 120, id_)) + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?, ?, ?)', + (id_, 1, block_id)) self.assert_fsck(self.fsck.check_inode_sizes) + # Case 2 + self.db.execute('DELETE FROM inode_blocks WHERE inode=?', (id_,)) + self.db.execute('UPDATE inodes SET block_id=?, size=? WHERE id=?', + (block_id, 129, id_)) + self.assert_fsck(self.fsck.check_inode_sizes) + # Case 3 + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?, ?, ?)', + (id_, 1, block_id)) + self.db.execute('UPDATE inodes SET block_id=?, size=? WHERE id=?', + (block_id, self.blocksize + 120, id_)) + self.db.execute('UPDATE blocks SET refcount = refcount + 1 WHERE id = ?', + (block_id,)) + self.assert_fsck(self.fsck.check_inode_sizes) + def test_keylist(self): # Create an object that only exists in the bucket @@ -143,10 +220,23 @@ class fsck_tests(TestCase): self.assert_fsck(self.fsck.check_keylist) # Create an object that does not exist in the bucket - self.db.execute('INSERT INTO objects (id, refcount, size) VALUES(?, ?, ?)', - (34, 1, 0)) + self.db.execute('INSERT INTO objects (id, refcount) VALUES(?, ?)', (34, 1)) self.assert_fsck(self.fsck.check_keylist) + def test_missing_obj(self): + + obj_id = self.db.rowid('INSERT INTO objects (refcount) VALUES(1)') + block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)', + (1, obj_id, 128)) + + id_ = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size,block_id) " + "VALUES (?,?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time.time(), time.time(), time.time(), 1, 128, block_id)) + + self._link('test-entry', id_) + self.assert_fsck(self.fsck.check_keylist) + @staticmethod def random_data(len_): with open("/dev/urandom", "rb") as fd: @@ -156,16 +246,16 @@ class fsck_tests(TestCase): # Create some directory inodes inodes = [ self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount) " - "VALUES (?,?,?,?,?,?,?)", - (stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR, - 0, 0, time.time(), time.time(), time.time(), 1)) + "VALUES (?,?,?,?,?,?,?)", + (stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time.time(), time.time(), time.time(), 1)) for dummy in range(3) ] inodes.append(inodes[0]) last = inodes[0] for inode in inodes[1:]: - self.db.execute('INSERT INTO contents (name, inode, parent_inode) VALUES(?, ?, ?)', - (bytes(inode), inode, last)) + self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?, ?, ?)', + (self._add_name(bytes(inode)), inode, last)) last = inode self.fsck.found_errors = False @@ -177,31 +267,57 @@ class fsck_tests(TestCase): def test_obj_refcounts(self): - obj_id = 42 - inode = 42 - self.db.execute("INSERT INTO inodes (id, mode,uid,gid,mtime,atime,ctime,refcount,size) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (inode, stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, - os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1, 0)) - - self.db.execute('INSERT INTO objects (id, refcount, size) VALUES(?, ?, ?)', - (obj_id, 2, 0)) - self.db.execute('INSERT INTO blocks (inode, blockno, obj_id) VALUES(?, ?, ?)', - (inode, 1, obj_id)) - self.db.execute('INSERT INTO blocks (inode, blockno, obj_id) VALUES(?, ?, ?)', - (inode, 2, obj_id)) + obj_id = self.db.rowid('INSERT INTO objects (refcount) VALUES(1)') + block_id_1 = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)', + (1, obj_id, 0)) + block_id_2 = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)', + (1, obj_id, 0)) + self.bucket['s3ql_data_%d' % obj_id] = 'foo' + + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + os.getuid(), os.getgid(), time.time(), time.time(), time.time(), + 1, 2048)) + self._link('test-entry', inode) + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)', + (inode, 1, block_id_1)) + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)', + (inode, 2, block_id_2)) + + self.assert_fsck(self.fsck.check_obj_refcounts) - self.fsck.found_errors = False - self.fsck.check_obj_refcounts() - self.assertFalse(self.fsck.found_errors) + def test_orphaned_obj(self): - self.db.execute('INSERT INTO blocks (inode, blockno, obj_id) VALUES(?, ?, ?)', - (inode, 3, obj_id)) + self.db.rowid('INSERT INTO objects (refcount) VALUES(1)') self.assert_fsck(self.fsck.check_obj_refcounts) + + def test_wrong_block_refcount(self): - self.db.execute('DELETE FROM blocks WHERE obj_id=?', (obj_id,)) - self.assert_fsck(self.fsck.check_obj_refcounts) + obj_id = self.db.rowid('INSERT INTO objects (refcount) VALUES(1)') + self.bucket['s3ql_data_%d' % obj_id] = 'foo' + block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)', + (1, obj_id, 0)) + + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size,block_id) " + "VALUES (?,?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, os.getuid(), os.getgid(), + time.time(), time.time(), time.time(), 1, self.blocksize, block_id)) + self._link('test-entry', inode) + + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)', + (inode, 1, block_id)) + self.assert_fsck(self.fsck.check_block_refcount) + + def test_orphaned_block(self): + + obj_id = self.db.rowid('INSERT INTO objects (refcount) VALUES(1)') + self.bucket['s3ql_data_%d' % obj_id] = 'foo' + self.db.rowid('INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)', + (1, obj_id, 0)) + self.assert_fsck(self.fsck.check_block_refcount) + def test_unix_size(self): inode = 42 @@ -209,9 +325,7 @@ class fsck_tests(TestCase): "VALUES (?,?,?,?,?,?,?,?,?)", (inode, stat.S_IFIFO | stat.S_IRUSR | stat.S_IWUSR, os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1, 0)) - - self.db.execute('INSERT INTO contents (name, inode, parent_inode) VALUES(?,?,?)', - ('test-entry', inode, ROOT_INODE)) + self._link('test-entry', inode) self.fsck.found_errors = False self.fsck.check_inode_unix() @@ -226,14 +340,13 @@ class fsck_tests(TestCase): inode = 42 target = 'some funny random string' - self.db.execute("INSERT INTO inodes (id, mode,uid,gid,mtime,atime,ctime,refcount,target,size) " - "VALUES (?,?,?,?,?,?,?,?,?,?)", + self.db.execute("INSERT INTO inodes (id, mode,uid,gid,mtime,atime,ctime,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?,?)", (inode, stat.S_IFLNK | stat.S_IRUSR | stat.S_IWUSR, os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1, - target, len(target))) - - self.db.execute('INSERT INTO contents (name, inode, parent_inode) VALUES(?,?,?)', - ('test-entry', inode, ROOT_INODE)) + len(target))) + self.db.execute('INSERT INTO symlink_targets (inode, target) VALUES(?,?)', (inode, target)) + self._link('test-entry', inode) self.fsck.found_errors = False self.fsck.check_inode_unix() @@ -250,18 +363,26 @@ class fsck_tests(TestCase): "VALUES (?,?,?,?,?,?,?,?)", (inode, stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR, os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1)) - - self.db.execute('INSERT INTO contents (name, inode, parent_inode) VALUES(?,?,?)', - ('test-entry', inode, ROOT_INODE)) + self._link('test-entry', inode) self.fsck.found_errors = False self.fsck.check_inode_unix() self.assertFalse(self.fsck.found_errors) - self.db.execute('UPDATE inodes SET target = ? WHERE id=?', ('foo', inode)) + self.db.execute('INSERT INTO symlink_targets (inode, target) VALUES(?,?)', (inode, 'foo')) self.fsck.check_inode_unix() self.assertTrue(self.fsck.found_errors) + def test_symlink_no_target(self): + + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount) " + "VALUES (?,?,?,?,?,?,?)", + (stat.S_IFLNK | stat.S_IRUSR | stat.S_IWUSR, + os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1)) + self._link('test-entry', inode) + self.fsck.check_inode_unix() + self.assertTrue(self.fsck.found_errors) + def test_unix_rdev(self): inode = 42 @@ -269,8 +390,7 @@ class fsck_tests(TestCase): "VALUES (?,?,?,?,?,?,?,?)", (inode, stat.S_IFIFO | stat.S_IRUSR | stat.S_IWUSR, os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1)) - self.db.execute('INSERT INTO contents (name, inode, parent_inode) VALUES(?,?,?)', - ('test-entry', inode, ROOT_INODE)) + self._link('test-entry', inode) self.fsck.found_errors = False self.fsck.check_inode_unix() @@ -286,41 +406,40 @@ class fsck_tests(TestCase): "VALUES (?,?,?,?,?,?,?)", (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1)) - - self.db.execute('INSERT INTO contents (name, inode, parent_inode) VALUES(?,?,?)', - ('test-entry', inode, ROOT_INODE)) + self._link('test-entry', inode) self.fsck.found_errors = False self.fsck.check_inode_unix() self.assertFalse(self.fsck.found_errors) - self.db.execute('INSERT INTO contents (name, inode, parent_inode) VALUES(?,?,?)', - ('foo', ROOT_INODE, inode)) + self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)', + (self._add_name('foo'), ROOT_INODE, inode)) self.fsck.check_inode_unix() self.assertTrue(self.fsck.found_errors) def test_unix_blocks(self): - obj_id = 87 inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount) " - "VALUES (?,?,?,?,?,?,?)", - (stat.S_IFSOCK | stat.S_IRUSR | stat.S_IWUSR, - os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1)) - - self.db.execute('INSERT INTO contents (name, inode, parent_inode) VALUES(?,?,?)', - ('test-entry', inode, ROOT_INODE)) + "VALUES (?,?,?,?,?,?,?)", + (stat.S_IFSOCK | stat.S_IRUSR | stat.S_IWUSR, + os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1)) + self._link('test-entry', inode) self.fsck.found_errors = False self.fsck.check_inode_unix() self.assertFalse(self.fsck.found_errors) + + obj_id = self.db.rowid('INSERT INTO objects (refcount) VALUES(1)') + block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)', + (1, obj_id, 0)) + + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)', + (inode, 1, block_id)) - self.db.execute('INSERT INTO objects (id, refcount, size) VALUES(?, ?, ?)', - (obj_id, 2, 0)) - self.db.execute('INSERT INTO blocks (inode, blockno, obj_id) VALUES(?, ?, ?)', - (inode, 1, obj_id)) self.fsck.check_inode_unix() self.assertTrue(self.fsck.found_errors) + # Somehow important according to pyunit documentation def suite(): return unittest.makeSuite(fsck_tests) diff --git a/tests/t3_inode_cache.py b/tests/t3_inode_cache.py index b1d8814..e5678da 100644 --- a/tests/t3_inode_cache.py +++ b/tests/t3_inode_cache.py @@ -3,7 +3,7 @@ t2_inode_cache.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2010 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function @@ -35,7 +35,6 @@ class cache_tests(TestCase): 'uid': 7, 'gid': 2, 'size': 34674, - 'target': 'foobar', 'rdev': 11, 'atime': time.time(), 'ctime': time.time(), @@ -46,15 +45,13 @@ class cache_tests(TestCase): for key in attrs.keys(): self.assertEqual(attrs[key], getattr(inode, key)) - self.assertTrue(self.db.has_val('SELECT 1 FROM inodes WHERE id=?', - (inode.id,))) + self.assertTrue(self.db.has_val('SELECT 1 FROM inodes WHERE id=?', (inode.id,))) def test_del(self): attrs = {'mode': 784, 'refcount': 3, 'uid': 7, - 'target': 'foobar', 'gid': 2, 'size': 34674, 'rdev': 11, @@ -71,22 +68,26 @@ class cache_tests(TestCase): 'refcount': 3, 'uid': 7, 'gid': 2, - 'target': 'foobar', 'size': 34674, 'rdev': 11, 'atime': time.time(), 'ctime': time.time(), 'mtime': time.time() } + inode = self.cache.create_inode(**attrs) - self.assertEqual(inode, self.cache[inode.id]) + for (key, val) in attrs.iteritems(): + self.assertEqual(getattr(inode, key), val) + # Create another inode + self.cache.create_inode(**attrs) + self.db.execute('DELETE FROM inodes WHERE id=?', (inode.id,)) # Entry should still be in cache self.assertEqual(inode, self.cache[inode.id]) # Now it should be out of the cache for _ in xrange(inode_cache.CACHE_SIZE + 1): - dummy = self.cache[self.cache.create_inode(**attrs).id] + self.cache.create_inode(**attrs) self.assertRaises(KeyError, self.cache.__getitem__, inode.id) diff --git a/tests/t4_adm.py b/tests/t4_adm.py index c033fd3..25cd6e5 100644 --- a/tests/t4_adm.py +++ b/tests/t4_adm.py @@ -3,20 +3,26 @@ t4_adm.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function from _common import TestCase -import unittest2 as unittest -import tempfile -import sys -import os from cStringIO import StringIO -import shutil -import s3ql.cli.mkfs -import s3ql.cli.adm from s3ql.backends import local +from s3ql.backends.common import BetterBucket +import shutil +import sys +import tempfile +import unittest2 as unittest +import subprocess +import os.path + +if __name__ == '__main__': + mypath = sys.argv[0] +else: + mypath = __file__ +BASEDIR = os.path.abspath(os.path.join(os.path.dirname(mypath), '..')) class AdmTests(TestCase): @@ -24,7 +30,7 @@ class AdmTests(TestCase): self.cache_dir = tempfile.mkdtemp() self.bucket_dir = tempfile.mkdtemp() - self.bucketname = 'local://' + os.path.join(self.bucket_dir, 'mybucket') + self.bucketname = 'local://' + self.bucket_dir self.passphrase = 'oeut3d' def tearDown(self): @@ -32,29 +38,36 @@ class AdmTests(TestCase): shutil.rmtree(self.bucket_dir) def mkfs(self): - sys.stdin = StringIO('%s\n%s\n' % (self.passphrase, self.passphrase)) - try: - s3ql.cli.mkfs.main(['--homedir', self.cache_dir, self.bucketname ]) - except BaseException as exc: - self.fail("mkfs.s3ql failed: %s" % exc) + proc = subprocess.Popen([os.path.join(BASEDIR, 'bin', 'mkfs.s3ql'), + '-L', 'test fs', '--blocksize', '500', + '--cachedir', self.cache_dir, '--quiet', + self.bucketname ], stdin=subprocess.PIPE) + + print(self.passphrase, file=proc.stdin) + print(self.passphrase, file=proc.stdin) + proc.stdin.close() + + self.assertEqual(proc.wait(), 0) def test_passphrase(self): self.mkfs() passphrase_new = 'sd982jhd' - sys.stdin = StringIO('%s\n%s\n%s\n' % (self.passphrase, - passphrase_new, passphrase_new)) - try: - s3ql.cli.adm.main(['passphrase', self.bucketname ]) - except BaseException as exc: - self.fail("s3qladm failed: %s" % exc) - - - bucket = local.Connection().get_bucket(os.path.join(self.bucket_dir, 'mybucket')) - bucket.passphrase = passphrase_new - - bucket.passphrase = bucket['s3ql_passphrase'] - self.assertTrue(isinstance(bucket['s3ql_seq_no_0'], str)) + + proc = subprocess.Popen([os.path.join(BASEDIR, 'bin', 's3qladm'), + '--quiet', 'passphrase', + self.bucketname ], stdin=subprocess.PIPE) + + print(self.passphrase, file=proc.stdin) + print(passphrase_new, file=proc.stdin) + print(passphrase_new, file=proc.stdin) + proc.stdin.close() + + self.assertEqual(proc.wait(), 0) + + plain_bucket = local.Bucket(self.bucket_dir, None, None) + bucket = BetterBucket(passphrase_new, 'bzip2', plain_bucket) + self.assertTrue(isinstance(bucket['s3ql_passphrase'], str)) # Somehow important according to pyunit documentation diff --git a/tests/t4_fuse.py b/tests/t4_fuse.py index 8c5daa1..aa1a90c 100644 --- a/tests/t4_fuse.py +++ b/tests/t4_fuse.py @@ -3,32 +3,130 @@ t4_fuse.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' -from __future__ import division, print_function +from __future__ import absolute_import, division, print_function from _common import TestCase -from cStringIO import StringIO from os.path import basename -from s3ql.common import retry, AsyncFn import filecmp +import llfuse import os.path -import s3ql.cli.fsck -import s3ql.cli.mkfs -import s3ql.cli.mount -import s3ql.cli.umount import shutil import stat -import llfuse import subprocess import sys import tempfile +import threading import time +import traceback import unittest2 as unittest +import logging + +log = logging.getLogger() # For debugging USE_VALGRIND = False +class ExceptionStoringThread(threading.Thread): + def __init__(self): + super(ExceptionStoringThread, self).__init__() + self._exc_info = None + self._joined = False + + def run_protected(self): + pass + + def run(self): + try: + self.run_protected() + except: + # This creates a circular reference chain + self._exc_info = sys.exc_info() + + def join_get_exc(self): + self._joined = True + self.join() + return self._exc_info + + def join_and_raise(self): + '''Wait for the thread to finish, raise any occurred exceptions''' + + self._joined = True + if self.is_alive(): + self.join() + + if self._exc_info is not None: + # Break reference chain + exc_info = self._exc_info + self._exc_info = None + raise EmbeddedException(exc_info, self.name) + + def __del__(self): + if not self._joined: + raise RuntimeError("ExceptionStoringThread instance was destroyed " + "without calling join_and_raise()!") + +class EmbeddedException(Exception): + '''Encapsulates an exception that happened in a different thread + ''' + + def __init__(self, exc_info, threadname): + super(EmbeddedException, self).__init__() + self.exc_info = exc_info + self.threadname = threadname + + log.error('Thread %s terminated with exception:\n%s', + self.threadname, ''.join(traceback.format_exception(*self.exc_info))) + + def __str__(self): + return ''.join(['caused by an exception in thread %s.\n' % self.threadname, + 'Original/inner traceback (most recent call last): \n' ] + + traceback.format_exception(*self.exc_info)) + +class AsyncFn(ExceptionStoringThread): + def __init__(self, fn, *args, **kwargs): + super(AsyncFn, self).__init__() + self.target = fn + self.args = args + self.kwargs = kwargs + + def run_protected(self): + self.target(*self.args, **self.kwargs) + +def retry(timeout, fn, *a, **kw): + """Wait for fn(*a, **kw) to return True. + + If the return value of fn() returns something True, this value + is returned. Otherwise, the function is called repeatedly for + `timeout` seconds. If the timeout is reached, `TimeoutError` is + raised. + """ + + step = 0.2 + waited = 0 + while waited < timeout: + ret = fn(*a, **kw) + if ret: + return ret + time.sleep(step) + waited += step + if step < waited / 30: + step *= 2 + + raise TimeoutError() + +class TimeoutError(Exception): + '''Raised by `retry()` when a timeout is reached.''' + + pass + +if __name__ == '__main__': + mypath = sys.argv[0] +else: + mypath = __file__ +BASEDIR = os.path.abspath(os.path.join(os.path.dirname(mypath), '..')) + class fuse_tests(TestCase): def setUp(self): @@ -41,102 +139,70 @@ class fuse_tests(TestCase): self.cache_dir = tempfile.mkdtemp() self.bucket_dir = tempfile.mkdtemp() - self.bucketname = 'local://' + os.path.join(self.bucket_dir, 'mybucket') + self.bucketname = 'local://' + self.bucket_dir self.passphrase = 'oeut3d' - self.mount_thread = None + self.mount_process = None self.name_cnt = 0 - - def tearDown(self): - # Umount if still mounted - if os.path.ismount(self.mnt_dir): - subprocess.call(['fusermount', '-z', '-u', self.mnt_dir]) - - # Try to wait for mount thread to prevent spurious errors - # because the db file is being removed - if self.mount_thread and USE_VALGRIND: - retry(60, lambda: self.mount_thread.poll() is not None) - elif self.mount_thread: - self.mount_thread.join(60) - - shutil.rmtree(self.mnt_dir) - shutil.rmtree(self.cache_dir) - shutil.rmtree(self.bucket_dir) - - if not USE_VALGRIND and not self.mount_thread.is_alive(): - self.mount_thread.join_and_raise() - def mount(self): + def mkfs(self): + proc = subprocess.Popen([os.path.join(BASEDIR, 'bin', 'mkfs.s3ql'), + '-L', 'test fs', '--blocksize', '500', + '--cachedir', self.cache_dir, '--quiet', + self.bucketname ], stdin=subprocess.PIPE) - sys.stdin = StringIO('%s\n%s\n' % (self.passphrase, self.passphrase)) - try: - s3ql.cli.mkfs.main(['-L', 'test fs', '--blocksize', '500', - '--homedir', self.cache_dir, self.bucketname ]) - except BaseException as exc: - self.fail("mkfs.s3ql failed: %s" % exc) - + print(self.passphrase, file=proc.stdin) + print(self.passphrase, file=proc.stdin) + proc.stdin.close() + + self.assertEqual(proc.wait(), 0) - # Note: When running inside test suite, we have less available - # file descriptors - if USE_VALGRIND: - if __name__ == '__main__': - mypath = sys.argv[0] - else: - mypath = __file__ - basedir = os.path.abspath(os.path.join(os.path.dirname(mypath), '..')) - self.mount_thread = subprocess.Popen(['valgrind', 'python-dbg', - os.path.join(basedir, 'bin', 'mount.s3ql'), - "--fg", '--homedir', self.cache_dir, - '--max-cache-entries', '500', + def mount(self): + self.mount_process = subprocess.Popen([os.path.join(BASEDIR, 'bin', 'mount.s3ql'), + "--fg", '--cachedir', self.cache_dir, + '--log', 'none', '--quiet', self.bucketname, self.mnt_dir], stdin=subprocess.PIPE) - print(self.passphrase, file=self.mount_thread.stdin) - retry(30, os.path.ismount, self.mnt_dir) - else: - sys.stdin = StringIO('%s\n' % self.passphrase) - self.mount_thread = AsyncFn(s3ql.cli.mount.main, - ["--fg", '--homedir', self.cache_dir, - '--max-cache-entries', '500', - self.bucketname, self.mnt_dir]) - self.mount_thread.start() - - # Wait for mountpoint to come up - try: - retry(3, os.path.ismount, self.mnt_dir) - except: - self.mount_thread.join_and_raise() + print(self.passphrase, file=self.mount_process.stdin) + self.mount_process.stdin.close() + retry(30, os.path.ismount, self.mnt_dir) def umount(self): - time.sleep(0.5) devnull = open('/dev/null', 'wb') retry(5, lambda: subprocess.call(['fuser', '-m', self.mnt_dir], stdout=devnull, stderr=devnull) == 1) - s3ql.cli.umount.DONTWAIT = True - try: - s3ql.cli.umount.main([self.mnt_dir]) - except BaseException as exc: - self.fail("Umount failed: %s" % exc) - - # Now wait for server process - if USE_VALGRIND: - self.assertEqual(self.mount_thread.wait(), 0) - else: - exc = self.mount_thread.join_get_exc() - self.assertIsNone(exc) + + subprocess.check_call([os.path.join(BASEDIR, 'bin', 'umount.s3ql'), + '--quiet', self.mnt_dir]) + self.assertEqual(self.mount_process.wait(), 0) self.assertFalse(os.path.ismount(self.mnt_dir)) - # Now run an fsck - sys.stdin = StringIO('%s\n' % self.passphrase) - try: - s3ql.cli.fsck.main(['--force', '--homedir', self.cache_dir, - self.bucketname]) - except BaseException as exc: - self.fail("fsck failed: %s" % exc) + def fsck(self): + proc = subprocess.Popen([os.path.join(BASEDIR, 'bin', 'fsck.s3ql'), + '--force', '--quiet', '--log', 'none', + '--cachedir', self.cache_dir, + self.bucketname ], stdin=subprocess.PIPE) + print(self.passphrase, file=proc.stdin) + proc.stdin.close() + self.assertEqual(proc.wait(), 0) + + def tearDown(self): + subprocess.call(['fusermount', '-z', '-u', self.mnt_dir], + stderr=open('/dev/null', 'wb')) + os.rmdir(self.mnt_dir) + + # Give mount process a little while to terminate + retry(10, lambda : self.mount_process.poll() is not None) + + shutil.rmtree(self.cache_dir) + shutil.rmtree(self.bucket_dir) + def runTest(self): # Run all tests in same environment, mounting and umounting # just takes too long otherwise + self.mkfs() self.mount() self.tst_chown() self.tst_link() @@ -146,9 +212,23 @@ class fuse_tests(TestCase): self.tst_statvfs() self.tst_symlink() self.tst_truncate() + self.tst_truncate_nocache() self.tst_write() self.umount() - + self.fsck() + + # Empty cache + shutil.rmtree(self.cache_dir) + self.cache_dir = tempfile.mkdtemp() + + self.mount() + self.umount() + + # Empty cache + shutil.rmtree(self.cache_dir) + self.cache_dir = tempfile.mkdtemp() + self.fsck() + def newname(self): self.name_cnt += 1 return "s3ql_%d" % self.name_cnt @@ -290,7 +370,28 @@ class fuse_tests(TestCase): os.close(fd) os.unlink(filename) + def tst_truncate_nocache(self): + filename = os.path.join(self.mnt_dir, self.newname()) + src = self.src + shutil.copyfile(src, filename) + self.assertTrue(filecmp.cmp(filename, src, False)) + fstat = os.stat(filename) + size = fstat.st_size + subprocess.check_call([os.path.join(BASEDIR, 'bin', 's3qlctrl'), + '--quiet', 'flushcache', self.mnt_dir ]) + + fd = os.open(filename, os.O_RDWR) + + os.ftruncate(fd, size + 1024) # add > 1 block + self.assertEquals(os.stat(filename).st_size, size + 1024) + + os.ftruncate(fd, size - 1024) # Truncate > 1 block + self.assertEquals(os.stat(filename).st_size, size - 1024) + + os.close(fd) + os.unlink(filename) + # Somehow important according to pyunit documentation def suite(): return unittest.makeSuite(fuse_tests) diff --git a/tests/t5_cli.py b/tests/t5_cli.py index 8e6bf68..c2bae22 100644 --- a/tests/t5_cli.py +++ b/tests/t5_cli.py @@ -3,33 +3,37 @@ t5_cli.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function -import os.path import errno +import llfuse +import os.path import s3ql.cli.ctrl import s3ql.cli.lock import s3ql.cli.remove -import llfuse -import unittest2 as unittest +import sys import t4_fuse +import unittest2 as unittest class cliTests(t4_fuse.fuse_tests): def runTest(self): + self.mkfs() self.mount() self.tst_lock_rm() self.tst_ctrl_flush() self.umount() + self.fsck() def tst_ctrl_flush(self): try: s3ql.cli.ctrl.main(['flushcache', self.mnt_dir]) - except BaseException as exc: - self.fail("s3qladm failed: %s" % exc) + except: + sys.excepthook(*sys.exc_info()) + self.fail("s3qlctrl raised exception") def tst_lock_rm(self): @@ -43,8 +47,9 @@ class cliTests(t4_fuse.fuse_tests): # copy try: s3ql.cli.lock.main([tempdir]) - except BaseException as exc: - self.fail("s3qllock failed: %s" % exc) + except: + sys.excepthook(*sys.exc_info()) + self.fail("s3qllock raised exception") # Try to delete with self.assertRaises(OSError) as cm: @@ -59,8 +64,9 @@ class cliTests(t4_fuse.fuse_tests): # delete properly try: s3ql.cli.remove.main([tempdir]) - except BaseException as exc: - self.fail("s3qlrm failed: %s" % exc) + except: + sys.excepthook(*sys.exc_info()) + self.fail("s3qlrm raised exception") self.assertTrue('lock_dir' not in llfuse.listdir(self.mnt_dir)) diff --git a/tests/t5_cp.py b/tests/t5_cp.py index f7f758c..9d03ff1 100644 --- a/tests/t5_cp.py +++ b/tests/t5_cp.py @@ -3,19 +3,17 @@ t5_cp.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from __future__ import division, print_function +import errno import os.path -from s3ql.cli.cp import main as s3qlcp import subprocess +import t4_fuse import tarfile import tempfile -import errno import unittest2 as unittest -import t4_fuse - class cpTests(t4_fuse.fuse_tests): @@ -29,11 +27,12 @@ class cpTests(t4_fuse.fuse_tests): raise unittest.SkipTest('rsync not installed') raise + self.mkfs() self.mount() self.tst_cp() - self.umount() - + self.fsck() + def tst_cp(self): # Extract tar @@ -46,11 +45,10 @@ class cpTests(t4_fuse.fuse_tests): os.path.join(self.mnt_dir, 'orig') + '/']) # copy - try: - s3qlcp([os.path.join(self.mnt_dir, 'orig'), - os.path.join(self.mnt_dir, 'copy')]) - except BaseException as exc: - self.fail("s3qlcp failed: %s" % exc) + subprocess.check_call([os.path.join(t4_fuse.BASEDIR, 'bin', 's3qlcp'), + '--quiet', + os.path.join(self.mnt_dir, 'orig'), + os.path.join(self.mnt_dir, 'copy')]) # compare rsync = subprocess.Popen(['rsync', '-anciHAX', '--delete', diff --git a/util/cmdline_lexer.py b/util/cmdline_lexer.py index ac6df12..4927325 100644 --- a/util/cmdline_lexer.py +++ b/util/cmdline_lexer.py @@ -4,7 +4,7 @@ cmdline_lexer.py - this file is part of S3QL (http://s3ql.googlecode.com) Copyright (C) 2008-2009 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from pygments.token import Comment, Name, Generic, Literal diff --git a/util/sphinx_pipeinclude.py b/util/sphinx_pipeinclude.py index 3434825..3304350 100644 --- a/util/sphinx_pipeinclude.py +++ b/util/sphinx_pipeinclude.py @@ -8,7 +8,7 @@ to include the output of a program. Copyright (C) 2008-2011 Nikolaus Rath <Nikolaus@rath.org> -This program can be distributed under the terms of the GNU LGPL. +This program can be distributed under the terms of the GNU GPLv3. ''' from docutils.parsers.rst.directives.misc import Include |