From 0ff390756d6be25ba06c24bcfa0589d646e0cec6 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Fri, 28 Oct 2016 14:07:23 -0700 Subject: Import s3ql_2.21+dfsg-1.debian.tar.xz [dgit import tarball s3ql 2.21+dfsg-1 s3ql_2.21+dfsg-1.debian.tar.xz] --- .git-dpm | 11 + NEWS | 78 ++++ README.Debian | 22 + README.source | 20 + changelog | 416 +++++++++++++++++ compat | 1 + control | 106 +++++ copyright | 24 + patches/clock-granularity.diff | 32 ++ patches/ignore_cython_warnings.diff | 34 ++ patches/proc_mount.diff | 32 ++ patches/series | 5 + patches/show_pdflatex_output.diff | 30 ++ patches/support_jessie_upgrade.diff | 876 ++++++++++++++++++++++++++++++++++++ py3dist-overrides | 3 + rules | 112 +++++ s3ql-dbg.links | 1 + s3ql.doc-base | 11 + s3ql.docs | 2 + s3ql.examples | 1 + s3ql.install | 4 + s3ql.links | 3 + source/format | 1 + source/include-binaries | 2 + source/options | 2 + tests/control | 10 + tests/upstream-standard | 7 + tests/upstream-with-fuse | 4 + upstream-signing-key.pgp | Bin 0 -> 2722 bytes watch | 9 + 30 files changed, 1859 insertions(+) create mode 100644 .git-dpm create mode 100644 NEWS create mode 100644 README.Debian create mode 100644 README.source create mode 100644 changelog create mode 100644 compat create mode 100644 control create mode 100644 copyright create mode 100644 patches/clock-granularity.diff create mode 100644 patches/ignore_cython_warnings.diff create mode 100644 patches/proc_mount.diff create mode 100644 patches/series create mode 100644 patches/show_pdflatex_output.diff create mode 100644 patches/support_jessie_upgrade.diff create mode 100644 py3dist-overrides create mode 100755 rules create mode 100644 s3ql-dbg.links create mode 100644 s3ql.doc-base create mode 100644 s3ql.docs create mode 100644 s3ql.examples create mode 100644 s3ql.install create mode 100644 s3ql.links create mode 100644 source/format create mode 100644 source/include-binaries create mode 100644 source/options create mode 100644 tests/control create mode 100755 tests/upstream-standard create mode 100755 tests/upstream-with-fuse create mode 100644 upstream-signing-key.pgp create mode 100644 watch diff --git a/.git-dpm b/.git-dpm new file mode 100644 index 0000000..cec922a --- /dev/null +++ b/.git-dpm @@ -0,0 +1,11 @@ +# see git-dpm(1) from git-dpm package +f2e46743e3b446d31cdda56846e7d2e7c384e2a0 +f2e46743e3b446d31cdda56846e7d2e7c384e2a0 +7b20150ca23d8ce6e325799dbf735d26dfd0f03e +7b20150ca23d8ce6e325799dbf735d26dfd0f03e +s3ql_2.21+dfsg.orig.tar.xz +8e720bc48e67a202d85af5e60812c5bceda687fd +530132 +debianTag="debian/%e%v" +patchedTag="patched/%e%v" +upstreamTag="upstream/%e%u" diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..f79ba3f --- /dev/null +++ b/NEWS @@ -0,0 +1,78 @@ +s3ql (2.17+dfsg-1) unstable; urgency=medium + + The internal file system revision has changed. File systems created + with S3QL 2.17 or newer are not compatible with prior S3QL + versions. To update an existing file system to the newest revision, + use the `s3qladm upgrade` command. + + -- Nikolaus Rath Thu, 10 Mar 2016 08:13:37 -0800 + +s3ql (2.14+dfsg-1) unstable; urgency=medium + + The internal file system revision has changed. File systems created + with S3QL 2.14 or newer are not compatible with prior S3QL + versions. To update an existing file system to the newest revision, + use the 's3qladm upgrade' command. + + -- Nikolaus Rath Wed, 12 Aug 2015 14:57:22 -0700 + +s3ql (2.10.1+dfsg-1) unstable; urgency=medium + + The internal file system revision has changed. File systems created + with S3QL 2.10 or newer are not compatible with prior S3QL + versions. To update an existing file system to the newest revision, + use the 's3qladm upgrade' command. + + It is strongly recommended to run the (new) s3ql_verify command with + the --data option at shortly after the upgrade. This is necessary to + ensure that the upgrade to the next (2.11) S3QL release will run + smoothly. + + -- Nikolaus Rath Sun, 03 Aug 2014 20:49:06 -0700 + +s3ql (2.5-1) unstable; urgency=low + + * The file system structure has changed, existing file systems + need to be upgraded using "s3qladm upgrade" before they can + be used. + + -- Nikolaus Rath Sat, 19 Oct 2013 16:34:46 -0700 + +s3ql (1.11-1) unstable; urgency=low + + * The file system structure has changed, existing file systems + need to be upgraded using "s3qladm upgrade" before they can + be used. + + -- Nikolaus Rath Sun, 29 Apr 2012 15:23:20 -0400 + +s3ql (1.9-1) unstable; urgency=low + + * The file system structure has changed, existing file systems + need to be upgraded using "s3qladm upgrade" before they can + be used. + + -- Nikolaus Rath Sat, 21 Jan 2012 13:49:59 -0500 + +s3ql (1.7-1) unstable; urgency=low + + * The file system structure has changed, existing file systems + need to be upgraded using "s3qladm upgrade" before they can + be used. + + -- Nikolaus Rath Sun, 27 Nov 2011 16:43:06 -0500 + +s3ql (1.4-1) unstable; urgency=low + + Notes for upgrading from version 1.0 or earlier: + ------------------------------------------------ + + * The file system structure has changed, existing file systems + need to be upgraded using "s3qladm upgrade" before they can + be used. + + * The format and default name of the authentication file has + changed. The default location is now ~/.s3ql/authinfo2. + Please refer to the documentation for the file format. + + -- Nikolaus Rath Thu, 06 Oct 2011 11:25:44 -0400 diff --git a/README.Debian b/README.Debian new file mode 100644 index 0000000..3558364 --- /dev/null +++ b/README.Debian @@ -0,0 +1,22 @@ +S3QL for Debian +--------------- + + +Q: Where do I find the contents of the contrib/ directory in the + upstream tarball? + +A: The Python scripts are installed into /usr/lib/s3ql. Manpages have + been placed in the proper system directory (/usr/man/man1) and can + be viewed with the standard `man` command. The example sample + backup script (s3ql_backup.sh) is installed in + /usr/share/doc/s3ql/examples. + + Additionally, some scripts have been linked into /usr/bin under + different names (in these cases the manpages have been renamed + accordingly): + + Installed Name: Original name: + ------------------------------------------------------------- + expire_backups expire_backups.py + parallel-cp pcp.py + s3ql_remove_objects remove_objects.py diff --git a/README.source b/README.source new file mode 100644 index 0000000..e2a53fa --- /dev/null +++ b/README.source @@ -0,0 +1,20 @@ + +* As requested by ftpmaster, the upstream tarball is repacked to + remove minified javascript files. + +* Since removing just the minified javascript files would leave the + rest of them broken, we are removing the entire pre-rendered HTML + documentation. There is no loss of information, since the entire + documentation (including the problematic javascript files) is + regenerated during build. + +* Repackaging of the upstream tarball is done with uscan. The list of + excluded files is specified in the Files-Excluded paragraph in + debian/copyright. To retrieve and repack the latest upstream + tarball, run + + # PERL_LWP_SSL_VERIFY_HOSTNAME=0 uscan --repack + + (Deactivating hostname verification isn't a problem because the + upstream tarball is GPG signed, but necessary to work around + bug #749225.) diff --git a/changelog b/changelog new file mode 100644 index 0000000..72e236a --- /dev/null +++ b/changelog @@ -0,0 +1,416 @@ +s3ql (2.21+dfsg-1) unstable; urgency=medium + + * New upstream version + + -- Nikolaus Rath Fri, 28 Oct 2016 14:07:23 -0700 + +s3ql (2.20+dfsg-1) unstable; urgency=medium + + * new upstream version + + -- Nikolaus Rath Wed, 24 Aug 2016 12:10:58 -0700 + +s3ql (2.19+dfsg-1) unstable; urgency=medium + + * new upstream version + * Show full pdflatex output during build + * Added missing texlive-generic-extra dependency. + + -- Nikolaus Rath Mon, 27 Jun 2016 12:00:39 -0700 + +s3ql (2.18+dfsg-1) unstable; urgency=medium + + * Enable debug logging for s3ql.verify when running unit tests during + build. + * Moved packaging to git and collab-maint on alioth. + * New upstream release. + + -- Nikolaus Rath Fri, 29 Apr 2016 19:08:53 -0700 + +s3ql (2.17.1+hg2+dfsg-3) unstable; urgency=medium + + * Adding python3-pytest-catchlog dependency for unit tests. + + -- Nikolaus Rath Sun, 13 Mar 2016 14:39:48 -0700 + +s3ql (2.17.1+hg2+dfsg-2) unstable; urgency=medium + + * Fixed up broken rebase of patches/support_jessie_upgrade.diff + + -- Nikolaus Rath Fri, 11 Mar 2016 19:19:12 -0800 + +s3ql (2.17.1+hg2+dfsg-1) unstable; urgency=medium + + * Added forgotten NEWS entry for 2.17+dfsg-1 + * New release from upstream repository snapshot. + * Run tests with (some) debug logging enabled to debug sporadic test + failure of tests/t3_verify.py. + + -- Nikolaus Rath Fri, 11 Mar 2016 16:18:25 -0800 + +s3ql (2.17.1+dfsg-1) unstable; urgency=medium + + * New upstream release. + + -- Nikolaus Rath Wed, 09 Mar 2016 14:33:57 -0800 + +s3ql (2.17+dfsg-1) unstable; urgency=medium + + * New upstream version. + + -- Nikolaus Rath Wed, 09 Mar 2016 11:13:23 -0800 + +s3ql (2.16+dfsg-1) unstable; urgency=medium + + * Added dependency on python3-systemd. + * New upstream release. + + -- Nikolaus Rath Tue, 23 Feb 2016 15:44:51 -0800 + +s3ql (2.15+dfsg-1) unstable; urgency=medium + + * Add Breaks and Replaces to prevent upgrade problems due to the + extension for the debug interpreter having moved from s3ql-dbg to s3ql in + 2.14+dfsg-1. Closes: #799261. + * New upstream release. + * Dropped fix_setattr_test.diff (fixed upstream). + * Added use_cython3.diff and ignore_cython_warnings.diff to handle + Debian-specific Cython peculiarities. + + -- Nikolaus Rath Fri, 25 Sep 2015 13:50:51 -0700 + +s3ql (2.14+dfsg-1) unstable; urgency=medium + + * New upstream release. + * Dropped patches/check_dev_fuse_perms.diff (integrated upstream). + * Added /usr/bin/s3ql_remove_objects + * Added build-depends on fuse, rsync and psmisc (required for unit tests). + * Python extensions compiled for the debugging interpreter are now + actually shipped in the s3ql-dbg package (rather than the s3ql + package). + * Added patches/support_jessie_upgrade.diff (upstream dropped + compatibility with jessie's S3QL version). + * Added patches/fix_setattr_test.diff to un-break a unit test + that dependended on undocumented behavior in python3-llfuse (which changed + recently). Closes: #798596. + * Added lost changelog entry for 2.11.1+dfsg-2. Closes: #798595. + + -- Nikolaus Rath Thu, 10 Sep 2015 13:40:00 -0700 + +s3ql (2.13+dfsg-2) unstable; urgency=medium + + * Tighten dependency on python3-dugong, python3-apsw, and + python3-llfuse. Closes: #778487. + + * Added missing build-dependency on python3-llfuse-dbg. + Closes: #790525. + + -- Nikolaus Rath Wed, 01 Jul 2015 02:33:26 -0800 + +s3ql (2.13+dfsg-1) unstable; urgency=medium + + * New upstream release. + * Dropped patches/fix_test_check_objects_temp.diff (fixed upstream). + * Dropped patches/debug_test_thread_hang.diff (fixed upstream). + * Fixed a bug that caused fsck.s3ql to either abort with a + "apsw.ConstraintError" or to incorrectly consider storage objects + as missing when the connection to remote server is interrupted + while retrieving the object list. Closes: #771452. + * Fixed a problem where mount.s3ql would crash when unmouting the + file system because it could not delete the cache directory. This + could happen when the file system was not unmounted cleanly, but + fsck.s3ql was then run on a different system (or using a different + cache directory). Closes: #772052. + + -- Nikolaus Rath Sat, 31 Jan 2015 20:16:12 -0800 + +s3ql (2.12+dfsg-2) experimental; urgency=medium + + * Added patches/debug_test_thread_hang.diff to debug occasional test + failures on buildds. + * Added patches/fix_test_check_objects_temp.diff (cherry-picked + upstream) to fix occasional test failures during build. + * Upload sponsored by Petter Reinholdtsen. + + -- Nikolaus Rath Mon, 10 Nov 2014 18:58:07 -0800 + +s3ql (2.12+dfsg-1) experimental; urgency=medium + + * New upstream release. + * Update Standards-Version to 3.9.6 (no changes needed). + * Upload sponsored by Petter Reinholdtsen. + + -- Nikolaus Rath Sun, 09 Nov 2014 15:05:23 -0800 + +s3ql (2.11.1+dfsg-2) unstable; urgency=medium + + * Fixed a problem with fsck.s3ql aborting with an + "apsw.ConstraintError" or incorrectly considering storage + objects as missing when the connection to remote server is + interrupted. Closes: #771452. + * Upload sponsored by Petter Reinholdtsen. + + -- Nikolaus Rath Tue, 02 Dec 2014 21:44:27 -0800 + +s3ql (2.11.1+dfsg-1) unstable; urgency=medium + + * New upstream release. + * Dropped patches/cve_2014_0485.diff, integrated upstream. + * Dropped patches/fix_failsafe_test_race.diff, integrated upstream. + * Upload sponsored by Petter Reinholdtsen. + + -- Nikolaus Rath Thu, 04 Sep 2014 18:28:41 -0700 + +s3ql (2.10.1+dfsg-4) unstable; urgency=high + + * SECURITY UPDATE for CVE-2014-0485. + + A remote code execution vulnerability was fixed. + + An attacker with control over the communication with the storage + backend or the ability to manipulate the data stored in the + backend was able to trigger execution of arbitrary code by + mount.s3ql, fsck.s3ql, mkfs.s3ql, s3qladm and s3ql_verify. Both + encrypted and unencrypted file systems were vulnerable. + + * Upload sponsored by Petter Reinholdtsen. + + -- Nikolaus Rath Mon, 25 Aug 2014 17:21:09 -0700 + +s3ql (2.10.1+dfsg-3) unstable; urgency=medium + + * Added fix_failsafe_test_race.diff. Closes: #758013. + + -- Nikolaus Rath Wed, 20 Aug 2014 17:30:03 -0700 + +s3ql (2.10.1+dfsg-2) unstable; urgency=medium + + * Bumped python3-dugong dependency to 3.3 to fix build failures + on kFreeBSD (and presumably also similar problems when running + the software on both kFreeBSD and Linux). + + -- Nikolaus Rath Wed, 06 Aug 2014 18:55:19 -0700 + +s3ql (2.10.1+dfsg-1) unstable; urgency=medium + + * Made system clock resolution heuristic in unittests even more + conservative, there are still occasional build failures on i386. + * New upstream release. + * Dropped patches (integrated upstream): + - C_locale_compat.diff + - kfreebsd_compat.diff + - sphinx_build_locale_fix.diff + * Bumped python3-dugong dependency to 3.2 + * Upload sponsored by Petter Reinholdtsen. + + -- Nikolaus Rath Sun, 03 Aug 2014 20:49:06 -0700 + +s3ql (2.9+dfsg-2) unstable; urgency=medium + + * Cherry-picked kFreeBSD compatibility patch from upstream + (Closes: #755358) + * Cherry-picked upstream patch to support testing with C + locale. + * Upload sponsored by Petter Reinholdtsen. + + -- Nikolaus Rath Sat, 19 Jul 2014 20:24:55 -0700 + +s3ql (2.9+dfsg-1) unstable; urgency=medium + + * New upstream release. + - Fix automatic self test when running as root (Closes: #745476) + * Dropped patches/sphinx-1.1.1-compat.diff (no longer required now + that Sphinx 1.2 is in unstable). + * Added patches/sphinx_build_locale_fix.diff (cherry-picked from upstream) + to make documentation build work independent of system locale. + * Added patches/check_dev_fuse_perms.diff to avoid running FUSE + tests on continuous integration servers where /dev/fuse is + not accessible. + * Upload sponsored by Petter Reinholdtsen. + + -- Nikolaus Rath Sat, 28 Jun 2014 12:30:45 -0700 + +s3ql (2.8.1+dfsg-1) unstable; urgency=low + + * Updated upstream project URL. + * Added upstream PGP key and configured watch file for automatic + verification. + * New upstream release. + * Do not crash when building with snapshot Cython versions + (Closes: #742760) + * Update Standards-Version to 3.9.5 (no changes needed). + * Repackaged to exclude minified javascript files. + * Upload sponsored by Petter Reinholdtsen. + + -- Nikolaus Rath Fri, 14 Mar 2014 19:11:06 -0700 + +s3ql (2.7-1) unstable; urgency=low + + * New upstream release. + * Estimate clock resolution more conservatively when running unit + tests. + + -- Nikolaus Rath Mon, 16 Dec 2013 20:21:41 -0800 + +s3ql (2.5-2) unstable; urgency=low + + * Do not try to write logfiles into $HOME when running self tests. + + -- Nikolaus Rath Sun, 20 Oct 2013 11:38:32 -0700 + +s3ql (2.5-1) unstable; urgency=low + + * Added patch for Sphinx 1.1.3 compatibility. + * New upstream release. + * Fixed autopkgtests (this time a missing PYTHONPATH, LP: #1223514). + + -- Nikolaus Rath Tue, 27 Aug 2013 20:17:06 -0700 + +s3ql (1.16-2) unstable; urgency=low + + * Updated uploader email address. + * Removed obsolete lintian override. + * Extended clean target, re-build should now work. + * Updated package description. + * Bump build dependency on Cython to 0.17 + * Fixed autopkgtests to use installed S3QL. + + -- Nikolaus Rath Thu, 18 Jul 2013 21:41:29 -0700 + +s3ql (1.16-1) unstable; urgency=low + + * New upstream release. + + -- Nikolaus Rath Fri, 12 Jul 2013 21:11:01 -0700 + +s3ql (1.15-1) unstable; urgency=low + + [ Jakub Wilk ] + * Use canonical URIs for Vcs-* fields. + + [ Nikolaus Rath ] + * New upstream release. + * Added autopkgtest data. + + -- Nikolaus Rath Sat, 15 Jun 2013 15:06:37 -0700 + +s3ql (1.14-1) unstable; urgency=low + + * Update standards version to 3.9.4, no other changes + required. + * New upstream release. + + -- Nikolaus Rath Tue, 09 Apr 2013 19:08:15 -0700 + +s3ql (1.13.1-1) unstable; urgency=low + + * New upstream version. + * Fixed metadata corruption with recent eglibc (triggered test + failure on build). Closes: #701350 + + -- Nikolaus Rath Sat, 23 Feb 2013 17:46:56 -0800 + +s3ql (1.12-1) unstable; urgency=low + + * New upstream release. + * Bumped debhelper compatibility level to 9 to get support for + dpkg-buildflags. + * Added lintian override for hardening-no-fortify-functions, + this is a false positive. + * Added patches/buildd-fix.diff to prevent errors when trying + to write logs during autobuilding. + + -- Nikolaus Rath Wed, 05 Sep 2012 20:22:18 -0400 + +s3ql (1.11.1-2) unstable; urgency=low + + * Add dependency on python-pkg-resources. Closes: 672916. + + -- Nikolaus Rath Tue, 15 May 2012 18:41:38 -0400 + +s3ql (1.11.1-1) unstable; urgency=low + + * New upstream release. + + -- Nikolaus Rath Fri, 04 May 2012 11:12:29 -0400 + +s3ql (1.11-1) unstable; urgency=low + + * New upstream release. + + -- Nikolaus Rath Sun, 29 Apr 2012 15:22:56 -0400 + +s3ql (1.10-1) unstable; urgency=low + + * Bumped standards to 3.9.3, no changes required. + * New upstream release. + + -- Nikolaus Rath Fri, 24 Feb 2012 21:51:01 -0500 + +s3ql (1.9-1) unstable; urgency=low + + * Depend on python >= 2.7 (now required by upstream). Closes: #653641. + * New upstream release. + * Dropped obsolete argparse.diff patch. + + -- Nikolaus Rath Sat, 21 Jan 2012 13:38:25 -0500 + +s3ql (1.8.1-1) unstable; urgency=low + + * Add versioned depends on python-argparse. Closes: #652751. + * New upstream release. + + -- Nikolaus Rath Mon, 05 Dec 2011 20:36:44 -0500 + +s3ql (1.7-1) unstable; urgency=low + + * New upstream release. + + -- Nikolaus Rath Sun, 27 Nov 2011 14:40:58 -0500 + +s3ql (1.6-1) unstable; urgency=low + + * New upstream release. + + -- Nikolaus Rath Tue, 22 Nov 2011 11:04:35 -0500 + +s3ql (1.5-1) unstable; urgency=low + + * New upstream release. + * Don't crash when using Google Storage backend. Closes: #646232. + * Use dh_sphinxdoc. + + -- Nikolaus Rath Mon, 24 Oct 2011 19:16:12 -0400 + +s3ql (1.4-1) unstable; urgency=low + + * New upstream release. + + * Depend on fuse | fusebsd. Closes: #634337. + + -- Nikolaus Rath Thu, 06 Oct 2011 11:20:25 -0400 + +s3ql (1.2-1) unstable; urgency=low + + * New upstream release. + * Depend on python-llfuse >= 0.36, API was changed in non-backwards + compatible way. + * Depend on Python >= 2.6.6, previous versions unusable due to + Python bug 6312. + * Don't ship our own version of underscore JavaScript library, depend + on libjs-underscore package instead. + + -- Nikolaus Rath Wed, 28 Sep 2011 21:00:04 -0400 + +s3ql (1.0.1-2) unstable; urgency=low + + * Install pcp as parallel-cp to avoid file name conflict with + pcp ("Performance Co-Pilot") package. Closes: #632473. + + -- Nikolaus Rath Sun, 03 Jul 2011 10:44:00 -0400 + +s3ql (1.0.1-1) unstable; urgency=low + + * First official debian release. Closes: #626651 + + -- Nikolaus Rath Fri, 01 Jul 2011 14:02:17 -0400 diff --git a/compat b/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/compat @@ -0,0 +1 @@ +9 diff --git a/control b/control new file mode 100644 index 0000000..21b6fd3 --- /dev/null +++ b/control @@ -0,0 +1,106 @@ +Source: s3ql +Section: misc +Priority: optional +X-Python-Version: >= 3.3 +Maintainer: Nikolaus Rath +Uploaders: Debian Python Apps Team +Build-Depends: debhelper (>= 9), + python3 (>= 3.3), + python3-dev (>= 3.3), + python3-dbg, + python3-setuptools (>= 0.6.14), + python3-apsw (>= 3.7.0), + python3-sphinx (>= 1.2), + python3-llfuse (>= 1.0), python3-llfuse (<< 2.0), + python3-llfuse-dbg, + python3-crypto, + python3-requests, + python3-dugong (>= 3.4), + python3-pytest (>= 2.3.3), + python3-pytest-catchlog, + python3-defusedxml, + cython3 (>= 0.23), + texlive-latex-base, + texlive-latex-recommended, + texlive-latex-extra, + texlive-generic-extra, + texlive-fonts-recommended, + libsqlite3-dev (>= 3.7.0), + rsync, + fuse [linux-any], fuse4bsd [kfreebsd-any], + psmisc +Standards-Version: 3.9.6 +Homepage: https://bitbucket.org/nikratio/s3ql/ +Vcs-Git: git://anonscm.debian.org/collab-maint/s3ql.git +Vcs-Browser: https://anonscm.debian.org/gitweb/?p=collab-maint/s3ql.git + +Package: s3ql +Architecture: any +Depends: ${misc:Depends}, + ${python3:Depends}, + ${shlibs:Depends}, + ${sphinxdoc:Depends}, + fuse [linux-any] | fuse4bsd [kfreebsd-any], + psmisc, + python3-pkg-resources +Recommends: python3-systemd +Description: Full-featured file system for online data storage + S3QL is a file system that stores all its data online using storage + services like Google Storage, Amazon S3 or OpenStack. S3QL + effectively provides a hard disk of dynamic, infinite capacity that + can be accessed from any computer with internet access. + . + S3QL is a standard conforming, full featured UNIX file system that is + conceptually indistinguishable from any local file + system. Furthermore, S3QL has additional features like compression, + encryption, data de-duplication, immutable trees and snapshotting + which make it especially suitable for online backup and archival. + . + In addition to accessing online storage services directoly, S3QL can + also store its data underneath a regular mount point. This enables + the use of S3QL with e.g. NFS, CIFS or sftp servers. + . + S3QL is designed to favor simplicity and elegance over performance + and feature-creep. Care has been taken to make the source code as + readable and serviceable as possible. Solid error detection and error + handling have been included from the very first line, and S3QL comes + with extensive automated test cases. + +Package: s3ql-dbg +Architecture: any +Section: debug +Priority: extra +Recommends: python3-dbg, + python3-llfuse-dbg, + python3-apsw-dbg, + python3-crypto-dbg, +Depends: s3ql (= ${binary:Version}), + ${python3:Depends}, + ${shlibs:Depends}, + ${misc:Depends} +Replaces: s3ql (<< 2.14+dfsg-1) +Breaks: s3ql (<< 2.14+dfsg-1) +Description: Full-featured file system for online data storage (debugging symbols) + S3QL is a file system that stores all its data online using storage + services like Google Storage, Amazon S3 or OpenStack. S3QL + effectively provides a hard disk of dynamic, infinite capacity that + can be accessed from any computer with internet access. + . + S3QL is a standard conforming, full featured UNIX file system that is + conceptually indistinguishable from any local file + system. Furthermore, S3QL has additional features like compression, + encryption, data de-duplication, immutable trees and snapshotting + which make it especially suitable for online backup and archival. + . + In addition to accessing online storage services directoly, S3QL can + also store its data underneath a regular mount point. This enables + the use of S3QL with e.g. NFS, CIFS or sftp servers. + . + S3QL is designed to favor simplicity and elegance over performance + and feature-creep. Care has been taken to make the source code as + readable and serviceable as possible. Solid error detection and error + handling have been included from the very first line, and S3QL comes + with extensive automated test cases. + . + This package contains the debugging symbols as well as the extension + built for the Python 3 debug interpreter. diff --git a/copyright b/copyright new file mode 100644 index 0000000..8e1c4f7 --- /dev/null +++ b/copyright @@ -0,0 +1,24 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: s3ql +Upstream-Contact: s3ql@googlegroups.com +Source: https://bitbucket.org/nikratio/s3ql/ +Files-Excluded: doc/html + +Files: * +Copyright: Copyright (c) 1998 - 2012 Nikolaus Rath +License: GPL-3 + +License: GPL-3 + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License Version 3 as + published by the Free Software Foundation. + . + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + On Debian systems the full text of the GNU General Public License + Version 3 can be found in the `/usr/share/common-licenses/GPL-3' + file. + diff --git a/patches/clock-granularity.diff b/patches/clock-granularity.diff new file mode 100644 index 0000000..64a1d53 --- /dev/null +++ b/patches/clock-granularity.diff @@ -0,0 +1,32 @@ +From 324ac9933b9b4d2f477942c8b1f1c54241933a76 Mon Sep 17 00:00:00 2001 +From: Nikolaus Rath +Date: Tue, 23 Feb 2016 15:44:51 -0800 +Subject: Estimate system clock granularity more conservatively + +Origin: debian +Forwarded: no +Patch-Name: clock-granularity.diff + +Some unit tests check that file access time stamps are updated +correctly. For such a test to succeed, it's necessarily that the time +between two subsequent test-accesses can actually be resolved. The +heuristic to determine this granularity doesn't seem to work well on +e.g. the i386 buildd, so we make it much more conservative. This means +the test suite runs longer, but otherwise has no ill effects. +--- + tests/common.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/tests/common.py b/tests/common.py +index 93736..af5ed 100644 +--- a/tests/common.py ++++ b/tests/common.py +@@ -31,7 +31,7 @@ def get_clock_granularity(): + stamp2 = time.time() + resolution = min(resolution, stamp2 - stamp1) + time.sleep(0.01) +- return resolution ++ return max(1, 10 * resolution) + CLOCK_GRANULARITY = get_clock_granularity() + + # When testing, we want to make sure that we don't sleep for too short a time diff --git a/patches/ignore_cython_warnings.diff b/patches/ignore_cython_warnings.diff new file mode 100644 index 0000000..48188f7 --- /dev/null +++ b/patches/ignore_cython_warnings.diff @@ -0,0 +1,34 @@ +From c1a178861185e4f94feb36e7a4f7ff6da644b66f Mon Sep 17 00:00:00 2001 +From: Nikolaus Rath +Date: Tue, 23 Feb 2016 15:44:51 -0800 +Subject: Ignore compiler warnings due to old Cython version + +Origin: debian +Forwarded: not-needed +Patch-Name: ignore_cython_warnings.diff + +Cython versions prior to 0.24 generate Gcc compiler warnings. +Since Cython 0.24 isn't in Debian yet, we ignore these warnings. +--- + setup.py | 8 -------- + 1 file changed, 8 deletions(-) + +diff --git a/setup.py b/setup.py +index ae1c0..e2c9a 100755 +--- a/setup.py ++++ b/setup.py +@@ -111,14 +111,6 @@ def main(): + + compile_args = ['-Wall', '-Wextra', '-Wconversion', '-Wsign-compare'] + +- # Value-changing conversions should always be explicit. +- compile_args.append('-Werror=conversion') +- +- # Note that (i > -1) is false if i is unsigned (-1 will be converted to +- # a large positive value). We certainly don't want to do this by +- # accident. +- compile_args.append('-Werror=sign-compare') +- + # Enable all fatal warnings only when compiling from Mercurial tip. + # (otherwise we break forward compatibility because compilation with newer + # compiler may fail if additional warnings are added) diff --git a/patches/proc_mount.diff b/patches/proc_mount.diff new file mode 100644 index 0000000..5299ec8 --- /dev/null +++ b/patches/proc_mount.diff @@ -0,0 +1,32 @@ +From c9ba5b59e80b5468e17c900a93cfa549b234a9ff Mon Sep 17 00:00:00 2001 +From: Nikolaus Rath +Date: Tue, 23 Feb 2016 15:44:51 -0800 +Subject: Skip tests requiring /proc + +Origin: debian +Forwarded: no +Patch-Name: proc_mount.diff +--- + tests/t4_adm.py | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/tests/t4_adm.py b/tests/t4_adm.py +index a1726..36e7f 100755 +--- a/tests/t4_adm.py ++++ b/tests/t4_adm.py +@@ -19,6 +19,7 @@ import tempfile + import unittest + import subprocess + import pytest ++import os + + @pytest.mark.usefixtures('s3ql_cmd_argv', 'pass_reg_output') + class AdmTests(unittest.TestCase): +@@ -49,6 +50,7 @@ class AdmTests(unittest.TestCase): + self.reg_output(r'^WARNING: Maximum object sizes less than ' + '1 MiB will degrade performance\.$', count=1) + ++ @unittest.skipUnless(os.path.exists('/proc/mounts'), '/proc/mounts not available') + def test_passphrase(self): + self.mkfs() + diff --git a/patches/series b/patches/series new file mode 100644 index 0000000..38dc7f4 --- /dev/null +++ b/patches/series @@ -0,0 +1,5 @@ +proc_mount.diff +clock-granularity.diff +ignore_cython_warnings.diff +support_jessie_upgrade.diff +show_pdflatex_output.diff diff --git a/patches/show_pdflatex_output.diff b/patches/show_pdflatex_output.diff new file mode 100644 index 0000000..cbb9601 --- /dev/null +++ b/patches/show_pdflatex_output.diff @@ -0,0 +1,30 @@ +From f2e46743e3b446d31cdda56846e7d2e7c384e2a0 Mon Sep 17 00:00:00 2001 +From: Nikolaus Rath +Date: Mon, 27 Jun 2016 11:25:34 -0700 +Subject: Don't hide pdflatex output. + +Origin: debian +Forwarded: not-needed +Patch-Name: show_pdflatex_output.diff + +Upstream is not interested in this patch. +--- + setup.py | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/setup.py b/setup.py +index e2c9a..375c7 100755 +--- a/setup.py ++++ b/setup.py +@@ -97,9 +97,8 @@ class build_docs(setuptools.Command): + + print('Running pdflatex...') + for _ in range(3): +- with open('/dev/null', 'wb') as null: +- subprocess.check_call(['pdflatex', '-interaction', 'batchmode', 'manual.tex'], +- cwd=os.path.join(dest_dir, 'latex'), stdout=null) ++ subprocess.check_call(['pdflatex', '-interaction', 'nonstopmode', 'manual.tex'], ++ cwd=os.path.join(dest_dir, 'latex')) + os.rename(os.path.join(dest_dir, 'latex', 'manual.pdf'), + os.path.join(dest_dir, 'manual.pdf')) + diff --git a/patches/support_jessie_upgrade.diff b/patches/support_jessie_upgrade.diff new file mode 100644 index 0000000..4929879 --- /dev/null +++ b/patches/support_jessie_upgrade.diff @@ -0,0 +1,876 @@ +From fb58feac206d5a1987b8c7f74e5ee89b18f87dc6 Mon Sep 17 00:00:00 2001 +From: Nikolaus Rath +Date: Tue, 23 Feb 2016 15:44:51 -0800 +Subject: Allow upgrade of file systems created with jessie's s3ql + +Origin: debian +Forwarded: not-needed +Patch-Name: support_jessie_upgrade.diff + +Upstream has dropped support for upgrading file systems created with +the S3QL version in jessie. This patch forward-ports this capability. +It is based on upstream's Mercurial commit 773931c43368. +--- + src/s3ql/adm.py | 205 +++++++++++++++++++++-- + src/s3ql/backends/comprenc.py | 371 ++++++++++++++++++++++++++++++++++++++---- + src/s3ql/backends/local.py | 12 +- + src/s3ql/backends/s3c.py | 29 +++- + src/s3ql/upgrade_support.py | 75 +++++++++ + 5 files changed, 647 insertions(+), 45 deletions(-) + create mode 100644 src/s3ql/upgrade_support.py + +diff --git a/src/s3ql/adm.py b/src/s3ql/adm.py +index d0e67..3b5d9 100644 +--- a/src/s3ql/adm.py ++++ b/src/s3ql/adm.py +@@ -7,19 +7,23 @@ This work can be distributed under the terms of the GNU GPLv3. + ''' + + from .logging import logging, QuietError, setup_logging +-from . import CURRENT_FS_REV, REV_VER_MAP ++from . import CURRENT_FS_REV, REV_VER_MAP, BUFSIZE + from .backends.comprenc import ComprencBackend + from .database import Connection + from .deltadump import TIME, INTEGER + from .common import (get_backend_cachedir, get_seq_no, is_mounted, get_backend, +- load_params, save_params) ++ load_params, save_params, AsyncFn, get_backend_factory, ++ pretty_print_size, split_by_n, handle_on_return) + from .metadata import dump_and_upload_metadata, download_metadata + from . import metadata + from .parse_args import ArgumentParser + from datetime import datetime as Datetime + from getpass import getpass + from contextlib import contextmanager ++from base64 import b64encode ++from queue import Queue, Full as QueueFull + import os ++import tempfile + import shutil + import functools + import sys +@@ -86,12 +90,11 @@ def main(args=None): + if options.action == 'clear': + with get_backend(options, raw=True) as backend: + return clear(backend, options) ++ elif options.action == 'upgrade': ++ return upgrade(options) + + with get_backend(options) as backend: +- if options.action == 'upgrade': +- return upgrade(backend, get_backend_cachedir(options.storage_url, +- options.cachedir)) +- elif options.action == 'passphrase': ++ if options.action == 'passphrase': + return change_passphrase(backend) + + elif options.action == 'download-metadata': +@@ -206,11 +209,24 @@ def get_old_rev_msg(rev, prog): + ''' % { 'version': REV_VER_MAP[rev], + 'prog': prog }) + +-def upgrade(backend, cachepath): ++ ++@handle_on_return ++def upgrade(options, on_return): + '''Upgrade file system to newest revision''' + + log.info('Getting file system parameters..') + ++ from . import backends ++ backends.local.SUPPORT_LEGACY_FORMAT = True ++ backends.s3c.SUPPORT_LEGACY_FORMAT = True ++ backends.comprenc.SUPPORT_LEGACY_FORMAT = True ++ ++ cachepath = get_backend_cachedir(options.storage_url, options.cachedir) ++ ++ backend_factory = get_backend_factory(options.storage_url, options.backend_options, ++ options.authfile) ++ backend = on_return.enter_context(backend_factory()) ++ + # Check for cached metadata + db = None + seq_no = get_seq_no(backend) +@@ -249,7 +265,7 @@ def upgrade(backend, cachepath): + raise QuietError() + + # Check revision +- if param['revision'] < CURRENT_FS_REV-1: ++ if param['revision'] < 21: + print(textwrap.dedent(''' + File system revision too old to upgrade! + +@@ -285,13 +301,31 @@ def upgrade(backend, cachepath): + with monkeypatch_metadata_retrieval(): + db = download_metadata(backend, cachepath + '.db') + +- log.info('Upgrading from revision %d to %d...', param['revision'], CURRENT_FS_REV) +- +- param['revision'] = CURRENT_FS_REV + param['last-modified'] = time.time() + param['seq_no'] += 1 ++ ++ if param['revision'] == 21: ++ log.info('Upgrading from revision %d to %d...', 21, 22) ++ param['revision'] = 22 ++ ++ # Ensure that there are backups of the master key ++ if backend.passphrase is not None: ++ data_pw = backend.passphrase ++ backend.passphrase = backend.fs_passphrase ++ for i in range(1,4): ++ obj_id = 's3ql_passphrase_bak%d' % i ++ if obj_id not in backend: ++ backend[obj_id] = data_pw ++ backend.passphrase = data_pw ++ ++ # Upgrade to revision 22 ++ update_obj_metadata(backend, backend_factory, db, options.threads) ++ updated_from_21 = True ++ else: ++ updated_from_21 = False + +- # Upgrade ++ log.info('Upgrading from revision %d to %d...', 22, 23) ++ param['revision'] = 23 + for name in ('atime', 'mtime', 'ctime'): + db.execute('ALTER TABLE inodes ADD COLUMN {time}_ns ' + 'INT NOT NULL DEFAULT 0'.format(time=name)) +@@ -308,6 +342,14 @@ def upgrade(backend, cachepath): + + print('File system upgrade complete.') + ++ if updated_from_21 and backend.passphrase is not None: ++ print('\nPlease store the following master key in a safe location. It allows ', ++ 'decryption of the S3QL file system in case the storage objects holding ', ++ 'this information get corrupted:', ++ '---BEGIN MASTER KEY---', ++ ' '.join(split_by_n(b64encode(backend.passphrase).decode(), 4)), ++ '---END MASTER KEY---', ++ sep='\n') + + @contextmanager + def monkeypatch_metadata_retrieval(): +@@ -351,5 +393,144 @@ def monkeypatch_metadata_retrieval(): + metadata.DUMP_SPEC[2] = DUMP_SPEC_bak + metadata.create_tables = create_tables_bak + ++def update_obj_metadata(backend, backend_factory, db, thread_count): ++ '''Upgrade metadata of storage objects''' ++ ++ plain_backend = backend.backend ++ ++ # No need to update sequence number, since we are going to ++ # write out a new one after the upgrade. ++ if backend.passphrase is None: ++ extra_objects = { 's3ql_metadata' } ++ else: ++ extra_objects = { 's3ql_metadata', ++ 's3ql_passphrase', 's3ql_passphrase_bak1', ++ 's3ql_passphrase_bak2', 's3ql_passphrase_bak3' } ++ ++ for i in range(30): ++ obj_id = 's3ql_metadata_bak_%d' % i ++ if obj_id in plain_backend: ++ extra_objects.add(obj_id) ++ ++ def yield_objects(): ++ for (id_,) in db.query('SELECT id FROM objects'): ++ yield 's3ql_data_%d' % id_ ++ for obj_id in extra_objects: ++ yield obj_id ++ total = db.get_val('SELECT COUNT(id) FROM objects') + len(extra_objects) ++ ++ queue = Queue(maxsize=thread_count) ++ threads = [] ++ for _ in range(thread_count): ++ t = AsyncFn(upgrade_loop, queue, backend_factory) ++ # Don't wait for worker threads, gives deadlock if main thread ++ # terminates with exception ++ t.daemon = True ++ t.start() ++ threads.append(t) ++ ++ # Updating this value is prone to race conditions. However, ++ # we don't care because this is for an approximate progress ++ # output only. ++ queue.rewrote_size = 0 ++ stamp = 0 ++ for (i, obj_id) in enumerate(yield_objects()): ++ stamp2 = time.time() ++ if stamp2 - stamp > 1: ++ sys.stdout.write('\r..processed %d/%d objects (%.1f%%, %s rewritten)..' ++ % (i, total, i/total*100, ++ pretty_print_size(queue.rewrote_size))) ++ sys.stdout.flush() ++ stamp = stamp2 ++ ++ # Terminate early if any thread failed with an exception ++ for t in threads: ++ if not t.is_alive(): ++ t.join_and_raise() ++ ++ # Avoid blocking if all threads terminated ++ while True: ++ try: ++ queue.put(obj_id, timeout=1) ++ except QueueFull: ++ pass ++ else: ++ break ++ for t in threads: ++ if not t.is_alive(): ++ t.join_and_raise() ++ ++ queue.maxsize += len(threads) ++ for t in threads: ++ queue.put(None) ++ ++ for t in threads: ++ t.join_and_raise() ++ ++ sys.stdout.write('\n') ++ ++def upgrade_loop(queue, backend_factory): ++ ++ with backend_factory() as backend: ++ plain_backend = backend.backend ++ while True: ++ obj_id = queue.get() ++ if obj_id is None: ++ break ++ ++ meta = plain_backend.lookup(obj_id) ++ if meta.get('format_version', 0) == 2: ++ continue ++ ++ # For important objects, we make a copy first (just to be safe) ++ if not obj_id.startswith('s3ql_data'): ++ plain_backend.copy(obj_id, 's3ql_pre2.13' + obj_id[4:]) ++ ++ # When reading passphrase objects, we have to use the ++ # "outer" password ++ if obj_id.startswith('s3ql_passphrase'): ++ data_pw = backend.passphrase ++ backend.passphrase = backend.fs_passphrase ++ ++ meta = backend._convert_legacy_metadata(meta) ++ if meta['encryption'] == 'AES': ++ # Two statements to reduce likelihood of update races ++ size = rewrite_legacy_object(backend, obj_id) ++ queue.rewrote_size += size ++ else: ++ plain_backend.update_meta(obj_id, meta) ++ ++ if obj_id.startswith('s3ql_passphrase'): ++ backend.passphrase = data_pw ++ ++def rewrite_legacy_object(backend, obj_id): ++ with tempfile.TemporaryFile() as tmpfh: ++ ++ # Read object ++ def do_read(fh): ++ tmpfh.seek(0) ++ tmpfh.truncate() ++ while True: ++ buf = fh.read(BUFSIZE) ++ if not buf: ++ break ++ tmpfh.write(buf) ++ return fh.metadata ++ ++ meta = backend.perform_read(do_read, obj_id) ++ ++ # Write object ++ def do_write(fh): ++ tmpfh.seek(0) ++ while True: ++ buf = tmpfh.read(BUFSIZE) ++ if not buf: ++ break ++ fh.write(buf) ++ return fh ++ out_fh = backend.perform_write(do_write, obj_id, meta) ++ ++ return out_fh.get_obj_size() ++ + if __name__ == '__main__': + main(sys.argv[1:]) +diff --git a/src/s3ql/backends/comprenc.py b/src/s3ql/backends/comprenc.py +index 58e90..285f5 100644 +--- a/src/s3ql/backends/comprenc.py ++++ b/src/s3ql/backends/comprenc.py +@@ -25,6 +25,11 @@ import zlib + + log = logging.getLogger(__name__) + ++from ..upgrade_support import safe_unpickle, pickle ++from base64 import b64decode, b64encode ++import binascii ++SUPPORT_LEGACY_FORMAT=False ++ + HMAC_SIZE = 32 + + def sha256(s): +@@ -67,6 +72,8 @@ class ComprencBackend(AbstractBackend, metaclass=ABCDocstMeta): + @copy_ancestor_docstring + def lookup(self, key): + meta_raw = self.backend.lookup(key) ++ if SUPPORT_LEGACY_FORMAT and meta_raw.get('format_version', 0) < 2: ++ meta_raw = self._convert_legacy_metadata(meta_raw) + return self._verify_meta(key, meta_raw)[1] + + @prepend_ancestor_docstring +@@ -147,43 +154,80 @@ class ComprencBackend(AbstractBackend, metaclass=ABCDocstMeta): + """ + + fh = self.backend.open_read(key) ++ checksum_warning = False + try: +- meta_raw = fh.metadata +- (nonce, meta) = self._verify_meta(key, meta_raw) +- if nonce: +- data_key = sha256(self.passphrase + nonce) +- +- # The `payload_offset` key only exists if the storage object was +- # created with on old S3QL version. In order to avoid having to +- # download and re-upload the entire object during the upgrade, the +- # upgrade procedure adds this header to tell us how many bytes at +- # the beginning of the object we have to skip to get to the payload. +- if 'payload_offset' in meta_raw: +- to_skip = meta_raw['payload_offset'] +- while to_skip: +- to_skip -= len(fh.read(to_skip)) +- +- encr_alg = meta_raw['encryption'] +- if encr_alg == 'AES_v2': +- fh = DecryptFilter(fh, data_key) +- elif encr_alg != 'None': +- raise RuntimeError('Unsupported encryption: %s' % encr_alg) +- +- compr_alg = meta_raw['compression'] +- if compr_alg == 'BZIP2': +- fh = DecompressFilter(fh, bz2.BZ2Decompressor()) +- elif compr_alg == 'LZMA': +- fh = DecompressFilter(fh, lzma.LZMADecompressor()) +- elif compr_alg == 'ZLIB': +- fh = DecompressFilter(fh,zlib.decompressobj()) +- elif compr_alg != 'None': +- raise RuntimeError('Unsupported compression: %s' % compr_alg) ++ if SUPPORT_LEGACY_FORMAT: ++ if fh.metadata.get('format_version', 0) < 2: ++ meta_raw = self._convert_legacy_metadata(fh.metadata) ++ else: ++ meta_raw = fh.metadata ++ (nonce, meta) = self._verify_meta(key, meta_raw) ++ if nonce: ++ data_key = sha256(self.passphrase + nonce) ++ compr_alg = meta_raw['compression'] ++ encr_alg = meta_raw['encryption'] ++ if compr_alg == 'BZIP2': ++ decompressor = bz2.BZ2Decompressor() ++ elif compr_alg == 'LZMA': ++ decompressor = lzma.LZMADecompressor() ++ elif compr_alg == 'ZLIB': ++ decompressor = zlib.decompressobj() ++ elif compr_alg == 'None': ++ decompressor = None ++ else: ++ raise RuntimeError('Unsupported compression: %s' % compr_alg) ++ if 'payload_offset' in meta_raw: ++ to_skip = meta_raw['payload_offset'] ++ while to_skip: ++ to_skip -= len(fh.read(to_skip)) ++ checksum_warning = True ++ if encr_alg == 'AES': ++ fh = LegacyDecryptDecompressFilter(fh, data_key, decompressor) ++ decompressor = None ++ elif encr_alg == 'AES_v2': ++ fh = DecryptFilter(fh, data_key) ++ elif encr_alg != 'None': ++ raise RuntimeError('Unsupported encryption: %s' % encr_alg) ++ if decompressor: ++ fh = DecompressFilter(fh, decompressor) ++ else: ++ meta_raw = fh.metadata ++ ++ (nonce, meta) = self._verify_meta(key, meta_raw) ++ if nonce: ++ data_key = sha256(self.passphrase + nonce) ++ ++ # The `payload_offset` key only exists if the storage object was ++ # created with on old S3QL version. In order to avoid having to ++ # download and re-upload the entire object during the upgrade, the ++ # upgrade procedure adds this header to tell us how many bytes at ++ # the beginning of the object we have to skip to get to the payload. ++ if 'payload_offset' in meta_raw: ++ to_skip = meta_raw['payload_offset'] ++ while to_skip: ++ to_skip -= len(fh.read(to_skip)) ++ ++ encr_alg = meta_raw['encryption'] ++ if encr_alg == 'AES_v2': ++ fh = DecryptFilter(fh, data_key) ++ elif encr_alg != 'None': ++ raise RuntimeError('Unsupported encryption: %s' % encr_alg) ++ ++ compr_alg = meta_raw['compression'] ++ if compr_alg == 'BZIP2': ++ fh = DecompressFilter(fh, bz2.BZ2Decompressor()) ++ elif compr_alg == 'LZMA': ++ fh = DecompressFilter(fh, lzma.LZMADecompressor()) ++ elif compr_alg == 'ZLIB': ++ fh = DecompressFilter(fh,zlib.decompressobj()) ++ elif compr_alg != 'None': ++ raise RuntimeError('Unsupported compression: %s' % compr_alg) + + fh.metadata = meta + except: + # Don't emit checksum warning, caller hasn't even + # started reading anything. +- fh.close(checksum_warning=False) ++ fh.close(checksum_warning=checksum_warning) + raise + + return fh +@@ -275,6 +319,8 @@ class ComprencBackend(AbstractBackend, metaclass=ABCDocstMeta): + + def _copy_or_rename(self, src, dest, rename, metadata=None): + meta_raw = self.backend.lookup(src) ++ if SUPPORT_LEGACY_FORMAT and meta_raw.get('format_version', 0) < 2: ++ meta_raw = self._convert_legacy_metadata(meta_raw) + (nonce, meta_old) = self._verify_meta(src, meta_raw) + + if nonce: +@@ -303,6 +349,165 @@ class ComprencBackend(AbstractBackend, metaclass=ABCDocstMeta): + def close(self): + self.backend.close() + ++ def _convert_legacy_metadata(self, meta): ++ '''Convert metadata to newest format ++ ++ This method ensures that we can read objects written ++ by older S3QL versions. ++ ''' ++ ++ format_version = meta.get('format_version', 0) ++ assert format_version in (0,1) ++ if format_version == 0: ++ meta = self._convert_legacy_metadata0(meta) ++ return self._convert_legacy_metadata1(meta) ++ ++ def _convert_legacy_metadata0(self, meta, ++ LEN_BYTES = struct.calcsize(b' 2 or opcode.name not in SAFE_UNPICKLE_OPCODES: ++ raise pickle.UnpicklingError('opcode %s is unsafe' % opcode.name) ++ except (ValueError, EOFError): ++ raise pickle.UnpicklingError('corrupted data') ++ ++ fh.seek(pos) ++ ++ # Then use a custom Unpickler to ensure that we only give access to ++ # specific, whitelisted globals. Note that with the above opcodes, there is ++ # no way to trigger attribute access, so "brachiating" from a white listed ++ # object to __builtins__ is not possible. ++ return SafeUnpickler(fh, fix_imports=fix_imports, ++ encoding=encoding, errors=errors).load() ++ ++def safe_unpickle(buf, fix_imports=True, encoding="ASCII", ++ errors="strict"): ++ '''Safely unpickle untrusted data in *buf*''' ++ ++ return safe_unpickle_fh(io.BytesIO(buf), fix_imports=fix_imports, ++ encoding=encoding, errors=errors) diff --git a/py3dist-overrides b/py3dist-overrides new file mode 100644 index 0000000..e386e0b --- /dev/null +++ b/py3dist-overrides @@ -0,0 +1,3 @@ +dugong python3-dugong; PEP386 +apsw python3-apsw; PEP386 +llfuse python3-llfuse; PEP386 diff --git a/rules b/rules new file mode 100755 index 0000000..00354e3 --- /dev/null +++ b/rules @@ -0,0 +1,112 @@ +#!/usr/bin/make -f +# -*- makefile -*- + +#export DH_VERBOSE=1 +DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH) + +%: + dh $@ --with python3,sphinxdoc + +override_dh_auto_build: build_cython build_sphinx build_python + +build_cython: + dh_testdir + python3 setup.py build_cython + touch $@ + +build_sphinx: build_inplace + dh_testdir + python3 setup.py build_sphinx + touch $@ + +build_inplace: build_cython + python3 setup.py build_ext --inplace + touch $@ + +build_python: build_cython + dh_testdir + python3-dbg setup.py build -g + python3 setup.py build -g + touch $@ + +override_dh_auto_install: + dh_testdir + dh_testroot + dh_prep + dh_installdirs + + # Note: Install non -dbg last, so that scripts don't get a -dbg interpreter + python3-dbg setup.py install --force --root=debian/tmp \ + --install-lib=/usr/lib/s3ql \ + --install-scripts=/usr/lib/s3ql \ + --no-compile -O0 --install-layout=deb + + python3 setup.py install --force --root=debian/tmp \ + --install-lib=/usr/lib/s3ql \ + --install-scripts=/usr/lib/s3ql \ + --no-compile -O0 --install-layout=deb + +override_dh_auto_test: +ifeq (,$(findstring nocheck,$(DEB_BUILD_OPTIONS))) + python3 setup.py build_ext --inplace + # Some tests will try to create log files in $HOME + mkdir -p debian/tmphome + HOME=$(CURDIR)/debian/tmphome py.test-3 --logdebug=s3ql.verify tests/ +endif + +override_dh_install: + dh_install + + # Rename to avoid conflict with pcp package + (cd debian/s3ql/usr/share/man/man1/; mv pcp.1 parallel-cp.1) + + # Install debugging extension and remove from regular package + (cd debian/s3ql; \ + find . -regextype posix-egrep -regex ".+\\.[a-z]+-[0-9]{2,}[a-z]*d[a-z]*-$(DEB_HOST_MULTIARCH)\\.so" \ + -printf "%P\0" -delete) \ + | xargs -0 dh_install --autodest -ps3ql-dbg + + # Link all executable files not ending in .py into /usr/bin + mkdir -p debian/s3ql/usr/bin/ + for entry in debian/s3ql/usr/lib/s3ql/*; do \ + if [ -d "$$entry" ] || [ ! -x "$$entry" ] \ + || [ "$${entry%.py}" != "$$entry" ]; then \ + continue; \ + fi; \ + ln -s ../lib/s3ql/`basename "$$entry"` \ + "debian/s3ql/usr/bin/`basename \"$$entry\"`"; \ + done + +override_dh_link: + rm -rf debian/s3ql-dbg/usr/share/doc/s3ql-dbg + dh_link + + +# We don't want to call setup.py clean, because this +# requires a number of (potentially uninstalled) build-dependencies. +override_dh_auto_clean: + rm -rf build doc tests/test.log src/s3ql/deltadump.c contrib/*.1 + find \( \( -name '*.egg-info' -type d \) \ + -o \( -name __pycache__ -type d \) \ + -o \( -name '*.so' -type f \) \ + \) -prune -exec rm -rf '{}' + + rm -rf build-python build_cython build_sphinx build-stamp + rm -rf debian/tmphome + +override_dh_strip: +ifeq (,$(filter nostrip,$(DEB_BUILD_OPTIONS))) + dh_strip -ps3ql --dbg-package=s3ql-dbg +endif + +.PHONY: get-orig-source +get-orig-source: + # Work around bug #749225 + PERL_LWP_SSL_VERIFY_HOSTNAME=0 \ + uscan --rename --destdir=$(CURDIR)/.. --repack --force-download \ + --download-current-version --compression xz + +.PHONY: uscan +uscan: + # Work around bug #749225 + PERL_LWP_SSL_VERIFY_HOSTNAME=0 \ + uscan --rename --destdir=$(CURDIR)/.. --repack --compression xz diff --git a/s3ql-dbg.links b/s3ql-dbg.links new file mode 100644 index 0000000..6c91288 --- /dev/null +++ b/s3ql-dbg.links @@ -0,0 +1 @@ +/usr/share/doc/s3ql /usr/share/doc/s3ql-dbg diff --git a/s3ql.doc-base b/s3ql.doc-base new file mode 100644 index 0000000..fe8fa4f --- /dev/null +++ b/s3ql.doc-base @@ -0,0 +1,11 @@ +Document: s3ql +Title: S3QL User's Guide +Author: Nikolaus Rath +Section: File Management + +Format: HTML +Index: /usr/share/doc/s3ql/html/index.html +Files: /usr/share/doc/s3ql/html/*.html + +Format: PDF +Files: /usr/share/doc/s3ql/manual.pdf.gz diff --git a/s3ql.docs b/s3ql.docs new file mode 100644 index 0000000..5214ec8 --- /dev/null +++ b/s3ql.docs @@ -0,0 +1,2 @@ +doc/html/ +doc/manual.pdf diff --git a/s3ql.examples b/s3ql.examples new file mode 100644 index 0000000..59dbaa2 --- /dev/null +++ b/s3ql.examples @@ -0,0 +1 @@ +contrib/s3ql_backup.sh diff --git a/s3ql.install b/s3ql.install new file mode 100644 index 0000000..9ef8edd --- /dev/null +++ b/s3ql.install @@ -0,0 +1,4 @@ +usr/ +contrib/*.py /usr/lib/s3ql/ +contrib/*.1 /usr/share/man/man1/ + diff --git a/s3ql.links b/s3ql.links new file mode 100644 index 0000000..fdf4300 --- /dev/null +++ b/s3ql.links @@ -0,0 +1,3 @@ +/usr/lib/s3ql/pcp.py /usr/bin/parallel-cp +/usr/lib/s3ql/expire_backups.py /usr/bin/expire_backups +/usr/lib/s3ql/remove_objects.py /usr/bin/s3ql_remove_objects diff --git a/source/format b/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/source/include-binaries b/source/include-binaries new file mode 100644 index 0000000..fb217db --- /dev/null +++ b/source/include-binaries @@ -0,0 +1,2 @@ +debian/python.inv +debian/upstream-signing-key.pgp diff --git a/source/options b/source/options new file mode 100644 index 0000000..b306286 --- /dev/null +++ b/source/options @@ -0,0 +1,2 @@ +compression = "xz" +compression-level = 6 diff --git a/tests/control b/tests/control new file mode 100644 index 0000000..b4774c2 --- /dev/null +++ b/tests/control @@ -0,0 +1,10 @@ +Tests: upstream-standard +Depends: python3-pytest, python3-pytest-catchlog, rsync, s3ql, fuse, psmisc + +# This is really the same test suite, but when run with root +# access (or permission to mount fuse file systems), additional +# tests will be run. +Tests: upstream-with-fuse +Depends: python3-pytest, python3-pytest-catchlog, rsync, s3ql, fuse, psmisc +Restrictions: needs-root + diff --git a/tests/upstream-standard b/tests/upstream-standard new file mode 100755 index 0000000..19ceda8 --- /dev/null +++ b/tests/upstream-standard @@ -0,0 +1,7 @@ +#!/bin/sh + +# Test suite automatically skips over tests that cannot be +# executed because of insufficient priviliges. + +export PYTHONPATH="/usr/lib/s3ql:${PYTHONPATH}" +exec py.test-3 --installed --logdebug=s3ql.verify tests/ diff --git a/tests/upstream-with-fuse b/tests/upstream-with-fuse new file mode 100755 index 0000000..21dbbc4 --- /dev/null +++ b/tests/upstream-with-fuse @@ -0,0 +1,4 @@ +#!/bin/sh + +export PYTHONPATH="/usr/lib/s3ql:${PYTHONPATH}" +exec py.test-3 --installed --logdebug=s3ql.verify tests/ diff --git a/upstream-signing-key.pgp b/upstream-signing-key.pgp new file mode 100644 index 0000000..d2ea347 Binary files /dev/null and b/upstream-signing-key.pgp differ diff --git a/watch b/watch new file mode 100644 index 0000000..d7817fa --- /dev/null +++ b/watch @@ -0,0 +1,9 @@ +# watch control file for uscan +version=3 +opts=\ +pgpsigurlmangle=s/$/.asc/,\ +uversionmangle=s/$/+dfsg/ \ + https://bitbucket.org/nikratio/s3ql/downloads \ + .*/s3ql-(\d\S*)\.tar\.bz2 \ + debian + -- cgit v1.2.3 From 015a11c4be7d9699158e01f398b76abac35e0478 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Fri, 28 Oct 2016 14:07:23 -0700 Subject: Import s3ql_2.21+dfsg.orig.tar.xz [dgit import orig s3ql_2.21+dfsg.orig.tar.xz] --- Changes.txt | 1405 ++++ LICENSE | 687 ++ PKG-INFO | 175 + README.rst | 153 + bin/fsck.s3ql | 26 + bin/mkfs.s3ql | 26 + bin/mount.s3ql | 26 + bin/s3ql_oauth_client | 26 + bin/s3ql_verify | 26 + bin/s3qladm | 26 + bin/s3qlcp | 26 + bin/s3qlctrl | 26 + bin/s3qllock | 26 + bin/s3qlrm | 27 + bin/s3qlstat | 26 + bin/umount.s3ql | 26 + contrib/benchmark.py | 225 + contrib/clone_fs.py | 160 + contrib/expire_backups.1 | 184 + contrib/expire_backups.py | 288 + contrib/fsck_db.py | 64 + contrib/pcp.1 | 107 + contrib/pcp.py | 97 + contrib/remove_objects.py | 64 + contrib/s3ql_backup.sh | 60 + contrib/scramble_db.py | 107 + doc/latex/Makefile | 66 + doc/latex/fncychap.sty | 683 ++ doc/latex/manual.aux | 447 ++ doc/latex/manual.idx | 36 + doc/latex/manual.out | 80 + doc/latex/manual.tex | 4168 +++++++++++ doc/latex/manual.toc | 157 + doc/latex/python.ist | 11 + doc/latex/sphinx.sty | 524 ++ doc/latex/sphinxhowto.cls | 104 + doc/latex/sphinxmanual.cls | 148 + doc/latex/tabulary.sty | 452 ++ doc/man/fsck.s3ql.1 | 196 + doc/man/mkfs.s3ql.1 | 163 + doc/man/mount.s3ql.1 | 238 + doc/man/s3ql_oauth_client.1 | 111 + doc/man/s3ql_verify.1 | 184 + doc/man/s3qladm.1 | 186 + doc/man/s3qlcp.1 | 155 + doc/man/s3qlctrl.1 | 160 + doc/man/s3qllock.1 | 141 + doc/man/s3qlrm.1 | 113 + doc/man/s3qlstat.1 | 110 + doc/man/umount.s3ql.1 | 119 + doc/manual.pdf | Bin 0 -> 278252 bytes rst/_static/sphinxdoc.css | 340 + rst/_templates/layout.html | 72 + rst/about.rst | 12 + rst/adm.rst | 82 + rst/authinfo.rst | 68 + rst/backends.rst | 374 + rst/conf.py | 244 + rst/contrib.rst | 90 + rst/durability.rst | 239 + rst/fsck.rst | 86 + rst/impl_details.rst | 138 + rst/include/about.rst | 7 + rst/include/exitcodes.rst | 11 + rst/include/postman.rst | 10 + rst/index.rst | 26 + rst/installation.rst | 172 + rst/issues.rst | 100 + rst/man/adm.rst | 102 + rst/man/cp.rst | 100 + rst/man/ctrl.rst | 80 + rst/man/expire_backups.rst | 107 + rst/man/fsck.rst | 108 + rst/man/index.rst | 25 + rst/man/lock.rst | 89 + rst/man/mkfs.rst | 77 + rst/man/mount.rst | 109 + rst/man/oauth_client.rst | 54 + rst/man/pcp.rst | 51 + rst/man/rm.rst | 52 + rst/man/stat.rst | 46 + rst/man/umount.rst | 53 + rst/man/verify.rst | 93 + rst/mkfs.rst | 39 + rst/mount.rst | 181 + rst/resources.rst | 22 + rst/special.rst | 122 + rst/tips.rst | 99 + rst/umount.rst | 30 + setup.cfg | 8 + setup.py | 282 + src/s3ql.egg-info/PKG-INFO | 175 + src/s3ql.egg-info/SOURCES.txt | 251 + src/s3ql.egg-info/dependency_links.txt | 1 + src/s3ql.egg-info/entry_points.txt | 14 + src/s3ql.egg-info/requires.txt | 6 + src/s3ql.egg-info/top_level.txt | 2 + src/s3ql.egg-info/zip-safe | 1 + src/s3ql/__init__.py | 51 + src/s3ql/adm.py | 355 + src/s3ql/backends/__init__.py | 21 + src/s3ql/backends/common.py | 701 ++ src/s3ql/backends/comprenc.py | 677 ++ src/s3ql/backends/gs.py | 240 + src/s3ql/backends/local.py | 338 + src/s3ql/backends/pool.py | 76 + src/s3ql/backends/rackspace.py | 43 + src/s3ql/backends/s3.py | 163 + src/s3ql/backends/s3c.py | 1037 +++ src/s3ql/backends/swift.py | 527 ++ src/s3ql/backends/swiftks.py | 135 + src/s3ql/block_cache.py | 978 +++ src/s3ql/calc_mro.py | 120 + src/s3ql/common.py | 629 ++ src/s3ql/cp.py | 92 + src/s3ql/ctrl.py | 99 + src/s3ql/daemonize.py | 88 + src/s3ql/database.py | 210 + src/s3ql/deltadump.c | 11558 +++++++++++++++++++++++++++++++ src/s3ql/deltadump.pyx | 652 ++ src/s3ql/endian_indep.h | 37 + src/s3ql/exit_stack.py | 70 + src/s3ql/fs.py | 1212 ++++ src/s3ql/fsck.py | 1370 ++++ src/s3ql/inherit_docstrings.py | 120 + src/s3ql/inode_cache.py | 271 + src/s3ql/lock.py | 57 + src/s3ql/logging.py | 177 + src/s3ql/metadata.py | 324 + src/s3ql/mkfs.py | 184 + src/s3ql/mount.py | 757 ++ src/s3ql/multi_lock.py | 67 + src/s3ql/oauth_client.py | 106 + src/s3ql/parse_args.py | 257 + src/s3ql/remove.py | 59 + src/s3ql/statfs.py | 80 + src/s3ql/umount.py | 198 + src/s3ql/verify.py | 222 + tests/common.py | 309 + tests/conftest.py | 120 + tests/mock_server.py | 324 + tests/pytest.ini | 3 + tests/pytest_checklogs.py | 130 + tests/t1_backends.py | 1282 ++++ tests/t1_dump.py | 233 + tests/t1_retry.py | 78 + tests/t1_serialization.py | 66 + tests/t2_block_cache.py | 584 ++ tests/t3_fs_api.py | 1003 +++ tests/t3_fsck.py | 629 ++ tests/t3_inode_cache.py | 101 + tests/t3_verify.py | 107 + tests/t4_adm.py | 95 + tests/t4_fuse.py | 352 + tests/t5_cp.py | 61 + tests/t5_ctrl.py | 51 + tests/t5_failsafe.py | 152 + tests/t5_fsck.py | 75 + tests/t5_full.py | 109 + tests/t5_lock_rm.py | 63 + tests/t6_upgrade.py | 211 + util/cmdline_lexer.py | 31 + util/sphinx_pipeinclude.py | 54 + 163 files changed, 47872 insertions(+) create mode 100644 Changes.txt create mode 100644 LICENSE create mode 100644 PKG-INFO create mode 100644 README.rst create mode 100755 bin/fsck.s3ql create mode 100755 bin/mkfs.s3ql create mode 100755 bin/mount.s3ql create mode 100755 bin/s3ql_oauth_client create mode 100755 bin/s3ql_verify create mode 100755 bin/s3qladm create mode 100755 bin/s3qlcp create mode 100755 bin/s3qlctrl create mode 100755 bin/s3qllock create mode 100755 bin/s3qlrm create mode 100755 bin/s3qlstat create mode 100755 bin/umount.s3ql create mode 100755 contrib/benchmark.py create mode 100755 contrib/clone_fs.py create mode 100644 contrib/expire_backups.1 create mode 100755 contrib/expire_backups.py create mode 100755 contrib/fsck_db.py create mode 100644 contrib/pcp.1 create mode 100755 contrib/pcp.py create mode 100755 contrib/remove_objects.py create mode 100755 contrib/s3ql_backup.sh create mode 100755 contrib/scramble_db.py create mode 100644 doc/latex/Makefile create mode 100644 doc/latex/fncychap.sty create mode 100644 doc/latex/manual.aux create mode 100644 doc/latex/manual.idx create mode 100644 doc/latex/manual.out create mode 100644 doc/latex/manual.tex create mode 100644 doc/latex/manual.toc create mode 100644 doc/latex/python.ist create mode 100644 doc/latex/sphinx.sty create mode 100644 doc/latex/sphinxhowto.cls create mode 100644 doc/latex/sphinxmanual.cls create mode 100644 doc/latex/tabulary.sty create mode 100644 doc/man/fsck.s3ql.1 create mode 100644 doc/man/mkfs.s3ql.1 create mode 100644 doc/man/mount.s3ql.1 create mode 100644 doc/man/s3ql_oauth_client.1 create mode 100644 doc/man/s3ql_verify.1 create mode 100644 doc/man/s3qladm.1 create mode 100644 doc/man/s3qlcp.1 create mode 100644 doc/man/s3qlctrl.1 create mode 100644 doc/man/s3qllock.1 create mode 100644 doc/man/s3qlrm.1 create mode 100644 doc/man/s3qlstat.1 create mode 100644 doc/man/umount.s3ql.1 create mode 100644 doc/manual.pdf create mode 100644 rst/_static/sphinxdoc.css create mode 100644 rst/_templates/layout.html create mode 100644 rst/about.rst create mode 100644 rst/adm.rst create mode 100644 rst/authinfo.rst create mode 100644 rst/backends.rst create mode 100644 rst/conf.py create mode 100644 rst/contrib.rst create mode 100644 rst/durability.rst create mode 100644 rst/fsck.rst create mode 100644 rst/impl_details.rst create mode 100644 rst/include/about.rst create mode 100644 rst/include/exitcodes.rst create mode 100644 rst/include/postman.rst create mode 100644 rst/index.rst create mode 100644 rst/installation.rst create mode 100644 rst/issues.rst create mode 100644 rst/man/adm.rst create mode 100644 rst/man/cp.rst create mode 100644 rst/man/ctrl.rst create mode 100644 rst/man/expire_backups.rst create mode 100644 rst/man/fsck.rst create mode 100644 rst/man/index.rst create mode 100644 rst/man/lock.rst create mode 100644 rst/man/mkfs.rst create mode 100644 rst/man/mount.rst create mode 100644 rst/man/oauth_client.rst create mode 100644 rst/man/pcp.rst create mode 100644 rst/man/rm.rst create mode 100644 rst/man/stat.rst create mode 100644 rst/man/umount.rst create mode 100644 rst/man/verify.rst create mode 100644 rst/mkfs.rst create mode 100644 rst/mount.rst create mode 100644 rst/resources.rst create mode 100644 rst/special.rst create mode 100644 rst/tips.rst create mode 100644 rst/umount.rst create mode 100644 setup.cfg create mode 100755 setup.py create mode 100644 src/s3ql.egg-info/PKG-INFO create mode 100644 src/s3ql.egg-info/SOURCES.txt create mode 100644 src/s3ql.egg-info/dependency_links.txt create mode 100644 src/s3ql.egg-info/entry_points.txt create mode 100644 src/s3ql.egg-info/requires.txt create mode 100644 src/s3ql.egg-info/top_level.txt create mode 100644 src/s3ql.egg-info/zip-safe create mode 100644 src/s3ql/__init__.py create mode 100644 src/s3ql/adm.py create mode 100644 src/s3ql/backends/__init__.py create mode 100644 src/s3ql/backends/common.py create mode 100644 src/s3ql/backends/comprenc.py create mode 100644 src/s3ql/backends/gs.py create mode 100644 src/s3ql/backends/local.py create mode 100644 src/s3ql/backends/pool.py create mode 100644 src/s3ql/backends/rackspace.py create mode 100644 src/s3ql/backends/s3.py create mode 100644 src/s3ql/backends/s3c.py create mode 100644 src/s3ql/backends/swift.py create mode 100644 src/s3ql/backends/swiftks.py create mode 100644 src/s3ql/block_cache.py create mode 100644 src/s3ql/calc_mro.py create mode 100644 src/s3ql/common.py create mode 100644 src/s3ql/cp.py create mode 100644 src/s3ql/ctrl.py create mode 100644 src/s3ql/daemonize.py create mode 100644 src/s3ql/database.py create mode 100644 src/s3ql/deltadump.c create mode 100644 src/s3ql/deltadump.pyx create mode 100644 src/s3ql/endian_indep.h create mode 100644 src/s3ql/exit_stack.py create mode 100644 src/s3ql/fs.py create mode 100644 src/s3ql/fsck.py create mode 100644 src/s3ql/inherit_docstrings.py create mode 100644 src/s3ql/inode_cache.py create mode 100644 src/s3ql/lock.py create mode 100644 src/s3ql/logging.py create mode 100644 src/s3ql/metadata.py create mode 100644 src/s3ql/mkfs.py create mode 100644 src/s3ql/mount.py create mode 100644 src/s3ql/multi_lock.py create mode 100644 src/s3ql/oauth_client.py create mode 100644 src/s3ql/parse_args.py create mode 100644 src/s3ql/remove.py create mode 100644 src/s3ql/statfs.py create mode 100644 src/s3ql/umount.py create mode 100644 src/s3ql/verify.py create mode 100644 tests/common.py create mode 100644 tests/conftest.py create mode 100644 tests/mock_server.py create mode 100644 tests/pytest.ini create mode 100644 tests/pytest_checklogs.py create mode 100755 tests/t1_backends.py create mode 100755 tests/t1_dump.py create mode 100755 tests/t1_retry.py create mode 100755 tests/t1_serialization.py create mode 100755 tests/t2_block_cache.py create mode 100755 tests/t3_fs_api.py create mode 100755 tests/t3_fsck.py create mode 100755 tests/t3_inode_cache.py create mode 100755 tests/t3_verify.py create mode 100755 tests/t4_adm.py create mode 100755 tests/t4_fuse.py create mode 100755 tests/t5_cp.py create mode 100755 tests/t5_ctrl.py create mode 100755 tests/t5_failsafe.py create mode 100755 tests/t5_fsck.py create mode 100755 tests/t5_full.py create mode 100755 tests/t5_lock_rm.py create mode 100755 tests/t6_upgrade.py create mode 100644 util/cmdline_lexer.py create mode 100644 util/sphinx_pipeinclude.py diff --git a/Changes.txt b/Changes.txt new file mode 100644 index 0000000..75b7aeb --- /dev/null +++ b/Changes.txt @@ -0,0 +1,1405 @@ +2016-10-28, S3QL 2.21 + + * Added workaround for an issue where S3QL crashes with an + "ssl.SSLError: [SSL: BAD_WRITE_RETRY]" exception. This problem has + to be fixed in Python, but we now work around it by closing and + re-opening the connection when it occurs. + +2016-08-24, S3QL 2.20 + + * Fixed a problem with exporting S3QL file systems over NFS4, + where directory contents would appear in duplicate. + +2016-06-27, S3QL 2.19 + + * Fixed a crash when running s3qlstat on an almost empty + file system. + +2016-04-29, S3QL 2.18 + + * The `runtests.py` file has been dropped in favor of requiring + installation of py.test. + + * S3QL now uses semantic versioning. This means that + backwards-incompatible versions (i.e., versions that require an + upgrade of the file system revision) will be reflected in an + increase of the major version number, i.e. the next + backwards-incompatible version will have version 3.0. + + * Enabled several FUSE optimizations (most importantly, the + ``big_writes`` feature) that got accidentally disabled at + some point in the past. This should result in significant + performance improvements. + + Thanks to Simon Brown for the report! + + * Fixed a crash with "BUG ALERT: Dirty inode was destroyed". + +2016-03-08, S3QL 2.17.1 + + * Fixed a bug in the upgrade procedure that prevented file system + modifications when successive commands used the local metadata + cache. + + * Fixed a bug in the upgrade procedure that resulted in corrupted + file [acm]times when there was no cached metadata available for + both s3qladm and the first fsck.s3ql/mount.s3ql invocation after + the upgrade. + +2016-03-08, S3QL 2.17 + + * The internal file system revision has changed. File systems + created with S3QL 2.17 or newer are not compatible with prior S3QL + versions. To update an existing file system to the newest + revision, use the `s3qladm upgrade` command. + + * Due to a bug in the FUSE kernel module + (cf. https://github.com/libfuse/libfuse/issues/23) S3QL does not + (and did not) honor the write restriction bit. This limitation is + now documented. (It is unknown if the same problem exists when + running under BSD or OS-X). + + * S3QL now honors the setgid bit on directories (as opposed to just + storing them). + + * The `s3qlstat` command now accepts a --raw option to inhibit + pretty-printing of sizes. + + * The `s3qlctrl log` command is now working again. + + * The `s3qlctrl cachesize` command is now working again. + + * The s3 backend now supports the "infrequent access" storage class + (cf. https://aws.amazon.com/s3/storage-classes/) + + * The 's3qlstat' command now prints more information about cache + usage, including the number of cache entries and pending object + removals. + +2016-02-23, S3QL 2.16 + + * S3QL now supports (and requires) Python-LLFUSE 0.43 or newer + (but not never than 1.0). + + * S3QL now retries when receiving a "ServiceUnavailable" error from + an S3 (or compatible) backend. + + * When retrying network requests, the back-off time now includes a + random factor to avoid accidentally flooding the server with too + many simultanuous requests. + + * Fixed a problem with fsck.s3ql aborting with a "ConstraintError" + in some situations. + + * Fixed a problem with fsck.s3ql repeatedly renaming a file in + /lost+found because it may lack data. + + * In the authinfo2 file, storage urls for the local backend may now + be specified with a trailing slash. + +2015-09-18, S3QL 2.15 + + * Fixed a compatibility problem with recent versions of + Python-LLFUSE (0.41 and newer). + + * S3QL now supports using special characters in the prefix + for S3 buckets. Previously, characters that would need to + be percent-encoded when part of an URL (like + or %) would + result in a crash. + +2015-07-31, S3QL 2.14 + + * S3QL now longer crashes when the server sends a zero-length + response that claims to be XML. + + * The default tcp-timeout has been increased from 10 seconds to 20 + seconds. + + * Fixed test failures when FUSE is not available, tests should now + again be skipped. + + * S3QL now tracks the frequency of server errors and logs warnings + if they occur more often than 5 times per second over a period of + at least 60 seconds. Previously, retries would only be logged + if the same request has been retried more than 3 times. + + * The "s3qlctrl cachesize" command is now working again (previously + it would appear to succeed, but not actually change the cache + size). + + * 404 errors when retrying a delete request are now ignored - most + likely they mean that the server actually received and executed + the previous try. + + * Fixed an issue with fsck.s3ql crashing when trying to create a + file with a very long name in /lost+found. + +2015-01-31, S3QL 2.13 + + * The internal file system revision has changed. File systems + created with S3QL 2.13 or newer are not compatible with prior S3QL + versions. + + To update an existing file system to the newest revision, use the + 's3qladm upgrade' command. During the upgrade, all storage objects + that were created by S3QL 1.1 or earlier will be downloaded and + re-uploaded. For objects created by S3QL 1.1.2 or later, the + upgrade will only affect the storage object's metadata (so no + contents need to be transferred). + + * SECURITY ADVISORY: Previous mkfs.s3ql versions used /dev/urandom + to generate the master key when creating a new encrypted file + system. However, /dev/urandom is not guaranteed to provide the + 256-bits of entropy requested by S3QL. This may have allowed an + attacker to predict parts of the master key in situations where + little entropy is available (e.g. right after system boot, or if a + different program has previously drained the entropy pool). + + Note that the master key is not affected when the file system + passphrase is changed. The only way to recover from a potentially + compromised master key is to create a new file system and copy + over all data. + + * When creating new file systems, the master key is now generated by + reading /dev/random rather than /dev/urandom to ensure sufficient + entropy. + + * The 'no-ssl' option for the swift backend is now used only when + connecting to the authentication server. Whether SSL is used when + connecting to the storage server is determined by the + authentication server. + + * Fixed a crash when using the "disable-expect100" swift backend + option. + + * Fixed a race condition that could lead to a "KeyError" crash + when refreshing a Google Storage OAuth2 access token. + + * Fixed a race condition that could lead to a hanging mount.s3ql + process and hanging test_thread_hang() unit test. + + * Updated internal metadata storage and checksum format. The old + format was difficult to secure against malicious backend data and + could have resulted in false-positive checksum mismatches with + future or different Python interpreters. + + * Sizes (e.g. in the s3qlstat output) are now always reported + with three significant digits. + + * Fixed a bug that caused fsck.s3ql to either abort with a + "apsw.ConstraintError" or to incorrectly consider storage + objects as missing when the connection to remote server is + interrupted while retrieving the object list. + + * Storage urls without any prefix (e.g. s3://foo, but also + s3://foo/) are now using the same local cache + directory. Previously they would use different directories, + despite pointing to the same file system. + + (Note that this does not affect storage urls with a prefix, + s3://foo/bar and s3://foo/bar/ refer to different locations in the + bucket, and thus correctly use different cache directories). + + * Fixed a problem where mount.s3ql would crash when unmouting the + file system because it could not delete the cache directory. This + could happen when the file system was not unmounted cleanly, but + fsck.s3ql was then run on a different system (or using a different + cache directory). + + * S3QL now requires at least version 3.4 of the dugong Python + module. + +2014-11-09, S3QL 2.12 + + * The s3c backend has a new 'dumb-copy' option. If this option + is enabled, copy operations are assumed to have succeeded + as soon as the server returns a '200 OK' status, without + checking the contents of the response body. + + * The swift and s3c backends have a new 'disable-expect100' to + disable support for the 'Expect: continue' header. Using this + option allows S3QL to work with storage servers without proper + HTTP 1.1 support, but may decrease performance as object data will + be transmitted to the server more than once in some circumstances. + + * contrib/benchmark.py is now working again. + + * The `tcp-timeout` backend option is now actually working + instead of resulting in a "TypeError". + + * Fixed a problem where saving metadata would fail with + "ValueError: Can't dump NULL values". + + * s3qlstat now also gives information about cache usage. + +2014-09-04, S3QL 2.11.1 + + * By popular demand, mount.s3ql is now able to daemonize again + and also does so by default. + + * Un-encrypted file systems (created with mkfs.s3ql --plain) can now + be upgraded to the newest revision. + + * S3QL now automatically retries a request if a backend server + reports a 408 (Request Timeout) status code. + + * The Google Storage backend now copes better with server errors + when refreshing an OAuth2 access token. + +2014-08-27, S3QL 2.11 + + * SECURITY UPDATE (CVE-2014-0485). + + A remote code execution vulnerability was fixed. + + An attacker with control over the communication with the storage + backend or the ability to manipulate the data stored in the + backend was able to trigger execution of arbitrary code by + mount.s3ql, fsck.s3ql, mkfs.s3ql, s3qladm and s3ql_verify. Both + encrypted and unencrypted file systems were vulnerable. + + * s3ql_verify no longer crashes when checking an empty file system. + + * Fixed a crash when using Google OAuth2 and the first request after + the access token has expired is a write request. + + * mount.s3ql now supports systemd-style readyness notification. To + use this feature, make sure that you have the 'systemd' module + installed for Python 3. + + This feature has not been tested, if you use it or encounter + problems, please report back. + + * Fixed a race condition that could cause tests/t5_failsafe.py to + fail. + + * mount.s3ql no longer daemonizes on its own. With a modern init + system this should no longer be necessary, and when running + mount.s3ql from the command line the shell can be used to put the + process into background. + + * There is a new --backend-options parameter that can be used to + pass backend-specific options to any S3QL command that accepts a + storage url. + + * The --no-ssl and --ssl-ca-path parameters have been removed. For + those backends were these parameters make sense, you can use the + backend options of the same name instead (e.g. instead of + `--no-ssl` use `--backend-options no-ssl`). + + * Several backends now accept a `tcp-timeout` option. If S3QL is + unable to communicate with the remote server for longer than this + period, the TCP/IP connection is re-established. + + * The Amazon S3 backend now accepts a 'sse' option to enable server + side encryption. Both costs & benefits of S3 server side + encryption are probably very small, and this option does *not* + affect any client side encryption performed by S3QL itself. + + * The Amazon S3 backend now accepts a 'rrs' option to enable reduced + redundancy storage for any newly created objects. + +2014-07-28, S3QL 2.10.1 + + * 'setup.py install' now also installs the s3ql_verify command. + +2014-07-27, S3QL 2.10 + + * The internal file system revision has changed. File systems + created with S3QL 2.10 or newer are not compatible with prior S3QL + versions. To update an existing file system to the newest + revision, use the 's3qladm upgrade' command. + + It is strongly recommended to run the (new) s3ql_verify command + with the --data option at shortly after the upgrade. This is + necessary to ensure that the upgrade to the next (2.11) S3QL + release will run smoothly. + + * The User's Guide now contains a description of the possible + failure modes of mount.s3ql. + + * The --debug command line parameter now generates a bit less + output by default, and there is an additional --debug-module + parameter to activate additional messages. + + * When using encryption, S3QL now checks that a storage object's + key corresponds to the data stored under it. + + The lack of this check in previous versions allowed an attacker + with control over the storage server to interchange blocks within + the same file system (which would have resulted in apparent data + corruption in the file system). Targeted modification of specific + files, however, would have been very unlikely, because the + interchange of blocks had to be done blindly (with the attacker + not knowing which file any block belongs to, nor what its contents + are). + + Fixes https://bitbucket.org/nikratio/s3ql/issue/52/. + + * S3QL now aborts immediately instead of retrying if the server + storage server reports that the local clock is skewed. + + * There is a new 's3ql_verify' command. This program retrieves and + checks every storage object to ensure that all objects are + available and have not been tampered with. In contrast to + fsck.s3ql, s3ql_verify does not trust the object list provided by + the storage server and actually attempts to download the objects + one by one. + + * S3QL now requires version 3.2 or newer of the dugong module. + +2014-06-28, S3QL 2.9 + + * Fix crash when using swift backend and server uses an + authentication URL other than /v1.0. + + * Fixed two test failures when running unit tests as root. + + * Fixed problems when receiving an HTTP error without a well-formed + XML body from the remote server (this may happen e.g. due to + failure in a transparent proxy or load balancer). + + * S3QL now depends on the defusedxml Python module + (https://pypi.python.org/pypi/defusedxml/). + + * S3QL is no longer vulnerable to DOS attacks from malicious backend + servers. Previously, a malicious server could induce S3QL to + consume arbitrary amounts of memory by recursive XML entity + expansion. + + * S3QL now supports Google OAuth2 authentication. To use it, specify + 'oauth2' as backend login, and a valid OAuth2 refresh token as + backend password. To obtain the refresh token, the (new) + s3ql_oauth_client command may be used. + + * S3QL now requires version 3.1 or newer of the dugong module. + + * In some cases, transmission errors when reading storage objects + from a backend may have been misreported as corrupted backend + objects. This has been fixed. + + * S3QL no longer crashes when data corruption occurs in the first + few bytes of an LZMA compressed storage object. + + * S3QL now honors the "Retry-After" HTTP header when receiving an + XML error from a storage server. + + * Fixed a crash that could occur when the remote server (or some + intermediate proxy) sends a non-XML error response. + + * mount.s3ql and fsck.s3ql now use different exit codes to indicate + different failure conditions. + +2014-03-29, S3QL 2.8.1 + + * No changes in S3QL itself. + + * The S3QL 2.8 tarball accidentally included a copy of the Python + dugong module. This has been fixed in the 2.8.1 release. + +2014-03-13, S3QL 2.8 + + * Fixed various problems with using a proxy server. + + * Sending SIGUSR1 to mount.s3ql now generates a stacktrace + (debugging feature). + + * When passing --installed to the test runner, S3QL commands + are now loaded from $PATH instead of the packages bin/ + directory. + + * The runtest.py script now comes with the correct shebang + (i.e., it can now be called as "./runtests.py" instead of + "python3 runtests.py"). + + * S3QL now requires the python dugong module + (https://bitbucket.org/nikratio/python-dugong) + + * Fixed a file system hang when all upload threads encounter + unexpected backend problems. + +2013-12-16, S3QL 2.7 + + * Fixed another race condition that could lead to mount.s3ql + crashing with `ValueError: I/O operation on closed file`. + + * S3QL no longer generates warning messages for the first two times + that it has to resend a request to the storage backend. If there + is no success after the second try, messages are emitted as before. + + * S3QL now stores multiple copies of the master encryption key to + allow recovery if the backend looses the object holding the + primary copy. To take advantage of this functionality for existing + file systems, change the file system passphrase with s3qladm. + + * Fixed problem with automatic cache size detection (mount.s3ql was + treating bytes as kilobytes). Thanks to GVormayr@gmail.com for the + patch! + + * Fixed "AttributeError in LegacyDecryptDecompressFilter" crash when + reading objects written by old S3QL versions. + + * Fixed a problem with umount.s3ql giving a strange error when + the mountpoint is still in use. + +2013-12-03, S3QL 2.6 + + * Fixed a problem with mount.s3ql crashing with `KeyError` in + `collections/abc.py`. + + * Fixed a problem with mount.s3ql crashing with `ValueError: I/O + operation on closed file`. + + * Fixed a race condition that could lead to data corruption when + multiple processes trigger download of the same data block + simultaneously. + + * Fixed a problem with S3QL crashing with "SSLEOFError" when an SSL + connection is terminated unexpectedly. + + * Increased deduplication performance. Previously, block checksums + were accidentally computed without parallelism. + + * Fixed a problem with mount.s3ql (incorrectly) reporting corrupted + data for compressed blocks of some specific sizes. Many thanks to + Balázs for extensive debugging of this problem. + + * Specifying --debug when using syslog or file logging no longer + suppresses informational messages to stdout. + + * Fixed build under MacOS X. Thanks to john@nachtimwald.com for the + patch! + + * mount.s3ql now autodetects a sensible defaults for the + --max-cache-entries and --cachesize options. + +2013-10-19, S3QL 2.5 + + * Removed a race condition that resulted in a crash when a file was + deleted while the upload was in progress, and the upload was then + restarted because of a connection problem. + + * Specifying bzip2 as the compression algorithm for mount.s3ql + no longer gives an error. + + * Fixed another race condition that could lead to the (apparent) + vanishing of the file system metadata when the mount.s3ql is + killed at an unfortunate time. + + * S3QL programs no longer create an empty file in the default log + file location if a custom log file is specified with --log. + + * contrib/benchmark.py is now working again - thanks to + davidthomson@gmail.com. + + * If a backend returns corrupted data, mount.s3ql no longer crashes + but switches to read-only operation. + + * Fixed a problem with error messages from mount.s3ql being lost, + resulting in a file system shutdown for apparently no reason. + +2013-08-27, S3QL 2.4 + + * Attempts to upgrade a file system that is already at the newest + revision no longer render the local metadata copy unusable. + + * Google storage bucket names containing dots no longer cause + problems with SSL certificate validation. + + * Fixed crash when statvfs() is called on an almost empty file + system (which may happen automatically when using some desktop + environments, resulting in an immediate crash after mounting an + empty filesystem). + + * There is a new "swiftks" backend for OpenStack/Swift servers that + supports keystone authentication and allows specification of the + desired storage region. + + * Eliminated a race condition that could lead to mount.s3ql crashing + with a "KeyError in collections/abc.py" under some circumstances. + + * Fixed a crash that could occur when trying to clean up after + receiving a connection or server error when uploading an object. + +2013-07-18, S3QL 2.3 + + * Reading the file system passphrase from the authinfo2 file + is now working again. + + * The precedence of the authinfo2 sections has been fixed to agree + with the documentation: later entries override earlier entries + rather than the other way around. + + * Fixed a problem with mount.s3ql not recognizing the upgraded + revision when `s3qladm upgrade` was run with a different + --cachedir argument. + + * Fixed a crash of the s3 backend when multiple objects are deleted + and the s3 connection needs to be re-established. + +2013-07-12, S3QL 2.2 + + * Eliminated a race condition that resulted in an apparent loss of + remote metadata when the mount.s3ql process was interrupted after + renaming the remote metadata but before having completely uploaded + the local metadata. + + * Attempting to run mount.s3ql with --allow-user but without having + the necessary permissions no longer results in the file system + being marked as needing fsck. + + * The S3 backend now deletes multiple storage objects using one + request if possible, reducing latency, traffic and number of + requests to S3. + + * S3QL is now able to detect server errors and redirections before + all data has been sent to the server. This avoids pointless data + transfers and should fix a problem with S3QL operating very slow + when operating on recently created storage buckets/containers. + + * There is a new 'rackspace' backend that allows specification of + the object storage region in the storage URL. The rackspace + backend also uses a different authentication API than the swift + backend (which can still be used for Rackspace as well). When + using the 'rackspace' backend, use your regular Rackspace password + instead of the API key as the backend password. + + * The mount.s3ql --compress option now allows specification of the + compression level in addition to the compression algorithm. + +2013-06-14, S3QL 2.1 (beta) + + * S3QL now uses SSL connections by default. + + * S3QL now verifies server certificates when using SSL connections. + + * There is a new --no-ssl option that now works for every backend. + + * The Amazon S3 and Swift backends are now working again (regression + was introduced in 2.0-beta). + +2013-05-12, S3QL 2.0 (beta) + + * S3QL now requires Python 3.3 or newer. + + * S3QL now requires the pycrypto third-party Python module. + + * S3QL now requires at least version 0.39 of the llfuse Python + module. + + * Cycling of metadata backups when using the local backend now + takes much less time. + + * S3QL no longer requires the unittest2, pycryptopp and lzma + third-party Python modules. + + * It is no longer required that S3QL and Python APSW be linked to + the same SQLite shared library. + + * mount.s3ql: fixed crash when using --metadata-upload-interval 0. + + * Instead of completely unmounting the file system (and leaving the + mount point inaccessible) when backend data is missing or + corrupted, S3QL now returns an error and switches to read-only + operation. + +2013-04-09, S3QL 1.14 + + * Reduced CPU consumption when S3QL is idle. + + * S3QL now automatically retries requests when S3 reports an + "internal error" (this functionaly got accidentally broken + in S3QL 1.13). + + * Fixed a hang when using s3qlrm to delete a directory with + a very large number of sub-directories. + +2013-03-03, S3QL 1.13.2 + + * Allow S3 bucket names starting with a number. + + * Return EOPNOTSUP error on ACL operations. The previous value of + EINVAL works with FUSE, but confuses several applications. + +2013-02-24, S3QL 1.13.1 + + * Change error code returned to applications trying to set ACL + extended attributes. The previous error code of ENOSYS had the + unintended side effect that some FUSE versions gave up trying to + get/set any extended attributes at all. + +2013-02-23, S3QL 1.13 + + * Fixed issues when trying to access a directory at the root + of the file system that has recently been removed with s3qlrm. + (In that case the directory was no longer included in the + readdir() output, but it was still posssible to stat() it because + was still present in the kernel's dentry cache). + + * When trying to connect to the backend, S3QL now also retries at + increasing intervals if DNS appears to be unavailable (i.e., if + there's currently no network at all). This means that mount.s3ql + will no longer bail out when called without a network connection + but simply retry (printing warning messages in the process). + + * Work around undocumented "Timeout" reply from Google Storage and + retry instead of crashing. + + * Instead of failing with a bogus "Invalid credentials or skewed system + clock" error when trying to access a bucket with an invalid DNS + name, S3QL now prints a proper error message. + + (Such bucket names are still allowed when using the US Standard + regions, but generally not a good idea). + + * Removed ostensible ACL support. It was found that S3QL never kept + ACLs and file permission bits synchronized as mandated by POSIX. + This is due to a bug in the FUSE library and cannot easily be + fixed in S3QL. Consequently, S3QL no longer reports ACL support + unless a FUSE version with the necessary bugfix version is + installed. + + * Fixed a bug that caused malformed metadata to be written out when + using recent eglibc versions (the relevant change in eglibc + was introduced between eglibc versions 2.13 and 2.17). + + +2012-09-03, S3QL 1.12 + + * Be more verbose about how to remedy the situation when attempting + to upgrade file system needing fsck. + + * Fsck now detects and fixes directory entries which do not have an + entry type stored in their mode field. + +2012-05-04, S3QL 1.11.1 + + * Fixed crash when using S3 backend. + (Regression introduced in release 1.11). + + * Increase minimum reported file system size to 1 TB, and work + around df bug so that size is reported properly. + + * umount.s3ql now produces a more helpful error message if the + file system has crashed. + +2012-04-29, S3QL 1.11 + + * S3QL no longer uses the confusing "bucket" term (which is S3 + specific). The "bucket-passphrase" option in the authentication + file has been renamed to "fs-passphrase". + + * Metadata about stored objects is now split into multiple fields, + so that no field is longer than 256 bytes. This makes S3QL + compatible with a wider range of storage providers. + + Thanks to Stuart Wallace for reporting this + issue. + + * The SWIFT backend now retries if it receives 5xx error codes from + the remote server, and no longer fails when attempting to upload + data with an expired auth token. Thanks to Ken + for the report and initial patch! + +2012-02-24, S3QL 1.10 + + * The s3s, s3cs and gss backends have been removed. Use the new + --ssl option together with the s3, s3c and gs backends instead. + + * S3QL no longer keeps track of consistency guarantees for the + different backends. The increasing number of different storage + providers offering different regions, redundancy and availability + levels makes this no longer feasible. + + * The User's Guide contains a new section "Important Rules to Avoid + Losing Data". Reading it is strongly recommended. + +2012-01-21, S3QL 1.9 + + * Fixed a problem with file and directory time stamps + changing with the time zone of the mount.s3ql process. + + * Fixed a crash in contrib/benchmark.py and made output + (hopefully) easier to understand. + + * Fixed "Too many chained redirections" bug when trying to access + a new bucket immediately after creation. + + * When receiving 5xx errors from the server, S3QL now waits and + retries instead of aborting immediately. + + * contrib/s3ql_upstart.conf now supports running S3QL as an + ordinary user. + + * S3QL now has experimental, native support for OpenStack/SWIFT + servers. Thanks to the guys from www.bitbackup.de for providing + support and a test server! + + * S3QL now has experimental proxy support and will honor the + http_proxy and https_proxy environment variables. + +2011-12-06, S3QL 1.8.1 + + * Fixed direct updates from 1.6 to 1.8. + +2011-12-05, S3QL 1.8 + + * Fixed "ValueError: Can't dump NULL values" crash when file system + metadata was uploaded during heavy file system activity. + + * Deleting files no longer takes O(number of distinct filenames) + time (bug introduced in 1.7). + + * Improved s3qlcp and directory listing performance for very large + directories. + + * The --blocksize option of mkfs.s3ql has often been misunderstood, + leading people to create file systems with block sizes + significantly smaller than 1 MB. + + As a matter of fact, a small S3QL block size does *not* have any + advantage over a large block size when storing lots of small + files. A small block size, however, seriously degrades performance + when storing larger files. This is because S3QL is effectively + using a dynamic block size, and the --blocksize value merely + specifies an upper limit. + + To make this more clear, the --blocksize option has been renamed + to --max-obj-size. If you created your file system with a block + size of less than 1 MB, it is strongly recommended to recreate it + with a larger block size. Unfortunately, an in-place conversion of + the block size is not possible. + + * mount.s3ql has a new --min-obj-size option. In the future, files + smaller than the minimum object size may be combined into groups + that are stored as single objects in the storage backend. + + * Depend on python-llfuse 0.37. This fixes a problem with the file system + effectively blocking any other request while an s3qlrm, s3qllock + or s3qlcp instance is running. + + * Fixed some crashes happening under heavily concurrent file system + usage. + +2011-11-27, S3QL 1.7 + + * Extended attribute names are now de-duplicated. + + * Metadata is now stored in a custom, delta-encoded binary format and + then BZIP2 compressed, resulting in a 5-fold speedup when dumping + and compressing. + + * Inodes are now assigned sequentially rather than randomly, and old + inodes are not reused. This makes S3QL fully NFS compatible and + allows metadata to be stored much more efficiently, resulting in + a 4 to 8 fold decrease in compressed metadata size. + + * s3qlcp now also copies extended attributes. + + * s3qlcp no longer generates incorrect block reference counts when + copying a file that has identical blocks (i.e., that can be + de-duplicated within itself). + + * Eliminated a race condition in s3qlcp. When copying a file with + s3qlcp immediately after it was modified or created, it was + possible that s3qlcp would copy the new file attributes (size, + modification time, etc.) but the old data blocks. + + A copy of a newly created file would then contain only zero bytes, + while a copy of a modified file would look like the original but + contain data from before the modification. + + * "mkfs.s3ql --force" and "s3qladm clear" are working again. + +2011-11-20, S3QL 1.6 + + * fsck: gracefully recover if temporary indices already exist + (e.g. if previous fsck was interrupted). + + * Due not fail with "BadDigest" error when objects are only + partially retrieved from backend. + +2011-10-20, S3QL 1.5 + + * Fixed parsing of storage urls, s3s:// no longer generates bogus + error message. + + * Fixed support for prefix in storage urls. + + * Retry on timeout when transmitting data to remote server. + + * Do not free mount point when terminating due to unhandled + exception in background thread. + +2011-10-06, S3QL 1.4 + + * Metadata is now always LZMA compressed, mount.s3ql's --compress + option only affects compression of file contents. + + * Network errors that occur in the middle of a read operation + are now handled correctly as well. + + * s3qladm now uses cached metadata if available. + +2011-10-04, S3QL 1.3 + + * Fixed an s3qlcp crash happening when source or destination inode + number was a 64 bit value. + + * "Connection reset by peer" network errors are now handled + correctly. + + * fsck.s3ql can now renumber inodes. This is useful if the file + system was mounted using S3QL 1.2 or 1.1.x without the --nfs + option and can now no longer be accessed on 32bit systems. + + * Use only 32 bits for inodes, even when --nfs is not specified. + This ensures that file systems can be safely used by both 32 and + 64 bit systems. + +2011-09-28, S3QL 1.2 + + * Fixed a database problem that was responsible for file + system access becomer slower and slower the more data + was stored in the file system. + + * Fixed a race condition that could cause applications to get just + zero bytes when trying to read from a file that has just been + copied with s3qlcp. + +2011-09-20, S3QL 1.1.4 + + * Fixed a typo that caused errors when trying to remove any blocks + that have been committed to the backend. + + * Improved accuracy of s3qlstat during active file transfers + (but still not 100% accurate). + + * Fixed some theoretical deadlocks. + + * contrib/benchmark.py is now working again and also takes into + account the throughput from userspace to S3QL. + +2011-09-18, S3QL 1.1.3 + + * Fixed a race condition in the local backend that resulted in + errors of the form "[Errno 17] File exists: [bucket path]". + + * Added Google Storage backend. + + * Added backend for general, S3 compatible storage services. + + * Fixed a bug that caused S3QL to complain about the backend having + lost objects when trying to download an object before its upload + was completed. This could happen because locking was done based on + inode and block number rather than object id, and the + de-duplication feature can reuse an object for another inode + before the upload is completed. + + * Fixed a data corruption bug. If a data block was changed while + being uploaded to the backend, and a second, identical data block + was flushed while the upload was in progress, but before the first + block was changed, the second data block was linked to the + *modified* data. This happened because changes to an object in + transit were only checked for after the upload completed, leaving + a window in which the contents of the upload object did not agree + with its stored hash. + + This problem can be detected by verifying the hash values of all + stored data blocks. This procedure will automatically be done when + the file system is updated to the newest revision, but may take a + longer time since every object needs to be downloaded and checked. + +2011-09-08, S3QL 1.1.2 + + * The modules for communicating with the different storage providers + have been completely rewritten, resulting in improved performance, + more features and better maintainability. + + * S3 buckets can now be used with arbitrary prefixes, allowing to + store more than one S3QL file system in a bucket. + + * The format of the --authinfo file has changed. See the + documentation for details of the new format. To avoid breaking + backwards compatibility, the default file is now called authinfo2. + + * Network errors are now handled much more consistently. + + * There is a new --nfs option for mount.s3ql that needs to be used + when the S3QL file system will be exported over NFS. + + * The local backend now stores data and metadata in the same file, + so it needs only half as many files as before. + + * The --homedir option has been replaced by the more finely grained + --authfile, --cachedir and --log options. + + * S3QL can now log directly to syslog. + + * The sftp storage backend has been dropped. The recommended way to + use S3QL over ssh is to use sshfs + (http://fuse.sourceforge.net/sshfs.html) with S3QL's local + backend. + + * fsck now checks if all indices have been created. This avoids + a huge performance problem when mount.s3ql was interrupted + after downloading metadata, but before creating the indices. + +2011-07-23, S3QL 1.1 (development version) + + * Restructured metadata. This should also significantly reduce the + size of the SQLite database file. + + * Fixed license typo in file header comments, license + is GNU GPL Version 3, not LGPL. + + * Fixed problem with fsck.s3ql generating extraordinary long + filenames in /lost+found and then crashing. + + * When called as root, use umount rather than fusermount for + compatibility with FUSE4BSD. + +2011-05-20, S3QL 1.0.1 + + * Disabled WAL mode again for now because of unexpected problems + with s3qlcp, s3qllock and s3qlrm (performance going down orders + of magnitude, and *very* large *.db-wal file in ~/.s3ql). + +2011-05-13, S3QL 1.0 + + * S3QL has been declared stable after 2 years of beta-testing did + not reveal any data-critical bugs. + + * Fixed occasional assertion error when calling s3qlctrl flushcache + or unmounting the file system. + + * Fixed a race condition when a block is expired while it is in + transit but has already been modified again. + + * expire_backups.py no longer has an --init option, the state file + is created automatically if this operation is safe. Instead, there + is a --reconstruct-state option that can be used to try to + reconstruct a lost state file. + + * The size of symbolic links is now reported as the length of + the target instead of zero. This used to confuse revision + control systems like git and hg. + + * Added man pages for all S3QL commands. + +2011-02-04, S3QL 0.30 + + * S3QL now defaults to use unencrypted HTTP connections, which + significantly improves performance when using the S3 backend. For + an encrypted file system, all data is already encrypted anyway, + and authentication data is never transmitted in plain text even + for unencrypted file systems. Therefore, the use of SSL brings + little gain for most users. To force SSL usage, the new --ssl + option can be used. + + * mkfs.s3ql now has a --force option to overwrite an existing file + system. + +2010-12-30, S3QL 0.29 + + * The FUSE interface has been rewritten using Cython and factored + out into a separate package, http://code.google.com/p/python-llfuse/. + + This should result in easier installation, better performance and + better maintainability. + + +2010-12-19, S3QL 0.28 + + * "s3qlctrl upload-meta" now works even if the file system has been + mounted with --metadata-upload-interval=0. + + * File system metadata is now permanently cached locally. This + significantly reduces the time required to mount the file system. + + * The documentation is now also included in PDF format. + +2010-12-11, S3QL 0.27 + + * The authinfo file now supports passwords that include white space. + + * The s3qladm command can now be used to download metadata backups. + + * The --strip-meta option for mount.s3ql has been removed, redundant + data is now always stripped before upload. + + * mount.s3ql now has a --upstart option so that it can easily run as + an upstart job. An example job definition is included in + contrib/s3ql.conf. + + * s3qlctrl now has an 'upload-meta' command to trigger a metadata + upload while the file system is mounted. + + * Fixed a bug that preserved old data when truncating a file to + zero. If you ever had fsck errors of the form + + Size of inode [ddd] ([filename]) does not agree with number of + blocks + + then the affected files may contain additional, old data at the + end that is not supposed to be there. + + +2010-10-30, S3QL 0.26 + + * Fixed a problem with the sftp backend failing because it tries to + access a file locally. + + * Various minor bugfixes + +2010-09-28, S3QL 0.25 + + * A critical bug that could cause backups to be deleted too early + and potentially break the whole backup strategy has been fixed in + contrib/expire_backups.py. + + The new version has changed semantics that and also copes + significantly better when backups are made in irregular time + intervals. + + * S3QL should now respond with more consistent performance when + accessing the file system while s3qlcp, s3qllock or s3qlrm is + running at the same time. + + * When enabling debug output for the `UploadManager` module, S3QL + now logs detailed messages about de-duplication, upload and + compression performance. + +2010-09-18, S3QL 0.24 + + * Fixed a deadlock that could cause the mount.s3ql process to hang + around after umount. + + * Fixed a bug that caused S3QL to consider all downloaded blocks as + dirty and resulted in unneccessary checksum calculations on + expiration. + + * s3qlctrl can now change the log level at runtime. + + * s3qladm delete now also deletes any local stale cache files. + + * Periodic metadata upload can now be disabled completely by + specifying an interval of zero. + + +2010-09-03, S3QL 0.23 + + * Fixed problem with global_lock.py not being installed by setup.py + +2010-08-31, S3QL 0.22 + + * Fixed a bug that could cause file system errors when calling + s3qlrm on a lot of really large files. + + * The sftp backend is now significantly faster, thanks to skyang2009 + for the patch. + + * s3qlctrl can now change the cache size of a mounted file system. + +2010-08-09, S3QL 0.21 + + * The metadata is now only uploaded if there have been any changes. + + * mount.s3ql now supports several parallel compression and + encryption threads with the --compression-threads option. + + * S3QL now supports "immutable directories". This important new + feature can be used to protect backups from modification after + they have completed. See the User's Guide for details. + + * Using S3 RRS is now deprecated, see User's Guide for details. + + * fsck.s3ql now moves damaged files due to data lost by a backend + into /lost+found/ + + * expire_backups is no longer installed automatically and can now be + found in the contrib/ directory. + + * S3QL now comes with sample backup script in contrib/s3_backup.sh + + * Added contrib/pcp.py, an rsync wrapper to recursively copy + directory trees with several parallel rsync processes. + +2010-08-01, S3QL 0.20.1 + + * Hotfix for s3qladm upgrade. + +2010-08-01, S3QL 0.20 + + * Added contrib/make_dummy.py. This script creates a dummy copy of a + bucket that contains only the file system metadata. The resulting + dummy can be used for debugging. + + * Mounting with the local and sftp backends is now significantly + faster, because the object tree is no longer traversed completely. + + * Fixed a race condition that occasionally produced errors when + deleting large files (spanning multiple blocks). + + * The file system now stays responsive to other requests while + compressing blocks. + + * s3qlstat is now much faster since the size after de-duplication + does not need to be queried from the backend anymore but is stored + in the metadata. + + +2010-07-25, S3QL 0.19 + + * Fixed various smaller bugs, see Mercurial changelog for details. + +2010-07-11, S3QL 0.18 + + * Added --strip-meta option to mount.s3ql + + * Added --metadata-upload-interval option to mount.s3ql. This allows + to periodically upload updated metadata even while the file system + is mounted. + + * stat.s3ql, tune.s3ql and cp.s3ql have been renamed to s3qlstat, + s3qladm and s3qlcp respectively. + + * sftp backend is working again + + * Added the s3qlctrl command. + +2010-06-29, S3QL 0.17 + + * The local and sftp backends now spread their files into different + sub-directories. + + * Amazon S3 Reduced Redundancy Storage (RRS) is now supported. To + use it, use a storage url of the form s3rr:// instead of + s3://. + +2010-06-15, S3QL 0.16 + + * Fixed problem with readdir() returning entries twice or skipping + them if files are added or removed while readdir() is in progress. + + * Fixed build problem on Gentoo. + + * fsck.s3ql now does a rudimentary check if the file system is still + mounted to prevent checking a mounted file system. + + +2010-05-28, S3QL 0.15 + + * Fixed test cases to handle systems with low system clock + resolution. + + * Corrected installation instructions for Debian + + * mount.s3ql: instead of --bzip2, --zlib and --no-compress, there is + now just one option --compress=. + + * File system metadata is now uploaded with all indices. This makes + mounting the file system much faster. Only if LZMA compression has + been chosen, indices are stripped for storage and regenerated on + the next mount. + +2010-05-14, S3QL 0.14 + + * fsck now detects if a cached block is dirty and commits only dirty + blocks to the backend. + + * Installation in Debian and Ubuntu is now much simpler, it can be + done completely with aptitude. + +2010-05-04, S3QL 0.13 + + * S3QL now works with Ubuntu Karmic / 10.04 LTS + + * The test command no longer produces bogus error messages after all + tests have completed. + + * The umount.s3ql command now properly handles the 'fuser' output + with Kernel 2.6.32 (before it always refused to unmount, claiming + that the mount point was busy). + + * The compression can now be specified independently from the + encryption, so it is possible to have an unencrypted, but + compressed file system. + + * Apart from zlib, bzip2 and lzma, data can now also be stored + without any compression. + + * S3QL no longer emulates the . and .. directory entries since this + is not required by POSIX and makes the code much simpler. This + also means that the st_nlink value of a directory is not longer 2 + + number of subdirectories. + + * Fixed a bug that caused files to be listed with wrong sizes under + certain conditions. + + * Added `expire_backups` command and instructions for a simple + backup solution using S3QL and rsync. + +2010-04-27, S3QL 0.12 + + * fsck.s3ql now accepts a --batch option to not prompt for any user + input and requires a --force option to check the file system even + if it is marked as clean. + + * Fixed a bug in cp.s3ql that caused incorrect st_nlink values in + the copy. + + * fsck.s3ql has been even more optimized. + + * Fixed a problem with running out of file descriptors when lots of + objects are deleted. + + * Bucket encryption passwords can now also be stored in the + ~/.s3ql/authinfo file. + + * mount.s3ql doesn't complain any more if it receives any of the + standard mount(8) mount options. + + +2010-04-24, S3QL 0.11 + + * S3QL file system can now be mounted from /etc/fstab + + * Metadata now takes significantly less space. + + * Files with extended attributes can now be deleted. + + * Extended attributes can now be listed. + + * It is now possible to choose between zlib, BZip2 and LZMA + compression every time the file system is mounted. + + * Added contrib/benchmark.py to find out optimal compression method + for given network bandwidth. + + * fsck.s3ql no longer uses mknod(3) , since that lead to errors on + e.g. NFS mounted volumes under Fedora. + + * File access, modification and inode change times before 1972 are + now supported. + + * Fixed a deadlock when removing or overwriting files. + +2010-04-21, S3QL 0.10 + + * S3QL now depends on FUSE version 2.8 + + * Cached blocks are now flushed to the backend as soon as they have + not been accessed for more than 10 seconds. + + * The setup script now automatically checks and, if necessary, + installs the Python module dependencies. + + * mkfs.s3ql now creates compressed and encrypted file systems by + default. + + * Data is now compressed with LZMA instead of Bzip2. + + * Data compression and data upload is now done in parallel, so the + full network can bandwidth is continuously without breaks for data + compression. + + * fsck.s3ql is now several orders of magnitude faster. The code has + been optimized and some special database indices are now + precalculated. + + * When running cp.s3ql, the attributes of the target directory are + now immediately refreshed (so that e.g. ls immediately shows the + correct number of hard links). + + * File removal is now significantly faster, since the network + transactions are carried out asynchronously as part of the cache + expiration. + + * mount.s3ql no longer tries to create files with mknod(), since + that lead to errors on NFS mounted volumes under Fedora. + + * This releases includes a lot of new code, so it may be slightly + more beta-ish than usual. + + +2010-04-04, S3QL 0.9 + + * The --cachedir, --logfile, --awskey and --credfile options are + gone and have been replaced by --homedir. + + * ~/.awssecret is no longer read, instead there is a common file + with authentication data for all backends in ~/.s3ql/authinfo + + * The syntax for mounting S3 buckets is now s3://bucketname instead + of just the bucket name + + * There is now an SFTP backend. Thanks to Ron Knapp + for most of the code. + + +2010-03-07, S3QL 0.8 + + * S3QL now uses Python's default unittest.py instead of shipping its + own. + + * Most documentation has been moved from the Wiki into the tarball, + so that it always corresponds to the correct version. + + * setuptools is now used for installation. This allows .egg + creation, dependency resolution and generation of the HTML + documentation. + + * The S3 part of boto has been integrated into S3QL. + +2010-02-22, beta7 + + * mount.s3ql no longer chdirs into / when daemonizing but into the + cache directory. + + * Added example backup script in contrib/backup.py + + * tune.s3ql --change-passphrase is working again + + * Added testcase for tune.s3ql --change-passphrase + + * Internal S3 errors are now handled properly by retrying the + upload. + +2010-02-19, beta6 + + * tune.s3ql --copy is now *really* working properly (brrr) + +2010-02-19, beta5 + + * mkfs.s3ql now makes strict checks on the bucket name + + * Removed obsolete mount.s3ql_local from distribution + + * tune.s3ql --copy is now working properly + +2010-02-19, beta4 + + * tune.s3ql now has a --copy option to copy buckets + + * Storage location for new buckets can be specified in mkfs.s3ql + and tune.s3ql with --s3-location + + * Fixed a deadlock in mount.s3ql when using local buckets without + --fg + + * The local: bucket specifier is no longer artificially slow. + + * mount.s3ql: --allow_other is now working + +2010-02-04, beta3 + + * Fixed a deadlock that caused umount.s3ql to hang indefinitely + when mount was called without --fg + + * The '.' and '..' directory entries are no longer explicitly + stored in the database. + + * Rewrote the mechanism to handle delayed updates. Now it no + longer depends on a small object being propagated relatively + fast, but is able to handle arbitrary network outages. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..74bbce4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,687 @@ +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License Version 3 as +published by the Free Software Foundation. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +For reference, the full text of the GNU General Public +License Version 3 is included below: + + + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/PKG-INFO b/PKG-INFO new file mode 100644 index 0000000..2c99854 --- /dev/null +++ b/PKG-INFO @@ -0,0 +1,175 @@ +Metadata-Version: 1.1 +Name: s3ql +Version: 2.21 +Summary: a full-featured file system for online data storage +Home-page: https://bitbucket.org/nikratio/s3ql/ +Author: Nikolaus Rath +Author-email: Nikolaus@rath.org +License: GPLv3 +Download-URL: https://bitbucket.org/nikratio/s3ql/downloads +Description: .. + NOTE: We cannot use sophisticated ReST syntax here because this + file is rendered by Bitbucket. + + ============ + About S3QL + ============ + + S3QL is a file system that stores all its data online using storage + services like `Google Storage`_, `Amazon S3`_, or OpenStack_. S3QL + effectively provides a hard disk of dynamic, infinite capacity that + can be accessed from any computer with internet access. + + S3QL is a standard conforming, full featured UNIX file system that is + conceptually indistinguishable from any local file system. + Furthermore, S3QL has additional features like compression, + encryption, data de-duplication, immutable trees and snapshotting + which make it especially suitable for online backup and archival. + + S3QL is designed to favor simplicity and elegance over performance and + feature-creep. Care has been taken to make the source code as + readable and serviceable as possible. Solid error detection and error + handling have been included from the very first line, and S3QL comes + with extensive automated test cases for all its components. + + .. _`Google Storage`: http://code.google.com/apis/storage/ + .. _`Amazon S3`: http://aws.amazon.com/s3 + .. _OpenStack: http://openstack.org/projects/storage/ + + + Features + ======== + + * **Transparency.** Conceptually, S3QL is indistinguishable from a + local file system. For example, it supports hardlinks, symlinks, + standard unix permissions, extended attributes and file + sizes up to 2 TB. + + * **Dynamic Size.** The size of an S3QL file system grows and shrinks + dynamically as required. + + * **Compression.** Before storage, all data may compressed with the + LZMA, bzip2 or deflate (gzip) algorithm. + + * **Encryption.** After compression (but before upload), all data can be + AES encrypted with a 256 bit key. An additional SHA256 HMAC checksum + is used to protect the data against manipulation. + + * **Data De-duplication.** If several files have identical contents, + the redundant data will be stored only once. This works across all + files stored in the file system, and also if only some parts of the + files are identical while other parts differ. + + * **Immutable Trees.** Directory trees can be made immutable, so that + their contents can no longer be changed in any way whatsoever. This + can be used to ensure that backups can not be modified after they + have been made. + + * **Copy-on-Write/Snapshotting.** S3QL can replicate entire directory + trees without using any additional storage space. Only if one of the + copies is modified, the part of the data that has been modified will + take up additional storage space. This can be used to create + intelligent snapshots that preserve the state of a directory at + different points in time using a minimum amount of space. + + * **High Performance independent of network latency.** All operations + that do not write or read file contents (like creating directories + or moving, renaming, and changing permissions of files and + directories) are very fast because they are carried out without any + network transactions. + + S3QL achieves this by saving the entire file and directory structure + in a database. This database is locally cached and the remote + copy updated asynchronously. + + * **Support for low bandwidth connections.** S3QL splits file contents + into smaller blocks and caches blocks locally. This minimizes both + the number of network transactions required for reading and writing + data, and the amount of data that has to be transferred when only + parts of a file are read or written. + + + Development Status + ================== + + S3QL is considered stable and suitable for production use. Starting + with version 2.17.1, S3QL uses semantic versioning. This means that + backwards-incompatible versions (e.g., versions that require an + upgrade of the file system revision) will be reflected in an increase + of the major version number. + + + Supported Platforms + =================== + + S3QL is developed and tested under Linux. Users have also reported + running S3QL successfully on OS-X, FreeBSD and NetBSD. We try to + maintain compatibility with these systems, but (due to lack of + pre-release testers) we cannot guarantee that every release will run + on all non-Linux systems. Please report any bugs you find, and we will + try to fix them. + + + Typical Usage + ============= + + Before a file system can be mounted, the backend which will hold the + data has to be initialized. This is done with the *mkfs.s3ql* + command. Here we are using the Amazon S3 backend, and + *nikratio-s3ql-bucket* is the S3 bucket in which the file system will + be stored. :: + + mkfs.s3ql s3://nikratio-s3ql-bucket + + To mount the S3QL file system stored in the S3 bucket + *nikratio_s3ql_bucket* in the directory ``/mnt/s3ql``, enter:: + + mount.s3ql s3://nikratio-s3ql-bucket /mnt/s3ql + + Now you can instruct your favorite backup program to run a backup into + the directory ``/mnt/s3ql`` and the data will be stored an Amazon + S3. When you are done, the file system has to be unmounted with :: + + umount.s3ql /mnt/s3ql + + + Need Help? + ========== + + The following resources are available: + + * The `S3QL User's Guide`_. + * The `S3QL Wiki`_, which also contains the `S3QL FAQ`_. + * The `S3QL Mailing List`_. You can subscribe by sending a mail to + `s3ql+subscribe@googlegroups.com + `_. + + Please report any bugs you may encounter in the `Bitbucket Issue Tracker`_. + + Contributing + ============ + + The S3QL source code is available both on GitHub_ and BitBucket_. + + + .. _`S3QL User's Guide`: http://www.rath.org/s3ql-docs/index.html + .. _`S3QL Wiki`: https://bitbucket.org/nikratio/s3ql/wiki/ + .. _`Installation Instructions`: https://bitbucket.org/nikratio/s3ql/wiki/Installation + .. _`S3QL FAQ`: https://bitbucket.org/nikratio/s3ql/wiki/FAQ + .. _`S3QL Mailing List`: http://groups.google.com/group/s3ql + .. _`Bitbucket Issue Tracker`: https://bitbucket.org/nikratio/s3ql/issues + .. _BitBucket: https://bitbucket.org/nikratio/s3ql/ + .. _GitHub: https://github.com/s3ql/main + +Keywords: FUSE,backup,archival,compression,encryption,deduplication,aws,s3 +Platform: POSIX +Platform: UNIX +Platform: Linux +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: No Input/Output (Daemon) +Classifier: Environment :: Console +Classifier: License :: OSI Approved :: GNU Library or Lesser General Public License (GPLv3) +Classifier: Topic :: Internet +Classifier: Operating System :: POSIX +Classifier: Topic :: System :: Archiving +Provides: s3ql diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..3259d5e --- /dev/null +++ b/README.rst @@ -0,0 +1,153 @@ +.. + NOTE: We cannot use sophisticated ReST syntax here because this + file is rendered by Bitbucket. + +============ + About S3QL +============ + +S3QL is a file system that stores all its data online using storage +services like `Google Storage`_, `Amazon S3`_, or OpenStack_. S3QL +effectively provides a hard disk of dynamic, infinite capacity that +can be accessed from any computer with internet access. + +S3QL is a standard conforming, full featured UNIX file system that is +conceptually indistinguishable from any local file system. +Furthermore, S3QL has additional features like compression, +encryption, data de-duplication, immutable trees and snapshotting +which make it especially suitable for online backup and archival. + +S3QL is designed to favor simplicity and elegance over performance and +feature-creep. Care has been taken to make the source code as +readable and serviceable as possible. Solid error detection and error +handling have been included from the very first line, and S3QL comes +with extensive automated test cases for all its components. + +.. _`Google Storage`: http://code.google.com/apis/storage/ +.. _`Amazon S3`: http://aws.amazon.com/s3 +.. _OpenStack: http://openstack.org/projects/storage/ + + +Features +======== + +* **Transparency.** Conceptually, S3QL is indistinguishable from a + local file system. For example, it supports hardlinks, symlinks, + standard unix permissions, extended attributes and file + sizes up to 2 TB. + +* **Dynamic Size.** The size of an S3QL file system grows and shrinks + dynamically as required. + +* **Compression.** Before storage, all data may compressed with the + LZMA, bzip2 or deflate (gzip) algorithm. + +* **Encryption.** After compression (but before upload), all data can be + AES encrypted with a 256 bit key. An additional SHA256 HMAC checksum + is used to protect the data against manipulation. + +* **Data De-duplication.** If several files have identical contents, + the redundant data will be stored only once. This works across all + files stored in the file system, and also if only some parts of the + files are identical while other parts differ. + +* **Immutable Trees.** Directory trees can be made immutable, so that + their contents can no longer be changed in any way whatsoever. This + can be used to ensure that backups can not be modified after they + have been made. + +* **Copy-on-Write/Snapshotting.** S3QL can replicate entire directory + trees without using any additional storage space. Only if one of the + copies is modified, the part of the data that has been modified will + take up additional storage space. This can be used to create + intelligent snapshots that preserve the state of a directory at + different points in time using a minimum amount of space. + +* **High Performance independent of network latency.** All operations + that do not write or read file contents (like creating directories + or moving, renaming, and changing permissions of files and + directories) are very fast because they are carried out without any + network transactions. + + S3QL achieves this by saving the entire file and directory structure + in a database. This database is locally cached and the remote + copy updated asynchronously. + +* **Support for low bandwidth connections.** S3QL splits file contents + into smaller blocks and caches blocks locally. This minimizes both + the number of network transactions required for reading and writing + data, and the amount of data that has to be transferred when only + parts of a file are read or written. + + +Development Status +================== + +S3QL is considered stable and suitable for production use. Starting +with version 2.17.1, S3QL uses semantic versioning. This means that +backwards-incompatible versions (e.g., versions that require an +upgrade of the file system revision) will be reflected in an increase +of the major version number. + + +Supported Platforms +=================== + +S3QL is developed and tested under Linux. Users have also reported +running S3QL successfully on OS-X, FreeBSD and NetBSD. We try to +maintain compatibility with these systems, but (due to lack of +pre-release testers) we cannot guarantee that every release will run +on all non-Linux systems. Please report any bugs you find, and we will +try to fix them. + + +Typical Usage +============= + +Before a file system can be mounted, the backend which will hold the +data has to be initialized. This is done with the *mkfs.s3ql* +command. Here we are using the Amazon S3 backend, and +*nikratio-s3ql-bucket* is the S3 bucket in which the file system will +be stored. :: + + mkfs.s3ql s3://nikratio-s3ql-bucket + +To mount the S3QL file system stored in the S3 bucket +*nikratio_s3ql_bucket* in the directory ``/mnt/s3ql``, enter:: + + mount.s3ql s3://nikratio-s3ql-bucket /mnt/s3ql + +Now you can instruct your favorite backup program to run a backup into +the directory ``/mnt/s3ql`` and the data will be stored an Amazon +S3. When you are done, the file system has to be unmounted with :: + + umount.s3ql /mnt/s3ql + + +Need Help? +========== + +The following resources are available: + +* The `S3QL User's Guide`_. +* The `S3QL Wiki`_, which also contains the `S3QL FAQ`_. +* The `S3QL Mailing List`_. You can subscribe by sending a mail to + `s3ql+subscribe@googlegroups.com + `_. + +Please report any bugs you may encounter in the `Bitbucket Issue Tracker`_. + +Contributing +============ + +The S3QL source code is available both on GitHub_ and BitBucket_. + + +.. _`S3QL User's Guide`: http://www.rath.org/s3ql-docs/index.html +.. _`S3QL Wiki`: https://bitbucket.org/nikratio/s3ql/wiki/ +.. _`Installation Instructions`: https://bitbucket.org/nikratio/s3ql/wiki/Installation +.. _`S3QL FAQ`: https://bitbucket.org/nikratio/s3ql/wiki/FAQ +.. _`S3QL Mailing List`: http://groups.google.com/group/s3ql +.. _`Bitbucket Issue Tracker`: https://bitbucket.org/nikratio/s3ql/issues +.. _BitBucket: https://bitbucket.org/nikratio/s3ql/ +.. _GitHub: https://github.com/s3ql/main diff --git a/bin/fsck.s3ql b/bin/fsck.s3ql new file mode 100755 index 0000000..64f0814 --- /dev/null +++ b/bin/fsck.s3ql @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +''' +fsck.s3ql - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright © 2008 Nikolaus Rath + +This program can be distributed under the terms of the GNU GPLv3. +''' + +import sys +import os.path + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +# When running from HG repo, enable all warnings +if os.path.exists(os.path.join(basedir, 'MANIFEST.in')): + import warnings + warnings.simplefilter('default') + +import s3ql.fsck +s3ql.fsck.main(sys.argv[1:]) diff --git a/bin/mkfs.s3ql b/bin/mkfs.s3ql new file mode 100755 index 0000000..03f6e4a --- /dev/null +++ b/bin/mkfs.s3ql @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +''' +mkfs.s3ql - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright © 2008 Nikolaus Rath + +This program can be distributed under the terms of the GNU GPLv3. +''' + +import sys +import os.path + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +# When running from HG repo, enable all warnings +if os.path.exists(os.path.join(basedir, 'MANIFEST.in')): + import warnings + warnings.simplefilter('default') + +import s3ql.mkfs +s3ql.mkfs.main(sys.argv[1:]) diff --git a/bin/mount.s3ql b/bin/mount.s3ql new file mode 100755 index 0000000..d449015 --- /dev/null +++ b/bin/mount.s3ql @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +''' +mount.s3ql - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright © 2008 Nikolaus Rath + +This program can be distributed under the terms of the GNU GPLv3. +''' + +import sys +import os.path + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +# When running from HG repo, enable all warnings +if os.path.exists(os.path.join(basedir, 'MANIFEST.in')): + import warnings + warnings.simplefilter('default') + +import s3ql.mount +s3ql.mount.main(sys.argv[1:]) diff --git a/bin/s3ql_oauth_client b/bin/s3ql_oauth_client new file mode 100755 index 0000000..b5e5aac --- /dev/null +++ b/bin/s3ql_oauth_client @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +''' +s3ql_oauth_client - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright © 2008 Nikolaus Rath + +This program can be distributed under the terms of the GNU GPLv3. +''' + +import sys +import os.path + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +# When running from HG repo, enable all warnings +if os.path.exists(os.path.join(basedir, 'MANIFEST.in')): + import warnings + warnings.simplefilter('default') + +import s3ql.oauth_client +s3ql.oauth_client.main(sys.argv[1:]) diff --git a/bin/s3ql_verify b/bin/s3ql_verify new file mode 100755 index 0000000..da6daaf --- /dev/null +++ b/bin/s3ql_verify @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +''' +s3ql_verify - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright © 2014 Nikolaus Rath + +This program can be distributed under the terms of the GNU GPLv3. +''' + +import sys +import os.path + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +# When running from HG repo, enable all warnings +if os.path.exists(os.path.join(basedir, 'MANIFEST.in')): + import warnings + warnings.simplefilter('default') + +import s3ql.verify +s3ql.verify.main(sys.argv[1:]) diff --git a/bin/s3qladm b/bin/s3qladm new file mode 100755 index 0000000..d3f99db --- /dev/null +++ b/bin/s3qladm @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +''' +s3qladm - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright © 2008 Nikolaus Rath + +This program can be distributed under the terms of the GNU GPLv3. +''' + +import sys +import os.path + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +# When running from HG repo, enable all warnings +if os.path.exists(os.path.join(basedir, 'MANIFEST.in')): + import warnings + warnings.simplefilter('default') + +import s3ql.adm +s3ql.adm.main(sys.argv[1:]) diff --git a/bin/s3qlcp b/bin/s3qlcp new file mode 100755 index 0000000..9433102 --- /dev/null +++ b/bin/s3qlcp @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +''' +s3qlcp - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright © 2008 Nikolaus Rath + +This program can be distributed under the terms of the GNU GPLv3. +''' + +import sys +import os.path + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +# When running from HG repo, enable all warnings +if os.path.exists(os.path.join(basedir, 'MANIFEST.in')): + import warnings + warnings.simplefilter('default') + +import s3ql.cp +s3ql.cp.main(sys.argv[1:]) diff --git a/bin/s3qlctrl b/bin/s3qlctrl new file mode 100755 index 0000000..e698762 --- /dev/null +++ b/bin/s3qlctrl @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +''' +s3qlctrl - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright © 2008 Nikolaus Rath + +This program can be distributed under the terms of the GNU GPLv3. +''' + +import sys +import os.path + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +# When running from HG repo, enable all warnings +if os.path.exists(os.path.join(basedir, 'MANIFEST.in')): + import warnings + warnings.simplefilter('default') + +import s3ql.ctrl +s3ql.ctrl.main(sys.argv[1:]) diff --git a/bin/s3qllock b/bin/s3qllock new file mode 100755 index 0000000..99c6b05 --- /dev/null +++ b/bin/s3qllock @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +''' +s3qllock - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright © 2008 Nikolaus Rath + +This program can be distributed under the terms of the GNU GPLv3. +''' + +import sys +import os.path + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +# When running from HG repo, enable all warnings +if os.path.exists(os.path.join(basedir, 'MANIFEST.in')): + import warnings + warnings.simplefilter('default') + +import s3ql.lock +s3ql.lock.main(sys.argv[1:]) diff --git a/bin/s3qlrm b/bin/s3qlrm new file mode 100755 index 0000000..1bd2392 --- /dev/null +++ b/bin/s3qlrm @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +''' +s3qlrm - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright © 2008 Nikolaus Rath + +This program can be distributed under the terms of the GNU GPLv3. +''' + +import sys +import os.path + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +# When running from HG repo, enable all warnings +if os.path.exists(os.path.join(basedir, 'MANIFEST.in')): + import warnings + warnings.simplefilter('default') + + +import s3ql.remove +s3ql.remove.main(sys.argv[1:]) diff --git a/bin/s3qlstat b/bin/s3qlstat new file mode 100755 index 0000000..7d658e8 --- /dev/null +++ b/bin/s3qlstat @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +''' +s3qlstat - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright © 2008 Nikolaus Rath + +This program can be distributed under the terms of the GNU GPLv3. +''' + +import sys +import os.path + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +# When running from HG repo, enable all warnings +if os.path.exists(os.path.join(basedir, 'MANIFEST.in')): + import warnings + warnings.simplefilter('default') + +import s3ql.statfs +s3ql.statfs.main(sys.argv[1:]) diff --git a/bin/umount.s3ql b/bin/umount.s3ql new file mode 100755 index 0000000..8b52e09 --- /dev/null +++ b/bin/umount.s3ql @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +''' +umount.s3ql - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright © 2008 Nikolaus Rath + +This program can be distributed under the terms of the GNU GPLv3. +''' + +import sys +import os.path + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +# When running from HG repo, enable all warnings +if os.path.exists(os.path.join(basedir, 'MANIFEST.in')): + import warnings + warnings.simplefilter('default') + +import s3ql.umount +s3ql.umount.main(sys.argv[1:]) diff --git a/contrib/benchmark.py b/contrib/benchmark.py new file mode 100755 index 0000000..7ac9c2b --- /dev/null +++ b/contrib/benchmark.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +''' +benchmark.py - this file is part of S3QL. + +Benchmark compression and upload performance and recommend compression +algorithm that maximizes throughput. + +--- +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +import argparse +import atexit +import os +import shutil +import subprocess +import sys +import tempfile +import time + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + exec_prefix = os.path.join(basedir, 'bin', '') +else: + exec_prefix = '' + +from s3ql.logging import logging, setup_logging, QuietError +from s3ql.common import get_backend +from s3ql.backends.common import DanglingStorageURLError +from s3ql.backends.comprenc import ComprencBackend +from s3ql.backends.local import Backend +from s3ql import BUFSIZE +from s3ql.parse_args import ArgumentParser + +ALGS = ('lzma', 'bzip2', 'zlib') + +log = logging.getLogger(__name__) + +def parse_args(args): + '''Parse command line''' + + parser = ArgumentParser( + description='Measure S3QL write performance, uplink bandwidth and ' + 'compression speed and determine limiting factor.') + + parser.add_authfile() + parser.add_quiet() + parser.add_debug() + parser.add_backend_options() + parser.add_version() + parser.add_storage_url() + parser.add_argument('file', metavar='', type=argparse.FileType(mode='rb'), + help='File to transfer') + parser.add_argument('--threads', metavar='', type=int, default=None, + help='Also include statistics for threads in results.') + + parser.add_cachedir() + return parser.parse_args(args) + + +def main(args=None): + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + # /dev/urandom may be slow, so we cache the data first + log.info('Preparing test data...') + rnd_fh = tempfile.TemporaryFile() + with open('/dev/urandom', 'rb', 0) as src: + copied = 0 + while copied < 50 * 1024 * 1024: + buf = src.read(BUFSIZE) + rnd_fh.write(buf) + copied += len(buf) + + log.info('Measuring throughput to cache...') + backend_dir = tempfile.mkdtemp(prefix='s3ql-benchmark-') + mnt_dir = tempfile.mkdtemp(prefix='s3ql-mnt') + atexit.register(shutil.rmtree, backend_dir) + atexit.register(shutil.rmtree, mnt_dir) + + block_sizes = [ 2**b for b in range(12, 18) ] + for blocksize in block_sizes: + write_time = 0 + size = 50 * 1024 * 1024 + while write_time < 3: + log.debug('Write took %.3g seconds, retrying', write_time) + subprocess.check_call([exec_prefix + 'mkfs.s3ql', '--plain', 'local://%s' % backend_dir, + '--quiet', '--force', '--cachedir', options.cachedir]) + subprocess.check_call([exec_prefix + 'mount.s3ql', '--threads', '1', '--quiet', + '--cachesize', '%d' % (2 * size / 1024), '--log', + '%s/mount.log' % backend_dir, '--cachedir', options.cachedir, + 'local://%s' % backend_dir, mnt_dir]) + try: + size *= 2 + with open('%s/bigfile' % mnt_dir, 'wb', 0) as dst: + rnd_fh.seek(0) + write_time = time.time() + copied = 0 + while copied < size: + buf = rnd_fh.read(blocksize) + if not buf: + rnd_fh.seek(0) + continue + dst.write(buf) + copied += len(buf) + + write_time = time.time() - write_time + os.unlink('%s/bigfile' % mnt_dir) + finally: + subprocess.check_call([exec_prefix + 'umount.s3ql', mnt_dir]) + + fuse_speed = copied / write_time + log.info('Cache throughput with %3d KiB blocks: %d KiB/sec', + blocksize / 1024, fuse_speed / 1024) + + # Upload random data to prevent effects of compression + # on the network layer + log.info('Measuring raw backend throughput..') + try: + backend = get_backend(options, raw=True) + except DanglingStorageURLError as exc: + raise QuietError(str(exc)) from None + + upload_time = 0 + size = 512 * 1024 + while upload_time < 10: + size *= 2 + def do_write(dst): + rnd_fh.seek(0) + stamp = time.time() + copied = 0 + while copied < size: + buf = rnd_fh.read(BUFSIZE) + if not buf: + rnd_fh.seek(0) + continue + dst.write(buf) + copied += len(buf) + return (copied, stamp) + (upload_size, upload_time) = backend.perform_write(do_write, 's3ql_testdata') + upload_time = time.time() - upload_time + backend_speed = upload_size / upload_time + log.info('Backend throughput: %d KiB/sec', backend_speed / 1024) + backend.delete('s3ql_testdata') + + src = options.file + size = os.fstat(options.file.fileno()).st_size + log.info('Test file size: %.2f MiB', (size / 1024 ** 2)) + + in_speed = dict() + out_speed = dict() + for alg in ALGS: + log.info('compressing with %s-6...', alg) + backend = ComprencBackend(b'pass', (alg, 6), Backend('local://' + backend_dir, None, None)) + def do_write(dst): #pylint: disable=E0102 + src.seek(0) + stamp = time.time() + while True: + buf = src.read(BUFSIZE) + if not buf: + break + dst.write(buf) + return (dst, stamp) + (dst_fh, stamp) = backend.perform_write(do_write, 's3ql_testdata') + dt = time.time() - stamp + in_speed[alg] = size / dt + out_speed[alg] = dst_fh.get_obj_size() / dt + log.info('%s compression speed: %d KiB/sec per thread (in)', alg, in_speed[alg] / 1024) + log.info('%s compression speed: %d KiB/sec per thread (out)', alg, out_speed[alg] / 1024) + + print('') + print('With %d KiB blocks, maximum performance for different compression' + % (block_sizes[-1]/1024), 'algorithms and thread counts is:', '', sep='\n') + + threads = set([1,2,4,8]) + cores = os.sysconf('SC_NPROCESSORS_ONLN') + if cores != -1: + threads.add(cores) + if options.threads: + threads.add(options.threads) + + print('%-26s' % 'Threads:', + ('%12d' * len(threads)) % tuple(sorted(threads))) + + for alg in ALGS: + speeds = [] + limits = [] + for t in sorted(threads): + if fuse_speed > t * in_speed[alg]: + limit = 'CPU' + speed = t * in_speed[alg] + else: + limit = 'S3QL/FUSE' + speed = fuse_speed + + if speed / in_speed[alg] * out_speed[alg] > backend_speed: + limit = 'uplink' + speed = backend_speed * in_speed[alg] / out_speed[alg] + + limits.append(limit) + speeds.append(speed / 1024) + + print('%-26s' % ('Max FS throughput (%s):' % alg), + ('%7d KiB/s' * len(threads)) % tuple(speeds)) + print('%-26s' % '..limited by:', + ('%12s' * len(threads)) % tuple(limits)) + + print('') + print('All numbers assume that the test file is representative and that', + 'there are enough processor cores to run all active threads in parallel.', + 'To compensate for network latency, you should use about twice as', + 'many upload threads as indicated by the above table.\n', sep='\n') + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/contrib/clone_fs.py b/contrib/clone_fs.py new file mode 100755 index 0000000..470bcdc --- /dev/null +++ b/contrib/clone_fs.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +''' +clone_fs.py - this file is part of S3QL. + +Clone an S3QL file system from one backend into another, without +recompressing or reencrypting. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +import os +import sys +import tempfile +import time +from queue import Queue, Full as QueueFull + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +from s3ql.logging import logging, setup_logging, QuietError +from s3ql.common import get_backend_factory, AsyncFn, handle_on_return +from s3ql.backends.common import DanglingStorageURLError +from s3ql import BUFSIZE +from s3ql.parse_args import ArgumentParser, storage_url_type + +log = logging.getLogger(__name__) + +def parse_args(args): + '''Parse command line''' + + parser = ArgumentParser( + description='Clone an S3QL file system.') + + parser.add_authfile() + parser.add_quiet() + parser.add_debug() + parser.add_backend_options() + parser.add_version() + + parser.add_argument("src_storage_url", metavar='', + type=storage_url_type, + help='Storage URL of the source backend that contains the file system') + + parser.add_argument("dst_storage_url", metavar='', + type=storage_url_type, + help='Storage URL of the destination backend') + + parser.add_argument("--threads", type=int, default=3, + help='Number of threads to use') + + return parser.parse_args(args) + + +@handle_on_return +def copy_loop(queue, src_backend_factory, dst_backend_factory, on_return): + '''Copy keys arriving in *queue* from *src_backend* to *dst_backend* + + Terminate when None is received. + ''' + + src_backend = on_return.enter_context(src_backend_factory()) + dst_backend = on_return.enter_context(dst_backend_factory()) + tmpfh = on_return.enter_context(tempfile.TemporaryFile()) + while True: + key = queue.get() + if key is None: + break + + log.debug('reading object %s', key) + def do_read(fh): + tmpfh.seek(0) + tmpfh.truncate() + while True: + buf = fh.read(BUFSIZE) + if not buf: + break + tmpfh.write(buf) + return fh.metadata + metadata = src_backend.perform_read(do_read, key) + + log.debug('writing object %s', key) + def do_write(fh): + tmpfh.seek(0) + while True: + buf = tmpfh.read(BUFSIZE) + if not buf: + break + fh.write(buf) + dst_backend.perform_write(do_write, key, metadata) + +def main(args=None): + options = parse_args(args) + setup_logging(options) + + try: + options.storage_url = options.src_storage_url + src_backend_factory = get_backend_factory(options.src_storage_url, + options.backend_options, + options.authfile, raw=True) + + options.storage_url = options.dst_storage_url + dst_backend_factory = get_backend_factory(options.dst_storage_url, + options.backend_options, + options.authfile, raw=True) + except DanglingStorageURLError as exc: + raise QuietError(str(exc)) from None + + queue = Queue(maxsize=options.threads) + threads = [] + for _ in range(options.threads): + t = AsyncFn(copy_loop, queue, src_backend_factory, + dst_backend_factory) + # Don't wait for worker threads, gives deadlock if main thread + # terminates with exception + t.daemon = True + t.start() + threads.append(t) + + with src_backend_factory() as backend: + stamp1 = 0 + for (i, key) in enumerate(backend): + stamp2 = time.time() + if stamp2 - stamp1 > 1: + stamp1 = stamp2 + sys.stdout.write('\rCopied %d objects so far...' % i) + sys.stdout.flush() + + # Terminate early if any thread failed with an exception + for t in threads: + if not t.is_alive(): + t.join_and_raise() + + # Avoid blocking if all threads terminated + while True: + try: + queue.put(key, timeout=1) + except QueueFull: + pass + else: + break + for t in threads: + if not t.is_alive(): + t.join_and_raise() + sys.stdout.write('\n') + + queue.maxsize += len(threads) + for t in threads: + queue.put(None) + + for t in threads: + t.join_and_raise() + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/contrib/expire_backups.1 b/contrib/expire_backups.1 new file mode 100644 index 0000000..1ceb38e --- /dev/null +++ b/contrib/expire_backups.1 @@ -0,0 +1,184 @@ +.\" Man page generated from reStructuredText. +. +.TH "EXPIRE_BACKUPS" "1" "October 28, 2016" "2.21" "S3QL" +.SH NAME +expire_backups \- Intelligently expire old backups +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +expire_backups [options] [ ...] +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DESCRIPTION +.sp +The \fBexpire_backups\fP command intelligently remove old backups that are no +longer needed. +.sp +To define what backups you want to keep for how long, you define a +number of \fIage ranges\fP\&. \fBexpire_backups\fP ensures that you +will have at least one backup in each age range at all times. It will +keep exactly as many backups as are required for that and delete any +backups that become redundant. +.sp +Age ranges are specified by giving a list of range boundaries in terms +of backup cycles. Every time you create a new backup, the existing +backups age by one cycle. +.sp +Example: when \fBexpire_backups\fP is called with the age range +definition \fB1 3 7 14 31\fP, it will guarantee that you always have the +following backups available: +.INDENT 0.0 +.IP 1. 3 +A backup that is 0 to 1 cycles old (i.e, the most recent backup) +.IP 2. 3 +A backup that is 1 to 3 cycles old +.IP 3. 3 +A backup that is 3 to 7 cycles old +.IP 4. 3 +A backup that is 7 to 14 cycles old +.IP 5. 3 +A backup that is 14 to 31 cycles old +.UNINDENT +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +If you do backups in fixed intervals, then one cycle will be +equivalent to the backup interval. The advantage of specifying the +age ranges in terms of backup cycles rather than days or weeks is +that it allows you to gracefully handle irregular backup intervals. +Imagine that for some reason you do not turn on your computer for +one month. Now all your backups are at least a month old, and if you +had specified the above backup strategy in terms of absolute ages, +they would all be deleted! Specifying age ranges in terms of backup +cycles avoids these sort of problems. +.UNINDENT +.UNINDENT +.sp +\fBexpire_backups\fP usage is simple. It requires backups to be +stored in directories of the form \fByear\-month\-day_hour:minute:seconds\fP +(\fBYYYY\-MM\-DD_HH:mm:ss\fP) and works on all backups in the current +directory. So for the above backup strategy, the correct invocation +would be: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +expire_backups.py 1 3 7 14 31 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +When storing your backups on an S3QL file system, you probably want to +specify the \fB\-\-use\-s3qlrm\fP option as well. This tells +\fBexpire_backups\fP to use the \fIs3qlrm\fP command to +delete directories. +.sp +\fBexpire_backups\fP uses a "state file" to keep track which +backups are how many cycles old (since this cannot be inferred from +the dates contained in the directory names). The standard name for +this state file is \fB\&.expire_backups.dat\fP\&. If this file gets +damaged or deleted, \fBexpire_backups\fP no longer knows the ages +of the backups and refuses to work. In this case you can use the +\fB\-\-reconstruct\-state\fP option to try to reconstruct the state +from the backup dates. However, the accuracy of this reconstruction +depends strongly on how rigorous you have been with making backups (it +is only completely correct if the time between subsequent backups has +always been exactly the same), so it\(aqs generally a good idea not to +tamper with the state file. +.SH OPTIONS +.sp +The \fBexpire_backups\fP command accepts the following options: +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.B \-\-quiet +be really quiet +.TP +.BI \-\-debug\-modules \ +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\fB\-\-log\fP option. +.TP +.B \-\-debug +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \fB\-\-log\fP option. +.TP +.B \-\-version +just print program version and exit +.TP +.BI \-\-state \ +File to save state information in (default: +".expire_backups.dat") +.TP +.B \-n +Dry run. Just show which backups would be deleted. +.TP +.B \-\-reconstruct\-state +Try to reconstruct a missing state file from backup +dates. +.TP +.B \-\-use\-s3qlrm +Use \fBs3qlrm\fP command to delete backups. +.UNINDENT +.UNINDENT +.UNINDENT +.SH EXIT CODES +.sp +\fBexpire_backups\fP may terminate with the following exit codes: +.INDENT 0.0 +.TP +.B 0 +Everything went well. +.TP +.B 1 +An unexpected error occured. This may indicate a bug in the +program. +.TP +.B 2 +Invalid command line argument. +.UNINDENT +.SH SEE ALSO +.sp +\fBexpire_backups\fP is shipped as part of S3QL, \fI\%https://bitbucket.org/nikratio/s3ql/\fP\&. +.SH COPYRIGHT +© 2008 Nikolaus Rath +.\" Generated by docutils manpage writer. +. diff --git a/contrib/expire_backups.py b/contrib/expire_backups.py new file mode 100755 index 0000000..34efc02 --- /dev/null +++ b/contrib/expire_backups.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +''' +expire_backups.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +import sys +import os +import re +import textwrap +import shutil +import pickle +from datetime import datetime, timedelta +from collections import defaultdict + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +from s3ql.logging import setup_logging, QuietError, logging +from s3ql.common import thaw_basic_mapping, freeze_basic_mapping +from s3ql.parse_args import ArgumentParser +from s3ql.remove import main as s3qlrm + +log = logging.getLogger(__name__) + +def parse_args(args): + '''Parse command line''' + + parser = ArgumentParser( + description=textwrap.dedent('''\ + ``expire_backups.py`` is a program to intelligently remove old backups + that are no longer needed. + + To define what backups you want to keep for how long, you define a + number of *age ranges*. ``expire_backups`` ensures that you will + have at least one backup in each age range at all times. It will keep + exactly as many backups as are required for that and delete any + backups that become redundant. + + Age ranges are specified by giving a list of range boundaries in terms + of backup cycles. Every time you create a new backup, the existing + backups age by one cycle. + + Please refer to the S3QL documentation for details. + ''')) + + parser.add_quiet() + parser.add_debug() + parser.add_version() + + parser.add_argument('cycles', nargs='+', type=int, metavar='', + help='Age range boundaries in terms of backup cycles') + parser.add_argument('--state', metavar='', type=str, + default='.expire_backups.dat', + # Add quotes around default to prevent groff + # from choking on leading . generated by buggy + # docutils man page generator. + help='File to save state information in (default: "%(default)s")') + parser.add_argument("-n", action="store_true", default=False, + help="Dry run. Just show which backups would be deleted.") + parser.add_argument('--reconstruct-state', action='store_true', default=False, + help='Try to reconstruct a missing state file from backup dates.') + + parser.add_argument("--use-s3qlrm", action="store_true", + help="Use `s3qlrm` command to delete backups.") + + options = parser.parse_args(args) + + if sorted(options.cycles) != options.cycles: + parser.error('Age range boundaries must be in increasing order') + + return options + +def main(args=None): + + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + # Determine available backups + backup_list = set(x for x in os.listdir('.') + if re.match(r'^\d{4}-\d\d-\d\d_\d\d:\d\d:\d\d$', x)) + + if not os.path.exists(options.state) and len(backup_list) > 1: + if not options.reconstruct_state: + raise QuietError('Found more than one backup but no state file! Aborting.') + + log.warning('Trying to reconstruct state file..') + state = upgrade_to_state(backup_list) + if not options.n: + log.info('Saving reconstructed state..') + with open(options.state, 'wb') as fh: + fh.write(freeze_basic_mapping(state)) + elif not os.path.exists(options.state): + log.warning('Creating state file..') + state = dict() + else: + log.info('Reading state...') + with open(options.state, 'rb') as fh: + state = thaw_basic_mapping(fh.read()) + + to_delete = process_backups(backup_list, state, options.cycles) + + for x in to_delete: + log.info('Backup %s is no longer needed, removing...', x) + if not options.n: + if options.use_s3qlrm: + s3qlrm([x]) + else: + shutil.rmtree(x) + + if options.n: + log.info('Dry run, not saving state.') + else: + log.info('Saving state..') + with open(options.state, 'wb') as fh: + fh.write(freeze_basic_mapping(state)) + +def upgrade_to_state(backup_list): + log.info('Several existing backups detected, trying to convert absolute ages to cycles') + + now = datetime.now() + age = dict() + for x in sorted(backup_list): + age[x] = now - datetime.strptime(x, '%Y-%m-%d_%H:%M:%S') + log.info('Backup %s is %s hours old', x, age[x]) + + deltas = [ abs(x - y) for x in age.values() + for y in age.values() if x != y ] + step = min(deltas) + log.info('Assuming backup interval of %s hours', step) + + state = dict() + for x in sorted(age): + state[x] = 0 + while age[x] > timedelta(0): + state[x] += 1 + age[x] -= step + log.info('Backup %s is %d cycles old', x, state[x]) + + log.info('State construction complete.') + return state + +def simulate(args): + + options = parse_args(args) + setup_logging(options) + + state = dict() + backup_list = set() + for i in range(50): + backup_list.add('backup-%2d' % i) + delete = process_backups(backup_list, state, options.cycles) + log.info('Deleting %s', delete) + backup_list -= delete + + log.info('Available backups on day %d:', i) + for x in sorted(backup_list): + log.info(x) + +def process_backups(backup_list, state, cycles): + + # New backups + new_backups = backup_list - set(state) + for x in sorted(new_backups): + log.info('Found new backup %s', x) + for y in state: + state[y] += 1 + state[x] = 0 + + for x in state: + log.debug('Backup %s has age %d', x, state[x]) + + # Missing backups + missing_backups = set(state) - backup_list + for x in missing_backups: + log.warning('backup %s is missing. Did you delete it manually?', x) + del state[x] + + # Ranges + ranges = [ (0, cycles[0]) ] + for i in range(1, len(cycles)): + ranges.append((cycles[i - 1], cycles[i])) + + # Go forward in time to see what backups need to be kept + simstate = dict() + keep = set() + missing = defaultdict(list) + for step in range(max(cycles)): + + log.debug('Considering situation after %d more backups', step) + for x in simstate: + simstate[x] += 1 + log.debug('Backup x now has simulated age %d', simstate[x]) + + # Add the hypothetical backup that has been made "just now" + if step != 0: + simstate[step] = 0 + + for (min_, max_) in ranges: + log.debug('Looking for backup for age range %d to %d', min_, max_) + + # Look in simstate + found = False + for (backup, age) in simstate.items(): + if min_ <= age < max_: + found = True + break + if found: + # backup and age will be defined + #pylint: disable=W0631 + log.debug('Using backup %s (age %d)', backup, age) + continue + + # Look in state + for (backup, age) in state.items(): + age += step + if min_ <= age < max_: + log.info('Keeping backup %s (current age %d) for age range %d to %d%s', + backup, state[backup], min_, max_, + (' in %d cycles' % step) if step else '') + simstate[backup] = age + keep.add(backup) + break + + else: + if step == 0: + log.info('Note: there is currently no backup available ' + 'for age range %d to %d', min_, max_) + else: + missing['%d to %d' % (min_, max_)].append(step) + + for range_ in sorted(missing): + log.info('Note: there will be no backup for age range %s ' + 'in (forthcoming) cycle(s): %s', + range_, format_list(missing[range_])) + + to_delete = set(state) - keep + for x in to_delete: + del state[x] + + return to_delete + + +def format_list(l): + if not l: + return '' + l = l[:] + + # Append bogus end element + l.append(l[-1] + 2) + + range_start = l.pop(0) + cur = range_start + res = list() + for n in l: + if n == cur + 1: + pass + elif range_start == cur: + res.append('%d' % cur) + elif range_start == cur - 1: + res.append('%d' % range_start) + res.append('%d' % cur) + else: + res.append('%d-%d' % (range_start, cur)) + + if n != cur + 1: + range_start = n + cur = n + + if len(res) > 1: + return ('%s and %s' % (', '.join(res[:-1]), res[-1])) + else: + return ', '.join(res) + + +if __name__ == '__main__': + #simulate(sys.argv[1:]) + main(sys.argv[1:]) diff --git a/contrib/fsck_db.py b/contrib/fsck_db.py new file mode 100755 index 0000000..662e220 --- /dev/null +++ b/contrib/fsck_db.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +''' +fsck_db.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from argparse import ArgumentTypeError +import os +import sys + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +from s3ql.logging import logging, setup_logging +from s3ql.fsck import ROFsck +from s3ql.parse_args import ArgumentParser + +log = logging.getLogger(__name__) + +def parse_args(args): + + parser = ArgumentParser( + description="Checks S3QL file system metadata") + + parser.add_log('~/.s3ql/fsck_db.log') + parser.add_debug() + parser.add_quiet() + parser.add_version() + + def db_path(s): + s = os.path.splitext(s)[0] + if not os.path.exists(s + '.db'): + raise ArgumentTypeError('Unable to read %s.db' % s) + if not os.path.exists(s + '.params'): + raise ArgumentTypeError('Unable to read %s.params' % s) + return s + + parser.add_argument("path", metavar='', type=db_path, + help='Database to be checked') + + options = parser.parse_args(args) + + return options + +def main(args=None): + + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + fsck = ROFsck(options.path) + fsck.check() + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/contrib/pcp.1 b/contrib/pcp.1 new file mode 100644 index 0000000..5f61702 --- /dev/null +++ b/contrib/pcp.1 @@ -0,0 +1,107 @@ +.\" Man page generated from reStructuredText. +. +.TH "PCP" "1" "October 28, 2016" "2.21" "S3QL" +.SH NAME +pcp \- Recursive, parallel copy of directory trees +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pcp [options] [ ...] +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DESCRIPTION +.sp +The \fBpcp\fP command is a is a wrapper that starts several +\fBsync\fP processes to copy directory trees in parallel. This is +allows much better copying performance on file system that have +relatively high latency when retrieving individual files like S3QL. +.sp +\fBNote\fP: Using this program only improves performance when copying +\fIfrom\fP an S3QL file system. When copying \fIto\fP an S3QL file system, +using \fBpcp\fP is more likely to \fIdecrease\fP performance. +.SH OPTIONS +.sp +The \fBpcp\fP command accepts the following options: +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.B \-\-quiet +be really quiet +.TP +.BI \-\-debug\-modules \ +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\fB\-\-log\fP option. +.TP +.B \-\-debug +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \fB\-\-log\fP option. +.TP +.B \-\-version +just print program version and exit +.TP +.B \-a +Pass \-aHAX option to rsync. +.TP +.BI \-\-processes \ +Number of rsync processes to use (default: 10). +.UNINDENT +.UNINDENT +.UNINDENT +.SH EXIT CODES +.sp +\fBpcp\fP may terminate with the following exit codes: +.INDENT 0.0 +.TP +.B 0 +Everything went well. +.TP +.B 1 +An unexpected error occured. This may indicate a bug in the +program. +.TP +.B 2 +Invalid command line argument. +.UNINDENT +.SH SEE ALSO +.sp +\fBpcp\fP is shipped as part of S3QL, \fI\%https://bitbucket.org/nikratio/s3ql/\fP\&. +.SH COPYRIGHT +© 2008 Nikolaus Rath +.\" Generated by docutils manpage writer. +. diff --git a/contrib/pcp.py b/contrib/pcp.py new file mode 100755 index 0000000..f18699a --- /dev/null +++ b/contrib/pcp.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +''' +pcp.py - this file is part of S3QL. + +Parallel, recursive copy of directory trees. + +--- +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +import sys +import os +import subprocess + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +from s3ql.logging import logging, setup_logging +from s3ql.parse_args import ArgumentParser + +log = logging.getLogger(__name__) + +def parse_args(args): + '''Parse command line''' + + parser = ArgumentParser( + description='Recursively copy source(s) to destination using multiple ' + 'parallel rsync processes.') + + parser.add_quiet() + parser.add_debug() + parser.add_version() + + parser.add_argument("-a", action="store_true", + help='Pass -aHAX option to rsync.') + parser.add_argument("--processes", action="store", type=int, metavar='', + default=10, + help='Number of rsync processes to use (default: %(default)s).') + + parser.add_argument('source', metavar='', nargs='+', + help='Directories to copy') + parser.add_argument('dest', metavar='', + help="Target directory") + + options = parser.parse_args(args) + options.pps = options.source + [ options.dest ] + + return options + +def main(args=None): + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + pool = ('abcdefghijklmnopqrstuvwxyz', + 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', + '0123456789') + steps = [ len(x) / (options.processes - 1) for x in pool ] + prefixes = list() + for i in range(options.processes - 1): + parts = [ x[int(i * y):int((i + 1) * y)] for (x, y) in zip(pool, steps) ] + prefixes.append(''.join(parts)) + + filters = [ '-! [%s]*' % x for x in prefixes ] + + # Catch all + filters.append('- [%s]*' % ''.join(prefixes)) + + rsync_args = [ 'rsync', '-f', '+ */' ] + if not options.quiet: + rsync_args.append('--out-format') + rsync_args.append('%n%L') + if options.a: + rsync_args.append('-aHAX') + + processes = list() + for filter_ in filters: + cmd = rsync_args + [ '-f', filter_ ] + options.pps + log.debug('Calling %s', cmd) + processes.append(subprocess.Popen(cmd)) + + if all([ c.wait() == 0 for c in processes]): + sys.exit(0) + else: + sys.exit(1) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/contrib/remove_objects.py b/contrib/remove_objects.py new file mode 100755 index 0000000..167fefd --- /dev/null +++ b/contrib/remove_objects.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +''' +remove_objects.py - this file is part of S3QL. + +Copyright © 2014 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +import os +import sys +import argparse +import atexit + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +from s3ql.logging import logging, setup_logging +from s3ql.common import get_backend +from s3ql.parse_args import ArgumentParser, storage_url_type + +log = logging.getLogger(__name__) + +def parse_args(args): + '''Parse command line''' + + parser = ArgumentParser(description='Batch remove objects from an S3QL backend') + + parser.add_authfile() + parser.add_quiet() + parser.add_debug() + parser.add_backend_options() + parser.add_version() + + parser.add_argument("storage_url", type=storage_url_type, + help='Storage URL of the backend to delete from') + + parser.add_argument("file", type=argparse.FileType(mode='r', encoding='utf-8'), + help='File with newline separated object keys to delete') + + return parser.parse_args(args) + + +def main(args=None): + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + backend = get_backend(options, raw=True) + atexit.register(backend.close) + + for line in options.file: + key = line.rstrip() + log.info('Deleting %s', key) + backend.delete(key) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/contrib/s3ql_backup.sh b/contrib/s3ql_backup.sh new file mode 100755 index 0000000..c032527 --- /dev/null +++ b/contrib/s3ql_backup.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Abort entire script if any command fails +set -e + +# Backup destination (storage url) +storage_url="s3://my_backup_bucket" + +# Recover cache if e.g. system was shut down while fs was mounted +fsck.s3ql --batch "$storage_url" + +# Create a temporary mountpoint and mount file system +mountpoint="/tmp/s3ql_backup_$$" +mkdir "$mountpoint" +mount.s3ql "$storage_url" "$mountpoint" + +# Make sure the file system is unmounted when we are done +# Note that this overwrites the earlier trap, so we +# also delete the lock file here. +trap "cd /; umount.s3ql '$mountpoint'; rmdir '$mountpoint'" EXIT + +# Figure out the most recent backup +cd "$mountpoint" +last_backup=`python < + +This work can be distributed under the terms of the GNU GPLv3. +''' + +# Data can be restored with: +#from s3ql.metadata import restore_metadata +#from s3ql.database import Connection +#restore_metadata(open('s3ql_metadata.dat', 'rb+'), 'data.sqlite') + +import os +import shutil +import sys +import tempfile +import hashlib + +# We are running from the S3QL source directory, make sure +# that we use modules from this directory +basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + +from s3ql.logging import logging, setup_logging, QuietError +from s3ql import CURRENT_FS_REV +from s3ql.common import get_backend_cachedir, load_params +from s3ql.database import Connection +from s3ql.metadata import dump_metadata +from s3ql.parse_args import ArgumentParser + +log = logging.getLogger(__name__) + +DBNAME = 's3ql_metadata.dat' + +def parse_args(args): + + parser = ArgumentParser( + description="Create metadata copy where all file- and directory names, " + "and extended attribute names and values have been scrambled. " + "This is intended to preserve privacy when a metadata copy " + "needs to be provided to the developers for debugging.") + + parser.add_debug() + parser.add_quiet() + parser.add_version() + parser.add_cachedir() + parser.add_storage_url() + + options = parser.parse_args(args) + + return options + +def main(args=None): + + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + # Check for cached metadata + cachepath = get_backend_cachedir(options.storage_url, options.cachedir) + if not os.path.exists(cachepath + '.params'): + raise QuietError("No local metadata found.") + + param = load_params(cachepath) + + # Check revision + if param['revision'] < CURRENT_FS_REV: + raise QuietError('File system revision too old.') + elif param['revision'] > CURRENT_FS_REV: + raise QuietError('File system revision too new.') + + if os.path.exists(DBNAME): + raise QuietError('%s exists, aborting.' % DBNAME) + + log.info('Copying database...') + dst = tempfile.NamedTemporaryFile() + with open(cachepath + '.db', 'rb') as src: + shutil.copyfileobj(src, dst) + dst.flush() + db = Connection(dst.name) + + log.info('Scrambling...') + md5 = lambda x: hashlib.md5(x).hexdigest() + for (id_, name) in db.query('SELECT id, name FROM names'): + db.execute('UPDATE names SET name=? WHERE id=?', + (md5(name), id_)) + + for (id_, name) in db.query('SELECT inode, target FROM symlink_targets'): + db.execute('UPDATE symlink_targets SET target=? WHERE inode=?', + (md5(name), id_)) + + for (id_, name) in db.query('SELECT rowid, value FROM ext_attributes'): + db.execute('UPDATE ext_attributes SET value=? WHERE rowid=?', + (md5(name), id_)) + + log.info('Saving...') + with open(DBNAME, 'wb+') as fh: + dump_metadata(db, fh) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/doc/latex/Makefile b/doc/latex/Makefile new file mode 100644 index 0000000..6b87ad8 --- /dev/null +++ b/doc/latex/Makefile @@ -0,0 +1,66 @@ +# Makefile for Sphinx LaTeX output + +ALLDOCS = $(basename $(wildcard *.tex)) +ALLPDF = $(addsuffix .pdf,$(ALLDOCS)) +ALLDVI = $(addsuffix .dvi,$(ALLDOCS)) + +# Prefix for archive names +ARCHIVEPRREFIX = +# Additional LaTeX options +LATEXOPTS = + +all: $(ALLPDF) +all-pdf: $(ALLPDF) +all-dvi: $(ALLDVI) +all-ps: all-dvi + for f in *.dvi; do dvips $$f; done + +all-pdf-ja: + for f in *.pdf *.png *.gif *.jpg *.jpeg; do extractbb $$f; done + for f in *.tex; do platex -kanji=utf8 $(LATEXOPTS) $$f; done + for f in *.tex; do platex -kanji=utf8 $(LATEXOPTS) $$f; done + for f in *.tex; do platex -kanji=utf8 $(LATEXOPTS) $$f; done + -for f in *.idx; do mendex -U -f -d "`basename $$f .idx`.dic" -s python.ist $$f; done + for f in *.tex; do platex -kanji=utf8 $(LATEXOPTS) $$f; done + for f in *.tex; do platex -kanji=utf8 $(LATEXOPTS) $$f; done + for f in *.dvi; do dvipdfmx $$f; done + +zip: all-$(FMT) + mkdir $(ARCHIVEPREFIX)docs-$(FMT) + cp $(ALLPDF) $(ARCHIVEPREFIX)docs-$(FMT) + zip -q -r -9 $(ARCHIVEPREFIX)docs-$(FMT).zip $(ARCHIVEPREFIX)docs-$(FMT) + rm -r $(ARCHIVEPREFIX)docs-$(FMT) + +tar: all-$(FMT) + mkdir $(ARCHIVEPREFIX)docs-$(FMT) + cp $(ALLPDF) $(ARCHIVEPREFIX)docs-$(FMT) + tar cf $(ARCHIVEPREFIX)docs-$(FMT).tar $(ARCHIVEPREFIX)docs-$(FMT) + rm -r $(ARCHIVEPREFIX)docs-$(FMT) + +bz2: tar + bzip2 -9 -k $(ARCHIVEPREFIX)docs-$(FMT).tar + +# The number of LaTeX runs is quite conservative, but I don't expect it +# to get run often, so the little extra time won't hurt. +%.dvi: %.tex + latex $(LATEXOPTS) '$<' + latex $(LATEXOPTS) '$<' + latex $(LATEXOPTS) '$<' + -makeindex -s python.ist '$(basename $<).idx' + latex $(LATEXOPTS) '$<' + latex $(LATEXOPTS) '$<' + +%.pdf: %.tex + pdflatex $(LATEXOPTS) '$<' + pdflatex $(LATEXOPTS) '$<' + pdflatex $(LATEXOPTS) '$<' + -makeindex -s python.ist '$(basename $<).idx' + pdflatex $(LATEXOPTS) '$<' + pdflatex $(LATEXOPTS) '$<' + +clean: + rm -f *.dvi *.log *.ind *.aux *.toc *.syn *.idx *.out *.ilg *.pla + +.PHONY: all all-pdf all-dvi all-ps clean +.PHONY: all-pdf-ja + diff --git a/doc/latex/fncychap.sty b/doc/latex/fncychap.sty new file mode 100644 index 0000000..9a56c04 --- /dev/null +++ b/doc/latex/fncychap.sty @@ -0,0 +1,683 @@ +%%% Copyright Ulf A. Lindgren +%%% +%%% Note Premission is granted to modify this file under +%%% the condition that it is saved using another +%%% file and package name. +%%% +%%% Revision 1.1 (1997) +%%% +%%% Jan. 8th Modified package name base date option +%%% Jan. 22th Modified FmN and FmTi for error in book.cls +%%% \MakeUppercase{#}->{\MakeUppercase#} +%%% Apr. 6th Modified Lenny option to prevent undesired +%%% skip of line. +%%% Nov. 8th Fixed \@chapapp for AMS +%%% +%%% Revision 1.2 (1998) +%%% +%%% Feb. 11th Fixed appendix problem related to Bjarne +%%% Aug. 11th Fixed problem related to 11pt and 12pt +%%% suggested by Tomas Lundberg. THANKS! +%%% +%%% Revision 1.3 (2004) +%%% Sep. 20th problem with frontmatter, mainmatter and +%%% backmatter, pointed out by Lapo Mori +%%% +%%% Revision 1.31 (2004) +%%% Sep. 21th problem with the Rejne definition streched text +%%% caused ugly gaps in the vrule aligned with the title +%%% text. Kindly pointed out to me by Hendri Adriaens +%%% +%%% Revision 1.32 (2005) +%%% Jun. 23th compatibility problem with the KOMA class 'scrbook.cls' +%%% a remedy is a redefinition of '\@schapter' in +%%% line with that used in KOMA. The problem was pointed +%%% out to me by Mikkel Holm Olsen +%%% +%%% Revision 1.33 (2005) +%%% Aug. 9th misspelled ``TWELV'' corrected, the error was pointed +%%% out to me by George Pearson +%%% +%%% Revision 1.34 (2007) +%%% Added an alternative to Lenny provided by Peter +%%% Osborne (2005-11-28) +%%% Corrected front, main and back matter, based on input +%%% from Bas van Gils (2006-04-24) +%%% Jul. 30th Added Bjornstrup option provided by Jean-Marc +%%% Francois (2007-01-05). +%%% Reverted to \MakeUppercase{#} see rev 1.1, solved +%%% problem with MakeUppercase and MakeLowercase pointed +%%% out by Marco Feuerstein (2007-06-06) + + +%%% Last modified Jul. 2007 + +\NeedsTeXFormat{LaTeX2e}[1995/12/01] +\ProvidesPackage{fncychap} + [2007/07/30 v1.34 + LaTeX package (Revised chapters)] + +%%%% For conditional inclusion of color +\newif\ifusecolor +\usecolorfalse + + + +%%%% DEFINITION OF Chapapp variables +\newcommand{\CNV}{\huge\bfseries} +\newcommand{\ChNameVar}[1]{\renewcommand{\CNV}{#1}} + + +%%%% DEFINITION OF TheChapter variables +\newcommand{\CNoV}{\huge\bfseries} +\newcommand{\ChNumVar}[1]{\renewcommand{\CNoV}{#1}} + +\newif\ifUCN +\UCNfalse +\newif\ifLCN +\LCNfalse +\def\ChNameLowerCase{\LCNtrue\UCNfalse} +\def\ChNameUpperCase{\UCNtrue\LCNfalse} +\def\ChNameAsIs{\UCNfalse\LCNfalse} + +%%%%% Fix for AMSBook 971008 + +\@ifundefined{@chapapp}{\let\@chapapp\chaptername}{} + + +%%%%% Fix for Bjarne and appendix 980211 + +\newif\ifinapp +\inappfalse +\renewcommand\appendix{\par + \setcounter{chapter}{0}% + \setcounter{section}{0}% + \inapptrue% + \renewcommand\@chapapp{\appendixname}% + \renewcommand\thechapter{\@Alph\c@chapter}} + +%%%%% Fix for frontmatter, mainmatter, and backmatter 040920 + +\@ifundefined{@mainmatter}{\newif\if@mainmatter \@mainmattertrue}{} + +%%%%% + + + +\newcommand{\FmN}[1]{% +\ifUCN + {\MakeUppercase{#1}}\LCNfalse +\else + \ifLCN + {\MakeLowercase{#1}}\UCNfalse + \else #1 + \fi +\fi} + + +%%%% DEFINITION OF Title variables +\newcommand{\CTV}{\Huge\bfseries} +\newcommand{\ChTitleVar}[1]{\renewcommand{\CTV}{#1}} + +%%%% DEFINITION OF the basic rule width +\newlength{\RW} +\setlength{\RW}{1pt} +\newcommand{\ChRuleWidth}[1]{\setlength{\RW}{#1}} + +\newif\ifUCT +\UCTfalse +\newif\ifLCT +\LCTfalse +\def\ChTitleLowerCase{\LCTtrue\UCTfalse} +\def\ChTitleUpperCase{\UCTtrue\LCTfalse} +\def\ChTitleAsIs{\UCTfalse\LCTfalse} +\newcommand{\FmTi}[1]{% +\ifUCT + {\MakeUppercase{#1}}\LCTfalse +\else + \ifLCT + {\MakeLowercase{#1}}\UCTfalse + \else {#1} + \fi +\fi} + + + +\newlength{\mylen} +\newlength{\myhi} +\newlength{\px} +\newlength{\py} +\newlength{\pyy} +\newlength{\pxx} + + +\def\mghrulefill#1{\leavevmode\leaders\hrule\@height #1\hfill\kern\z@} + +\newcommand{\DOCH}{% + \CNV\FmN{\@chapapp}\space \CNoV\thechapter + \par\nobreak + \vskip 20\p@ + } +\newcommand{\DOTI}[1]{% + \CTV\FmTi{#1}\par\nobreak + \vskip 40\p@ + } +\newcommand{\DOTIS}[1]{% + \CTV\FmTi{#1}\par\nobreak + \vskip 40\p@ + } + +%%%%%% SONNY DEF + +\DeclareOption{Sonny}{% + \ChNameVar{\Large\sf} + \ChNumVar{\Huge} + \ChTitleVar{\Large\sf} + \ChRuleWidth{0.5pt} + \ChNameUpperCase + \renewcommand{\DOCH}{% + \raggedleft + \CNV\FmN{\@chapapp}\space \CNoV\thechapter + \par\nobreak + \vskip 40\p@} + \renewcommand{\DOTI}[1]{% + \CTV\raggedleft\mghrulefill{\RW}\par\nobreak + \vskip 5\p@ + \CTV\FmTi{#1}\par\nobreak + \mghrulefill{\RW}\par\nobreak + \vskip 40\p@} + \renewcommand{\DOTIS}[1]{% + \CTV\raggedleft\mghrulefill{\RW}\par\nobreak + \vskip 5\p@ + \CTV\FmTi{#1}\par\nobreak + \mghrulefill{\RW}\par\nobreak + \vskip 40\p@} +} + +%%%%%% LENNY DEF + +\DeclareOption{Lenny}{% + + \ChNameVar{\fontsize{14}{16}\usefont{OT1}{phv}{m}{n}\selectfont} + \ChNumVar{\fontsize{60}{62}\usefont{OT1}{ptm}{m}{n}\selectfont} + \ChTitleVar{\Huge\bfseries\rm} + \ChRuleWidth{1pt} + \renewcommand{\DOCH}{% + \settowidth{\px}{\CNV\FmN{\@chapapp}} + \addtolength{\px}{2pt} + \settoheight{\py}{\CNV\FmN{\@chapapp}} + \addtolength{\py}{1pt} + + \settowidth{\mylen}{\CNV\FmN{\@chapapp}\space\CNoV\thechapter} + \addtolength{\mylen}{1pt} + \settowidth{\pxx}{\CNoV\thechapter} + \addtolength{\pxx}{-1pt} + + \settoheight{\pyy}{\CNoV\thechapter} + \addtolength{\pyy}{-2pt} + \setlength{\myhi}{\pyy} + \addtolength{\myhi}{-1\py} + \par + \parbox[b]{\textwidth}{% + \rule[\py]{\RW}{\myhi}% + \hskip -\RW% + \rule[\pyy]{\px}{\RW}% + \hskip -\px% + \raggedright% + \CNV\FmN{\@chapapp}\space\CNoV\thechapter% + \hskip1pt% + \mghrulefill{\RW}% + \rule{\RW}{\pyy}\par\nobreak% + \vskip -\baselineskip% + \vskip -\pyy% + \hskip \mylen% + \mghrulefill{\RW}\par\nobreak% + \vskip \pyy}% + \vskip 20\p@} + + + \renewcommand{\DOTI}[1]{% + \raggedright + \CTV\FmTi{#1}\par\nobreak + \vskip 40\p@} + + \renewcommand{\DOTIS}[1]{% + \raggedright + \CTV\FmTi{#1}\par\nobreak + \vskip 40\p@} + } + +%%%%%% Peter Osbornes' version of LENNY DEF + +\DeclareOption{PetersLenny}{% + +% five new lengths +\newlength{\bl} % bottom left : orig \space +\setlength{\bl}{6pt} +\newcommand{\BL}[1]{\setlength{\bl}{#1}} +\newlength{\br} % bottom right : orig 1pt +\setlength{\br}{1pt} +\newcommand{\BR}[1]{\setlength{\br}{#1}} +\newlength{\tl} % top left : orig 2pt +\setlength{\tl}{2pt} +\newcommand{\TL}[1]{\setlength{\tl}{#1}} +\newlength{\trr} % top right :orig 1pt +\setlength{\trr}{1pt} +\newcommand{\TR}[1]{\setlength{\trr}{#1}} +\newlength{\blrule} % top right :orig 1pt +\setlength{\trr}{0pt} +\newcommand{\BLrule}[1]{\setlength{\blrule}{#1}} + + + \ChNameVar{\fontsize{14}{16}\usefont{OT1}{phv}{m}{n}\selectfont} + \ChNumVar{\fontsize{60}{62}\usefont{OT1}{ptm}{m}{n}\selectfont} + \ChTitleVar{\Huge\bfseries\rm} + \ChRuleWidth{1pt} +\renewcommand{\DOCH}{% + + +%%%%%%% tweaks for 1--9 and A--Z +\ifcase\c@chapter\relax% +\or\BL{-3pt}\TL{-4pt}\BR{0pt}\TR{-6pt}%1 +\or\BL{0pt}\TL{-4pt}\BR{2pt}\TR{-4pt}%2 +\or\BL{0pt}\TL{-4pt}\BR{2pt}\TR{-4pt}%3 +\or\BL{0pt}\TL{5pt}\BR{2pt}\TR{-4pt}%4 +\or\BL{0pt}\TL{3pt}\BR{2pt}\TR{-4pt}%5 +\or\BL{-1pt}\TL{0pt}\BR{2pt}\TR{-2pt}%6 +\or\BL{0pt}\TL{-3pt}\BR{2pt}\TR{-2pt}%7 +\or\BL{0pt}\TL{-3pt}\BR{2pt}\TR{-2pt}%8 +\or\BL{0pt}\TL{-3pt}\BR{-4pt}\TR{-2pt}%9 +\or\BL{-3pt}\TL{-3pt}\BR{2pt}\TR{-7pt}%10 +\or\BL{-6pt}\TL{-6pt}\BR{0pt}\TR{-9pt}%11 +\or\BL{-6pt}\TL{-6pt}\BR{2pt}\TR{-7pt}%12 +\or\BL{-5pt}\TL{-5pt}\BR{0pt}\TR{-9pt}%13 +\or\BL{-6pt}\TL{-6pt}\BR{0pt}\TR{-9pt}%14 +\or\BL{-3pt}\TL{-3pt}\BR{3pt}\TR{-6pt}%15 +\or\BL{-3pt}\TL{-3pt}\BR{3pt}\TR{-6pt}%16 +\or\BL{-5pt}\TL{-3pt}\BR{-8pt}\TR{-6pt}%17 +\or\BL{-5pt}\TL{-5pt}\BR{0pt}\TR{-9pt}%18 +\or\BL{-3pt}\TL{-3pt}\BR{-6pt}\TR{-9pt}%19 +\or\BL{0pt}\TL{0pt}\BR{0pt}\TR{-5pt}%20 +\fi + +\ifinapp\ifcase\c@chapter\relax% +\or\BL{0pt}\TL{14pt}\BR{5pt}\TR{-19pt}%A +\or\BL{0pt}\TL{-5pt}\BR{-3pt}\TR{-8pt}%B +\or\BL{-3pt}\TL{-2pt}\BR{1pt}\TR{-6pt}\BLrule{0pt}%C +\or\BL{0pt}\TL{-5pt}\BR{-3pt}\TR{-8pt}\BLrule{0pt}%D +\or\BL{0pt}\TL{-5pt}\BR{2pt}\TR{-3pt}%E +\or\BL{0pt}\TL{-5pt}\BR{-10pt}\TR{-1pt}%F +\or\BL{-3pt}\TL{0pt}\BR{0pt}\TR{-7pt}%G +\or\BL{0pt}\TL{-5pt}\BR{3pt}\TR{-1pt}%H +\or\BL{0pt}\TL{-5pt}\BR{3pt}\TR{-1pt}%I +\or\BL{2pt}\TL{0pt}\BR{-3pt}\TR{1pt}%J +\or\BL{0pt}\TL{-5pt}\BR{3pt}\TR{-1pt}%K +\or\BL{0pt}\TL{-5pt}\BR{2pt}\TR{-19pt}%L +\or\BL{0pt}\TL{-5pt}\BR{3pt}\TR{-1pt}%M +\or\BL{0pt}\TL{-5pt}\BR{-2pt}\TR{-1pt}%N +\or\BL{-3pt}\TL{-2pt}\BR{-3pt}\TR{-11pt}%O +\or\BL{0pt}\TL{-5pt}\BR{-9pt}\TR{-3pt}%P +\or\BL{-3pt}\TL{-2pt}\BR{-3pt}\TR{-11pt}%Q +\or\BL{0pt}\TL{-5pt}\BR{4pt}\TR{-8pt}%R +\or\BL{-2pt}\TL{-2pt}\BR{-2pt}\TR{-7pt}%S +\or\BL{-3pt}\TL{0pt}\BR{-5pt}\TR{4pt}\BLrule{8pt}%T +\or\BL{-7pt}\TL{-11pt}\BR{-5pt}\TR{-7pt}\BLrule{0pt}%U +\or\BL{-14pt}\TL{-5pt}\BR{-14pt}\TR{-1pt}\BLrule{14pt}%V +\or\BL{-10pt}\TL{-9pt}\BR{-13pt}\TR{-3pt}\BLrule{7pt}%W +\or\BL{0pt}\TL{-5pt}\BR{3pt}\TR{-1pt}\BLrule{0pt}%X +\or\BL{-6pt}\TL{-4pt}\BR{-7pt}\TR{1pt}\BLrule{7pt}%Y +\or\BL{0pt}\TL{-5pt}\BR{3pt}\TR{-1pt}\BLrule{0pt}%Z +\fi\fi +%%%%%%% + \settowidth{\px}{\CNV\FmN{\@chapapp}} + \addtolength{\px}{\tl} %MOD change 2pt to \tl + \settoheight{\py}{\CNV\FmN{\@chapapp}} + \addtolength{\py}{1pt} + + \settowidth{\mylen}{\CNV\FmN{\@chapapp}\space\CNoV\thechapter} + \addtolength{\mylen}{\trr}% MOD change 1pt to \tr + \settowidth{\pxx}{\CNoV\thechapter} + \addtolength{\pxx}{-1pt} + + \settoheight{\pyy}{\CNoV\thechapter} + \addtolength{\pyy}{-2pt} + \setlength{\myhi}{\pyy} + \addtolength{\myhi}{-1\py} + \par + \parbox[b]{\textwidth}{% + \rule[\py]{\RW}{\myhi}% + \hskip -\RW% + \rule[\pyy]{\px}{\RW}% + \hskip -\px% + \raggedright% + \CNV\FmN{\@chapapp}\rule{\blrule}{\RW}\hskip\bl\CNoV\thechapter%MOD +% \CNV\FmN{\@chapapp}\space\CNoV\thechapter %ORIGINAL + \hskip\br% %MOD 1pt to \br + \mghrulefill{\RW}% + \rule{\RW}{\pyy}\par\nobreak% + \vskip -\baselineskip% + \vskip -\pyy% + \hskip \mylen% + \mghrulefill{\RW}\par\nobreak% + \vskip \pyy}% + \vskip 20\p@} + + + \renewcommand{\DOTI}[1]{% + \raggedright + \CTV\FmTi{#1}\par\nobreak + \vskip 40\p@} + + \renewcommand{\DOTIS}[1]{% + \raggedright + \CTV\FmTi{#1}\par\nobreak + \vskip 40\p@} + } + + +% + + +%%%%%% BJORNSTRUP DEF + +\DeclareOption{Bjornstrup}{% + \usecolortrue + % pzc (Zapf Chancelery) is nice. ppl (Palatino) is cool too. + \ChNumVar{\fontsize{76}{80}\usefont{OT1}{pzc}{m}{n}\selectfont} + \ChTitleVar{\raggedleft\Large\sffamily\bfseries} + + \setlength{\myhi}{10pt} % Space between grey box border and text + \setlength{\mylen}{\textwidth} + \addtolength{\mylen}{-2\myhi} + \renewcommand{\DOCH}{% + \settowidth{\py}{\CNoV\thechapter} + \addtolength{\py}{-10pt} % Amount of space by which the +% % number is shifted right + \fboxsep=0pt% + \colorbox[gray]{.85}{\rule{0pt}{40pt}\parbox[b]{\textwidth}{\hfill}}% + \kern-\py\raise20pt% + \hbox{\color[gray]{.5}\CNoV\thechapter}\\% + } + + \renewcommand{\DOTI}[1]{% + \nointerlineskip\raggedright% + \fboxsep=\myhi% + \vskip-1ex% + \colorbox[gray]{.85}{\parbox[t]{\mylen}{\CTV\FmTi{#1}}}\par\nobreak% + \vskip 40\p@% + } + + \renewcommand{\DOTIS}[1]{% + \fboxsep=0pt + \colorbox[gray]{.85}{\rule{0pt}{40pt}\parbox[b]{\textwidth}{\hfill}}\\% + \nointerlineskip\raggedright% + \fboxsep=\myhi% + \colorbox[gray]{.85}{\parbox[t]{\mylen}{\CTV\FmTi{#1}}}\par\nobreak% + \vskip 40\p@% + } +} + + +%%%%%%% GLENN DEF + + +\DeclareOption{Glenn}{% + \ChNameVar{\bfseries\Large\sf} + \ChNumVar{\Huge} + \ChTitleVar{\bfseries\Large\rm} + \ChRuleWidth{1pt} + \ChNameUpperCase + \ChTitleUpperCase + \renewcommand{\DOCH}{% + \settoheight{\myhi}{\CTV\FmTi{Test}} + \setlength{\py}{\baselineskip} + \addtolength{\py}{\RW} + \addtolength{\py}{\myhi} + \setlength{\pyy}{\py} + \addtolength{\pyy}{-1\RW} + + \raggedright + \CNV\FmN{\@chapapp}\space\CNoV\thechapter + \hskip 3pt\mghrulefill{\RW}\rule[-1\pyy]{2\RW}{\py}\par\nobreak} + + \renewcommand{\DOTI}[1]{% + \addtolength{\pyy}{-4pt} + \settoheight{\myhi}{\CTV\FmTi{#1}} + \addtolength{\myhi}{\py} + \addtolength{\myhi}{-1\RW} + \vskip -1\pyy + \rule{2\RW}{\myhi}\mghrulefill{\RW}\hskip 2pt + \raggedleft\CTV\FmTi{#1}\par\nobreak + \vskip 80\p@} + +\newlength{\backskip} + \renewcommand{\DOTIS}[1]{% +% \setlength{\py}{10pt} +% \setlength{\pyy}{\py} +% \addtolength{\pyy}{\RW} +% \setlength{\myhi}{\baselineskip} +% \addtolength{\myhi}{\pyy} +% \mghrulefill{\RW}\rule[-1\py]{2\RW}{\pyy}\par\nobreak +% \addtolength{}{} +%\vskip -1\baselineskip +% \rule{2\RW}{\myhi}\mghrulefill{\RW}\hskip 2pt +% \raggedleft\CTV\FmTi{#1}\par\nobreak +% \vskip 60\p@} +%% Fix suggested by Tomas Lundberg + \setlength{\py}{25pt} % eller vad man vill + \setlength{\pyy}{\py} + \setlength{\backskip}{\py} + \addtolength{\backskip}{2pt} + \addtolength{\pyy}{\RW} + \setlength{\myhi}{\baselineskip} + \addtolength{\myhi}{\pyy} + \mghrulefill{\RW}\rule[-1\py]{2\RW}{\pyy}\par\nobreak + \vskip -1\backskip + \rule{2\RW}{\myhi}\mghrulefill{\RW}\hskip 3pt % + \raggedleft\CTV\FmTi{#1}\par\nobreak + \vskip 40\p@} + } + +%%%%%%% CONNY DEF + +\DeclareOption{Conny}{% + \ChNameUpperCase + \ChTitleUpperCase + \ChNameVar{\centering\Huge\rm\bfseries} + \ChNumVar{\Huge} + \ChTitleVar{\centering\Huge\rm} + \ChRuleWidth{2pt} + + \renewcommand{\DOCH}{% + \mghrulefill{3\RW}\par\nobreak + \vskip -0.5\baselineskip + \mghrulefill{\RW}\par\nobreak + \CNV\FmN{\@chapapp}\space \CNoV\thechapter + \par\nobreak + \vskip -0.5\baselineskip + } + \renewcommand{\DOTI}[1]{% + \mghrulefill{\RW}\par\nobreak + \CTV\FmTi{#1}\par\nobreak + \vskip 60\p@ + } + \renewcommand{\DOTIS}[1]{% + \mghrulefill{\RW}\par\nobreak + \CTV\FmTi{#1}\par\nobreak + \vskip 60\p@ + } + } + +%%%%%%% REJNE DEF + +\DeclareOption{Rejne}{% + + \ChNameUpperCase + \ChTitleUpperCase + \ChNameVar{\centering\Large\rm} + \ChNumVar{\Huge} + \ChTitleVar{\centering\Huge\rm} + \ChRuleWidth{1pt} + \renewcommand{\DOCH}{% + \settoheight{\py}{\CNoV\thechapter} + \parskip=0pt plus 1pt % Set parskip to default, just in case v1.31 + \addtolength{\py}{-1pt} + \CNV\FmN{\@chapapp}\par\nobreak + \vskip 20\p@ + \setlength{\myhi}{2\baselineskip} + \setlength{\px}{\myhi} + \addtolength{\px}{-1\RW} + \rule[-1\px]{\RW}{\myhi}\mghrulefill{\RW}\hskip + 10pt\raisebox{-0.5\py}{\CNoV\thechapter}\hskip 10pt\mghrulefill{\RW}\rule[-1\px]{\RW}{\myhi}\par\nobreak + \vskip -3\p@% Added -2pt vskip to correct for streched text v1.31 + } + \renewcommand{\DOTI}[1]{% + \setlength{\mylen}{\textwidth} + \parskip=0pt plus 1pt % Set parskip to default, just in case v1.31 + \addtolength{\mylen}{-2\RW} + {\vrule width\RW}\parbox{\mylen}{\CTV\FmTi{#1}}{\vrule width\RW}\par\nobreak% + \vskip -3pt\rule{\RW}{2\baselineskip}\mghrulefill{\RW}\rule{\RW}{2\baselineskip}% + \vskip 60\p@% Added -2pt in vskip to correct for streched text v1.31 + } + \renewcommand{\DOTIS}[1]{% + \setlength{\py}{\fboxrule} + \setlength{\fboxrule}{\RW} + \setlength{\mylen}{\textwidth} + \addtolength{\mylen}{-2\RW} + \fbox{\parbox{\mylen}{\vskip 2\baselineskip\CTV\FmTi{#1}\par\nobreak\vskip \baselineskip}} + \setlength{\fboxrule}{\py} + \vskip 60\p@ + } + } + + +%%%%%%% BJARNE DEF + +\DeclareOption{Bjarne}{% + \ChNameUpperCase + \ChTitleUpperCase + \ChNameVar{\raggedleft\normalsize\rm} + \ChNumVar{\raggedleft \bfseries\Large} + \ChTitleVar{\raggedleft \Large\rm} + \ChRuleWidth{1pt} + + +%% Note thechapter -> c@chapter fix appendix bug +%% Fixed misspelled 12 + + \newcounter{AlphaCnt} + \newcounter{AlphaDecCnt} + \newcommand{\AlphaNo}{% + \ifcase\number\theAlphaCnt + \ifnum\c@chapter=0 + ZERO\else{}\fi + \or ONE\or TWO\or THREE\or FOUR\or FIVE + \or SIX\or SEVEN\or EIGHT\or NINE\or TEN + \or ELEVEN\or TWELVE\or THIRTEEN\or FOURTEEN\or FIFTEEN + \or SIXTEEN\or SEVENTEEN\or EIGHTEEN\or NINETEEN\fi +} + + \newcommand{\AlphaDecNo}{% + \setcounter{AlphaDecCnt}{0} + \@whilenum\number\theAlphaCnt>0\do + {\addtocounter{AlphaCnt}{-10} + \addtocounter{AlphaDecCnt}{1}} + \ifnum\number\theAlphaCnt=0 + \else + \addtocounter{AlphaDecCnt}{-1} + \addtocounter{AlphaCnt}{10} + \fi + + + \ifcase\number\theAlphaDecCnt\or TEN\or TWENTY\or THIRTY\or + FORTY\or FIFTY\or SIXTY\or SEVENTY\or EIGHTY\or NINETY\fi + } + \newcommand{\TheAlphaChapter}{% + + \ifinapp + \thechapter + \else + \setcounter{AlphaCnt}{\c@chapter} + \ifnum\c@chapter<20 + \AlphaNo + \else + \AlphaDecNo\AlphaNo + \fi + \fi + } + \renewcommand{\DOCH}{% + \mghrulefill{\RW}\par\nobreak + \CNV\FmN{\@chapapp}\par\nobreak + \CNoV\TheAlphaChapter\par\nobreak + \vskip -1\baselineskip\vskip 5pt\mghrulefill{\RW}\par\nobreak + \vskip 20\p@ + } + \renewcommand{\DOTI}[1]{% + \CTV\FmTi{#1}\par\nobreak + \vskip 40\p@ + } + \renewcommand{\DOTIS}[1]{% + \CTV\FmTi{#1}\par\nobreak + \vskip 40\p@ + } +} + +\DeclareOption*{% + \PackageWarning{fancychapter}{unknown style option} + } + +\ProcessOptions* \relax + +\ifusecolor + \RequirePackage{color} +\fi +\def\@makechapterhead#1{% + \vspace*{50\p@}% + {\parindent \z@ \raggedright \normalfont + \ifnum \c@secnumdepth >\m@ne + \if@mainmatter%%%%% Fix for frontmatter, mainmatter, and backmatter 040920 + \DOCH + \fi + \fi + \interlinepenalty\@M + \if@mainmatter%%%%% Fix for frontmatter, mainmatter, and backmatter 060424 + \DOTI{#1}% + \else% + \DOTIS{#1}% + \fi + }} + + +%%% Begin: To avoid problem with scrbook.cls (fncychap version 1.32) + +%%OUT: +%\def\@schapter#1{\if@twocolumn +% \@topnewpage[\@makeschapterhead{#1}]% +% \else +% \@makeschapterhead{#1}% +% \@afterheading +% \fi} + +%%IN: +\def\@schapter#1{% +\if@twocolumn% + \@makeschapterhead{#1}% +\else% + \@makeschapterhead{#1}% + \@afterheading% +\fi} + +%%% End: To avoid problem with scrbook.cls (fncychap version 1.32) + +\def\@makeschapterhead#1{% + \vspace*{50\p@}% + {\parindent \z@ \raggedright + \normalfont + \interlinepenalty\@M + \DOTIS{#1} + \vskip 40\p@ + }} + +\endinput + + diff --git a/doc/latex/manual.aux b/doc/latex/manual.aux new file mode 100644 index 0000000..79554fe --- /dev/null +++ b/doc/latex/manual.aux @@ -0,0 +1,447 @@ +\relax +\providecommand\hyper@newdestlabel[2]{} +\providecommand\HyperFirstAtBeginDocument{\AtBeginDocument} +\HyperFirstAtBeginDocument{\ifx\hyper@anchor\@undefined +\global\let\oldcontentsline\contentsline +\gdef\contentsline#1#2#3#4{\oldcontentsline{#1}{#2}{#3}} +\global\let\oldnewlabel\newlabel +\gdef\newlabel#1#2{\newlabelxx{#1}#2} +\gdef\newlabelxx#1#2#3#4#5#6{\oldnewlabel{#1}{{#2}{#3}}} +\AtEndDocument{\ifx\hyper@anchor\@undefined +\let\contentsline\oldcontentsline +\let\newlabel\oldnewlabel +\fi} +\fi} +\global\let\hyper@last\relax +\gdef\HyperFirstAtBeginDocument#1{#1} +\providecommand\HyField@AuxAddToFields[1]{} +\providecommand\HyField@AuxAddToCoFields[2]{} +\select@language{english} +\@writefile{toc}{\select@language{english}} +\@writefile{lof}{\select@language{english}} +\@writefile{lot}{\select@language{english}} +\newlabel{index::doc}{{}{1}{}{section*.2}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {1}About S3QL}{1}{chapter.1}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{about::doc}{{1}{1}{About S3QL}{chapter.1}{}} +\newlabel{about:s3ql-user-s-guide}{{1}{1}{About S3QL}{chapter.1}{}} +\newlabel{about:about-s3ql}{{1}{1}{About S3QL}{chapter.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {1.1}Features}{1}{section.1.1}} +\newlabel{about:openstack}{{1.1}{1}{Features}{section.1.1}{}} +\newlabel{about:features}{{1.1}{1}{Features}{section.1.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {1.2}Development Status}{2}{section.1.2}} +\newlabel{about:development-status}{{1.2}{2}{Development Status}{section.1.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {1.3}Supported Platforms}{2}{section.1.3}} +\newlabel{about:supported-platforms}{{1.3}{2}{Supported Platforms}{section.1.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {1.4}Contributing}{2}{section.1.4}} +\newlabel{about:contributing}{{1.4}{2}{Contributing}{section.1.4}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {2}Installation}{3}{chapter.2}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{installation::doc}{{2}{3}{Installation}{chapter.2}{}} +\newlabel{installation:installation}{{2}{3}{Installation}{chapter.2}{}} +\newlabel{installation:github}{{2}{3}{Installation}{chapter.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {2.1}Dependencies}{3}{section.2.1}} +\newlabel{installation:dependencies}{{2.1}{3}{Dependencies}{section.2.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {2.2}Installing S3QL}{4}{section.2.2}} +\newlabel{installation:inst-s3ql}{{2.2}{4}{Installing S3QL}{section.2.2}{}} +\newlabel{installation:installing-s3ql}{{2.2}{4}{Installing S3QL}{section.2.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {2.3}Development Version}{4}{section.2.3}} +\newlabel{installation:development-version}{{2.3}{4}{Development Version}{section.2.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {2.4}Running tests requiring remote servers}{4}{section.2.4}} +\newlabel{installation:running-tests-requiring-remote-servers}{{2.4}{4}{Running tests requiring remote servers}{section.2.4}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {3}Storage Backends}{7}{chapter.3}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{backends::doc}{{3}{7}{Storage Backends}{chapter.3}{}} +\newlabel{backends:storage-backends}{{3}{7}{Storage Backends}{chapter.3}{}} +\newlabel{backends:id1}{{3}{7}{Storage Backends}{chapter.3}{}} +\newlabel{backends:sphinx}{{3}{7}{Storage Backends}{chapter.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {3.1}Google Storage}{7}{section.3.1}} +\newlabel{backends:google-storage}{{3.1}{7}{Google Storage}{section.3.1}{}} +\newlabel{backends:cmdoption-gs_backend-arg-no-ssl}{{3.1}{7}{Google Storage}{section*.3}{}} +\newlabel{backends:cmdoption-gs_backend-arg-ssl-ca-path}{{3.1}{8}{Google Storage}{section*.4}{}} +\newlabel{backends:cmdoption-gs_backend-arg-tcp-timeout}{{3.1}{8}{Google Storage}{section*.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {3.2}Amazon S3}{8}{section.3.2}} +\newlabel{backends:amazon-s3}{{3.2}{8}{Amazon S3}{section.3.2}{}} +\newlabel{backends:google-storage-manager}{{3.2}{8}{Amazon S3}{section.3.2}{}} +\newlabel{backends:cmdoption-s3_backend-arg-no-ssl}{{3.2}{8}{Amazon S3}{section*.6}{}} +\newlabel{backends:cmdoption-s3_backend-arg-ssl-ca-path}{{3.2}{8}{Amazon S3}{section*.7}{}} +\newlabel{backends:cmdoption-s3_backend-arg-tcp-timeout}{{3.2}{8}{Amazon S3}{section*.8}{}} +\newlabel{backends:cmdoption-s3_backend-arg-sse}{{3.2}{8}{Amazon S3}{section*.9}{}} +\newlabel{backends:cmdoption-s3_backend-arg-ia}{{3.2}{8}{Amazon S3}{section*.10}{}} +\newlabel{backends:cmdoption-s3_backend-arg-rrs}{{3.2}{8}{Amazon S3}{section*.11}{}} +\@writefile{toc}{\contentsline {section}{\numberline {3.3}OpenStack/Swift}{9}{section.3.3}} +\newlabel{backends:openstack-swift}{{3.3}{9}{OpenStack/Swift}{section.3.3}{}} +\newlabel{backends:openstack-backend}{{3.3}{9}{OpenStack/Swift}{section.3.3}{}} +\newlabel{backends:cmdoption-swift_backend-arg-no-ssl}{{3.3}{9}{OpenStack/Swift}{section*.12}{}} +\newlabel{backends:cmdoption-swift_backend-arg-ssl-ca-path}{{3.3}{9}{OpenStack/Swift}{section*.13}{}} +\newlabel{backends:cmdoption-swift_backend-arg-tcp-timeout}{{3.3}{9}{OpenStack/Swift}{section*.14}{}} +\newlabel{backends:cmdoption-swift_backend-arg-disable-expect100}{{3.3}{9}{OpenStack/Swift}{section*.15}{}} +\@writefile{toc}{\contentsline {section}{\numberline {3.4}Rackspace CloudFiles}{9}{section.3.4}} +\newlabel{backends:swift}{{3.4}{9}{Rackspace CloudFiles}{section.3.4}{}} +\newlabel{backends:rackspace-cloudfiles}{{3.4}{9}{Rackspace CloudFiles}{section.3.4}{}} +\@writefile{toc}{\contentsline {section}{\numberline {3.5}S3 compatible}{10}{section.3.5}} +\newlabel{backends:rackspace}{{3.5}{10}{S3 compatible}{section.3.5}{}} +\newlabel{backends:s3-compatible}{{3.5}{10}{S3 compatible}{section.3.5}{}} +\newlabel{backends:cmdoption-s3c_backend-arg-no-ssl}{{3.5}{10}{S3 compatible}{section*.16}{}} +\newlabel{backends:cmdoption-s3c_backend-arg-ssl-ca-path}{{3.5}{10}{S3 compatible}{section*.17}{}} +\newlabel{backends:cmdoption-s3c_backend-arg-tcp-timeout}{{3.5}{10}{S3 compatible}{section*.18}{}} +\newlabel{backends:cmdoption-s3c_backend-arg-disable-expect100}{{3.5}{10}{S3 compatible}{section*.19}{}} +\newlabel{backends:cmdoption-s3c_backend-arg-dumb-copy}{{3.5}{10}{S3 compatible}{section*.20}{}} +\@writefile{toc}{\contentsline {section}{\numberline {3.6}Local}{10}{section.3.6}} +\newlabel{backends:id6}{{3.6}{10}{Local}{section.3.6}{}} +\newlabel{backends:local}{{3.6}{10}{Local}{section.3.6}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {4}Important Rules to Avoid Losing Data}{13}{chapter.4}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{durability::doc}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} +\newlabel{durability:sshfs}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} +\newlabel{durability:durability}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} +\newlabel{durability:important-rules-to-avoid-losing-data}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} +\@writefile{toc}{\contentsline {section}{\numberline {4.1}Rules in a Nutshell}{13}{section.4.1}} +\newlabel{durability:rules-in-a-nutshell}{{4.1}{13}{Rules in a Nutshell}{section.4.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {4.2}Consistency Window List}{14}{section.4.2}} +\newlabel{durability:consistency-window-list}{{4.2}{14}{Consistency Window List}{section.4.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {4.3}Data Consistency}{14}{section.4.3}} +\newlabel{durability:data-consistency}{{4.3}{14}{Data Consistency}{section.4.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {4.4}Data Durability}{15}{section.4.4}} +\newlabel{durability:backend-reliability}{{4.4}{15}{Data Durability}{section.4.4}{}} +\newlabel{durability:data-durability}{{4.4}{15}{Data Durability}{section.4.4}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {5}File System Creation}{17}{chapter.5}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{mkfs::doc}{{5}{17}{File System Creation}{chapter.5}{}} +\newlabel{mkfs:file-system-creation}{{5}{17}{File System Creation}{chapter.5}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {6}Managing File Systems}{19}{chapter.6}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{adm::doc}{{6}{19}{Managing File Systems}{chapter.6}{}} +\newlabel{adm:managing-file-systems}{{6}{19}{Managing File Systems}{chapter.6}{}} +\@writefile{toc}{\contentsline {section}{\numberline {6.1}Changing the Passphrase}{19}{section.6.1}} +\newlabel{adm:changing-the-passphrase}{{6.1}{19}{Changing the Passphrase}{section.6.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {6.2}Upgrading the file system}{20}{section.6.2}} +\newlabel{adm:upgrading-the-file-system}{{6.2}{20}{Upgrading the file system}{section.6.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {6.3}Deleting a file system}{20}{section.6.3}} +\newlabel{adm:deleting-a-file-system}{{6.3}{20}{Deleting a file system}{section.6.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {6.4}Restoring Metadata Backups}{20}{section.6.4}} +\newlabel{adm:restoring-metadata-backups}{{6.4}{20}{Restoring Metadata Backups}{section.6.4}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {7}Mounting}{21}{chapter.7}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{mount::doc}{{7}{21}{Mounting}{chapter.7}{}} +\newlabel{mount:mounting}{{7}{21}{Mounting}{chapter.7}{}} +\@writefile{toc}{\contentsline {section}{\numberline {7.1}Permission Checking}{22}{section.7.1}} +\newlabel{mount:permission-checking}{{7.1}{22}{Permission Checking}{section.7.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {7.2}Compression Algorithms}{22}{section.7.2}} +\newlabel{mount:compression-algorithms}{{7.2}{22}{Compression Algorithms}{section.7.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {7.3}Notes about Caching}{22}{section.7.3}} +\newlabel{mount:notes-about-caching}{{7.3}{22}{Notes about Caching}{section.7.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {7.3.1}Maximum Number of Cache Entries}{23}{subsection.7.3.1}} +\newlabel{mount:maximum-number-of-cache-entries}{{7.3.1}{23}{Maximum Number of Cache Entries}{subsection.7.3.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {7.3.2}Cache Flushing and Expiration}{23}{subsection.7.3.2}} +\newlabel{mount:cache-flushing-and-expiration}{{7.3.2}{23}{Cache Flushing and Expiration}{subsection.7.3.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {7.4}Failure Modes}{23}{section.7.4}} +\newlabel{mount:failure-modes}{{7.4}{23}{Failure Modes}{section.7.4}{}} +\@writefile{toc}{\contentsline {section}{\numberline {7.5}Automatic Mounting}{23}{section.7.5}} +\newlabel{mount:logcheck}{{7.5}{23}{Automatic Mounting}{section.7.5}{}} +\newlabel{mount:automatic-mounting}{{7.5}{23}{Automatic Mounting}{section.7.5}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {8}Advanced S3QL Features}{25}{chapter.8}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{special::doc}{{8}{25}{Advanced S3QL Features}{chapter.8}{}} +\newlabel{special:advanced-s3ql-features}{{8}{25}{Advanced S3QL Features}{chapter.8}{}} +\@writefile{toc}{\contentsline {section}{\numberline {8.1}Snapshotting and Copy-on-Write}{25}{section.8.1}} +\newlabel{special:s3qlcp}{{8.1}{25}{Snapshotting and Copy-on-Write}{section.8.1}{}} +\newlabel{special:snapshotting-and-copy-on-write}{{8.1}{25}{Snapshotting and Copy-on-Write}{section.8.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {8.1.1}Snapshotting vs Hardlinking}{25}{subsection.8.1.1}} +\newlabel{special:snapshotting-vs-hardlinking}{{8.1.1}{25}{Snapshotting vs Hardlinking}{subsection.8.1.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {8.2}Getting Statistics}{26}{section.8.2}} +\newlabel{special:s3qlstat}{{8.2}{26}{Getting Statistics}{section.8.2}{}} +\newlabel{special:getting-statistics}{{8.2}{26}{Getting Statistics}{section.8.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {8.3}Immutable Trees}{26}{section.8.3}} +\newlabel{special:s3qllock}{{8.3}{26}{Immutable Trees}{section.8.3}{}} +\newlabel{special:immutable-trees}{{8.3}{26}{Immutable Trees}{section.8.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {8.4}Fast Recursive Removal}{27}{section.8.4}} +\newlabel{special:s3qlrm}{{8.4}{27}{Fast Recursive Removal}{section.8.4}{}} +\newlabel{special:fast-recursive-removal}{{8.4}{27}{Fast Recursive Removal}{section.8.4}{}} +\@writefile{toc}{\contentsline {section}{\numberline {8.5}Runtime Configuration}{27}{section.8.5}} +\newlabel{special:runtime-configuration}{{8.5}{27}{Runtime Configuration}{section.8.5}{}} +\newlabel{special:s3qlctrl}{{8.5}{27}{Runtime Configuration}{section.8.5}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {9}Unmounting}{29}{chapter.9}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{umount::doc}{{9}{29}{Unmounting}{chapter.9}{}} +\newlabel{umount:unmounting}{{9}{29}{Unmounting}{chapter.9}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {10}Checking for Errors}{31}{chapter.10}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{fsck::doc}{{10}{31}{Checking for Errors}{chapter.10}{}} +\newlabel{fsck:checking-for-errors}{{10}{31}{Checking for Errors}{chapter.10}{}} +\@writefile{toc}{\contentsline {section}{\numberline {10.1}Checking and repairing internal file system errors}{31}{section.10.1}} +\newlabel{fsck:checking-and-repairing-internal-file-system-errors}{{10.1}{31}{Checking and repairing internal file system errors}{section.10.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {10.2}Detecting and handling backend data corruption}{32}{section.10.2}} +\newlabel{fsck:detecting-and-handling-backend-data-corruption}{{10.2}{32}{Detecting and handling backend data corruption}{section.10.2}{}} +\newlabel{fsck:s3ql-verify}{{10.2}{32}{Detecting and handling backend data corruption}{section.10.2}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {11}Storing Authentication Information}{33}{chapter.11}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{authinfo::doc}{{11}{33}{Storing Authentication Information}{chapter.11}{}} +\newlabel{authinfo:authinfo}{{11}{33}{Storing Authentication Information}{chapter.11}{}} +\newlabel{authinfo:storing-authentication-information}{{11}{33}{Storing Authentication Information}{chapter.11}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {12}Contributed Programs}{35}{chapter.12}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{contrib::doc}{{12}{35}{Contributed Programs}{chapter.12}{}} +\newlabel{contrib:contributed-programs}{{12}{35}{Contributed Programs}{chapter.12}{}} +\@writefile{toc}{\contentsline {section}{\numberline {12.1}benchmark.py}{35}{section.12.1}} +\newlabel{contrib:benchmark-py}{{12.1}{35}{benchmark.py}{section.12.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {12.2}clone\_fs.py}{35}{section.12.2}} +\newlabel{contrib:clone-fs-py}{{12.2}{35}{clone\_fs.py}{section.12.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {12.3}pcp.py}{35}{section.12.3}} +\newlabel{contrib:pcp}{{12.3}{35}{pcp.py}{section.12.3}{}} +\newlabel{contrib:pcp-py}{{12.3}{35}{pcp.py}{section.12.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {12.4}s3ql\_backup.sh}{35}{section.12.4}} +\newlabel{contrib:s3ql-backup-sh}{{12.4}{35}{s3ql\_backup.sh}{section.12.4}{}} +\@writefile{toc}{\contentsline {section}{\numberline {12.5}expire\_backups.py}{36}{section.12.5}} +\newlabel{contrib:expire-backups-py}{{12.5}{36}{expire\_backups.py}{section.12.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {12.6}remove\_objects.py}{37}{section.12.6}} +\newlabel{contrib:remove-objects}{{12.6}{37}{remove\_objects.py}{section.12.6}{}} +\newlabel{contrib:remove-objects-py}{{12.6}{37}{remove\_objects.py}{section.12.6}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {13}Tips \& Tricks}{39}{chapter.13}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{tips::doc}{{13}{39}{Tips \& Tricks}{chapter.13}{}} +\newlabel{tips:tips-tricks}{{13}{39}{Tips \& Tricks}{chapter.13}{}} +\@writefile{toc}{\contentsline {section}{\numberline {13.1}SSH Backend}{39}{section.13.1}} +\newlabel{tips:ssh-tipp}{{13.1}{39}{SSH Backend}{section.13.1}{}} +\newlabel{tips:ssh-backend}{{13.1}{39}{SSH Backend}{section.13.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {13.2}Permanently mounted backup file system}{39}{section.13.2}} +\newlabel{tips:permanently-mounted-backup-file-system}{{13.2}{39}{Permanently mounted backup file system}{section.13.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {13.3}Improving copy performance}{39}{section.13.3}} +\newlabel{tips:copy-performance}{{13.3}{39}{Improving copy performance}{section.13.3}{}} +\newlabel{tips:improving-copy-performance}{{13.3}{39}{Improving copy performance}{section.13.3}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {14}Known Issues}{41}{chapter.14}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{issues::doc}{{14}{41}{Known Issues}{chapter.14}{}} +\newlabel{issues:known-issues}{{14}{41}{Known Issues}{chapter.14}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {15}Manpages}{43}{chapter.15}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{man/index::doc}{{15}{43}{Manpages}{chapter.15}{}} +\newlabel{man/index:manpages}{{15}{43}{Manpages}{chapter.15}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.1}The \textbf {mkfs.s3ql} command}{43}{section.15.1}} +\newlabel{man/mkfs::doc}{{15.1}{43}{The \textbf {mkfs.s3ql} command}{section.15.1}{}} +\newlabel{man/mkfs:the-command-command}{{15.1}{43}{The \textbf {mkfs.s3ql} command}{section.15.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.1.1}Synopsis}{43}{subsection.15.1.1}} +\newlabel{man/mkfs:synopsis}{{15.1.1}{43}{Synopsis}{subsection.15.1.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.1.2}Description}{43}{subsection.15.1.2}} +\newlabel{man/mkfs:description}{{15.1.2}{43}{Description}{subsection.15.1.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.1.3}Options}{43}{subsection.15.1.3}} +\newlabel{man/mkfs:options}{{15.1.3}{43}{Options}{subsection.15.1.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.1.4}Exit Codes}{44}{subsection.15.1.4}} +\newlabel{man/mkfs:exit-codes}{{15.1.4}{44}{Exit Codes}{subsection.15.1.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.1.5}See Also}{44}{subsection.15.1.5}} +\newlabel{man/mkfs:see-also}{{15.1.5}{44}{See Also}{subsection.15.1.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.2}The \textbf {s3qladm} command}{44}{section.15.2}} +\newlabel{man/adm::doc}{{15.2}{44}{The \textbf {s3qladm} command}{section.15.2}{}} +\newlabel{man/adm:the-command-command}{{15.2}{44}{The \textbf {s3qladm} command}{section.15.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.1}Synopsis}{44}{subsection.15.2.1}} +\newlabel{man/adm:synopsis}{{15.2.1}{44}{Synopsis}{subsection.15.2.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.2}Description}{45}{subsection.15.2.2}} +\newlabel{man/adm:description}{{15.2.2}{45}{Description}{subsection.15.2.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.3}Options}{45}{subsection.15.2.3}} +\newlabel{man/adm:options}{{15.2.3}{45}{Options}{subsection.15.2.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.4}Actions}{45}{subsection.15.2.4}} +\newlabel{man/adm:actions}{{15.2.4}{45}{Actions}{subsection.15.2.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.5}Exit Codes}{45}{subsection.15.2.5}} +\newlabel{man/adm:exit-codes}{{15.2.5}{45}{Exit Codes}{subsection.15.2.5}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.2.6}See Also}{46}{subsection.15.2.6}} +\newlabel{man/adm:see-also}{{15.2.6}{46}{See Also}{subsection.15.2.6}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.3}The \textbf {mount.s3ql} command}{46}{section.15.3}} +\newlabel{man/mount::doc}{{15.3}{46}{The \textbf {mount.s3ql} command}{section.15.3}{}} +\newlabel{man/mount:the-command-command}{{15.3}{46}{The \textbf {mount.s3ql} command}{section.15.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.3.1}Synopsis}{46}{subsection.15.3.1}} +\newlabel{man/mount:synopsis}{{15.3.1}{46}{Synopsis}{subsection.15.3.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.3.2}Description}{46}{subsection.15.3.2}} +\newlabel{man/mount:description}{{15.3.2}{46}{Description}{subsection.15.3.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.3.3}Options}{46}{subsection.15.3.3}} +\newlabel{man/mount:options}{{15.3.3}{46}{Options}{subsection.15.3.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.3.4}Exit Codes}{47}{subsection.15.3.4}} +\newlabel{man/mount:exit-codes}{{15.3.4}{47}{Exit Codes}{subsection.15.3.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.3.5}See Also}{48}{subsection.15.3.5}} +\newlabel{man/mount:see-also}{{15.3.5}{48}{See Also}{subsection.15.3.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.4}The \textbf {s3qlstat} command}{48}{section.15.4}} +\newlabel{man/stat::doc}{{15.4}{48}{The \textbf {s3qlstat} command}{section.15.4}{}} +\newlabel{man/stat:the-command-command}{{15.4}{48}{The \textbf {s3qlstat} command}{section.15.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.4.1}Synopsis}{48}{subsection.15.4.1}} +\newlabel{man/stat:synopsis}{{15.4.1}{48}{Synopsis}{subsection.15.4.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.4.2}Description}{48}{subsection.15.4.2}} +\newlabel{man/stat:description}{{15.4.2}{48}{Description}{subsection.15.4.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.4.3}Options}{49}{subsection.15.4.3}} +\newlabel{man/stat:options}{{15.4.3}{49}{Options}{subsection.15.4.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.4.4}Exit Codes}{49}{subsection.15.4.4}} +\newlabel{man/stat:exit-codes}{{15.4.4}{49}{Exit Codes}{subsection.15.4.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.4.5}See Also}{49}{subsection.15.4.5}} +\newlabel{man/stat:see-also}{{15.4.5}{49}{See Also}{subsection.15.4.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.5}The \textbf {s3qlctrl} command}{49}{section.15.5}} +\newlabel{man/ctrl::doc}{{15.5}{49}{The \textbf {s3qlctrl} command}{section.15.5}{}} +\newlabel{man/ctrl:the-command-command}{{15.5}{49}{The \textbf {s3qlctrl} command}{section.15.5}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.5.1}Synopsis}{49}{subsection.15.5.1}} +\newlabel{man/ctrl:synopsis}{{15.5.1}{49}{Synopsis}{subsection.15.5.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.5.2}Description}{49}{subsection.15.5.2}} +\newlabel{man/ctrl:description}{{15.5.2}{49}{Description}{subsection.15.5.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.5.3}Options}{50}{subsection.15.5.3}} +\newlabel{man/ctrl:options}{{15.5.3}{50}{Options}{subsection.15.5.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.5.4}Exit Codes}{50}{subsection.15.5.4}} +\newlabel{man/ctrl:exit-codes}{{15.5.4}{50}{Exit Codes}{subsection.15.5.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.5.5}See Also}{50}{subsection.15.5.5}} +\newlabel{man/ctrl:see-also}{{15.5.5}{50}{See Also}{subsection.15.5.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.6}The \textbf {s3qlcp} command}{50}{section.15.6}} +\newlabel{man/cp::doc}{{15.6}{50}{The \textbf {s3qlcp} command}{section.15.6}{}} +\newlabel{man/cp:the-command-command}{{15.6}{50}{The \textbf {s3qlcp} command}{section.15.6}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.6.1}Synopsis}{50}{subsection.15.6.1}} +\newlabel{man/cp:synopsis}{{15.6.1}{50}{Synopsis}{subsection.15.6.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.6.2}Description}{51}{subsection.15.6.2}} +\newlabel{man/cp:description}{{15.6.2}{51}{Description}{subsection.15.6.2}{}} +\@writefile{toc}{\contentsline {subsubsection}{Snapshotting vs Hardlinking}{51}{subsubsection*.21}} +\newlabel{man/cp:snapshotting-vs-hardlinking}{{15.6.2}{51}{Snapshotting vs Hardlinking}{subsubsection*.21}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.6.3}Options}{51}{subsection.15.6.3}} +\newlabel{man/cp:options}{{15.6.3}{51}{Options}{subsection.15.6.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.6.4}Exit Codes}{51}{subsection.15.6.4}} +\newlabel{man/cp:exit-codes}{{15.6.4}{51}{Exit Codes}{subsection.15.6.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.6.5}See Also}{52}{subsection.15.6.5}} +\newlabel{man/cp:see-also}{{15.6.5}{52}{See Also}{subsection.15.6.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.7}The \textbf {s3qlrm} command}{52}{section.15.7}} +\newlabel{man/rm::doc}{{15.7}{52}{The \textbf {s3qlrm} command}{section.15.7}{}} +\newlabel{man/rm:the-command-command}{{15.7}{52}{The \textbf {s3qlrm} command}{section.15.7}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.7.1}Synopsis}{52}{subsection.15.7.1}} +\newlabel{man/rm:synopsis}{{15.7.1}{52}{Synopsis}{subsection.15.7.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.7.2}Description}{52}{subsection.15.7.2}} +\newlabel{man/rm:description}{{15.7.2}{52}{Description}{subsection.15.7.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.7.3}Options}{52}{subsection.15.7.3}} +\newlabel{man/rm:options}{{15.7.3}{52}{Options}{subsection.15.7.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.7.4}Exit Codes}{52}{subsection.15.7.4}} +\newlabel{man/rm:exit-codes}{{15.7.4}{52}{Exit Codes}{subsection.15.7.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.7.5}See Also}{53}{subsection.15.7.5}} +\newlabel{man/rm:see-also}{{15.7.5}{53}{See Also}{subsection.15.7.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.8}The \textbf {s3qllock} command}{53}{section.15.8}} +\newlabel{man/lock::doc}{{15.8}{53}{The \textbf {s3qllock} command}{section.15.8}{}} +\newlabel{man/lock:the-command-command}{{15.8}{53}{The \textbf {s3qllock} command}{section.15.8}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.1}Synopsis}{53}{subsection.15.8.1}} +\newlabel{man/lock:synopsis}{{15.8.1}{53}{Synopsis}{subsection.15.8.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.2}Description}{53}{subsection.15.8.2}} +\newlabel{man/lock:description}{{15.8.2}{53}{Description}{subsection.15.8.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.3}Rationale}{53}{subsection.15.8.3}} +\newlabel{man/lock:rationale}{{15.8.3}{53}{Rationale}{subsection.15.8.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.4}Options}{53}{subsection.15.8.4}} +\newlabel{man/lock:options}{{15.8.4}{53}{Options}{subsection.15.8.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.5}Exit Codes}{54}{subsection.15.8.5}} +\newlabel{man/lock:exit-codes}{{15.8.5}{54}{Exit Codes}{subsection.15.8.5}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.8.6}See Also}{54}{subsection.15.8.6}} +\newlabel{man/lock:see-also}{{15.8.6}{54}{See Also}{subsection.15.8.6}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.9}The \textbf {umount.s3ql} command}{54}{section.15.9}} +\newlabel{man/umount::doc}{{15.9}{54}{The \textbf {umount.s3ql} command}{section.15.9}{}} +\newlabel{man/umount:the-command-command}{{15.9}{54}{The \textbf {umount.s3ql} command}{section.15.9}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.9.1}Synopsis}{54}{subsection.15.9.1}} +\newlabel{man/umount:synopsis}{{15.9.1}{54}{Synopsis}{subsection.15.9.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.9.2}Description}{54}{subsection.15.9.2}} +\newlabel{man/umount:description}{{15.9.2}{54}{Description}{subsection.15.9.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.9.3}Options}{54}{subsection.15.9.3}} +\newlabel{man/umount:options}{{15.9.3}{54}{Options}{subsection.15.9.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.9.4}Exit Codes}{55}{subsection.15.9.4}} +\newlabel{man/umount:exit-codes}{{15.9.4}{55}{Exit Codes}{subsection.15.9.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.9.5}See Also}{55}{subsection.15.9.5}} +\newlabel{man/umount:see-also}{{15.9.5}{55}{See Also}{subsection.15.9.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.10}The \textbf {fsck.s3ql} command}{55}{section.15.10}} +\newlabel{man/fsck::doc}{{15.10}{55}{The \textbf {fsck.s3ql} command}{section.15.10}{}} +\newlabel{man/fsck:the-command-command}{{15.10}{55}{The \textbf {fsck.s3ql} command}{section.15.10}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.1}Synopsis}{55}{subsection.15.10.1}} +\newlabel{man/fsck:synopsis}{{15.10.1}{55}{Synopsis}{subsection.15.10.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.2}Description}{55}{subsection.15.10.2}} +\newlabel{man/fsck:description}{{15.10.2}{55}{Description}{subsection.15.10.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.3}Options}{55}{subsection.15.10.3}} +\newlabel{man/fsck:options}{{15.10.3}{55}{Options}{subsection.15.10.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.4}Exit Codes}{56}{subsection.15.10.4}} +\newlabel{man/fsck:exit-codes}{{15.10.4}{56}{Exit Codes}{subsection.15.10.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.10.5}See Also}{57}{subsection.15.10.5}} +\newlabel{man/fsck:see-also}{{15.10.5}{57}{See Also}{subsection.15.10.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.11}The \textbf {s3ql\_oauth\_client} command}{57}{section.15.11}} +\newlabel{man/oauth_client::doc}{{15.11}{57}{The \textbf {s3ql\_oauth\_client} command}{section.15.11}{}} +\newlabel{man/oauth_client:the-command-command}{{15.11}{57}{The \textbf {s3ql\_oauth\_client} command}{section.15.11}{}} +\newlabel{man/oauth_client:oauth-client}{{15.11}{57}{The \textbf {s3ql\_oauth\_client} command}{section.15.11}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.1}Synopsis}{57}{subsection.15.11.1}} +\newlabel{man/oauth_client:synopsis}{{15.11.1}{57}{Synopsis}{subsection.15.11.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.2}Description}{57}{subsection.15.11.2}} +\newlabel{man/oauth_client:description}{{15.11.2}{57}{Description}{subsection.15.11.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.3}Options}{57}{subsection.15.11.3}} +\newlabel{man/oauth_client:options}{{15.11.3}{57}{Options}{subsection.15.11.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.4}Exit Codes}{57}{subsection.15.11.4}} +\newlabel{man/oauth_client:exit-codes}{{15.11.4}{57}{Exit Codes}{subsection.15.11.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.11.5}See Also}{58}{subsection.15.11.5}} +\newlabel{man/oauth_client:see-also}{{15.11.5}{58}{See Also}{subsection.15.11.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.12}The \textbf {s3ql\_verify} command}{58}{section.15.12}} +\newlabel{man/verify::doc}{{15.12}{58}{The \textbf {s3ql\_verify} command}{section.15.12}{}} +\newlabel{man/verify:the-command-command}{{15.12}{58}{The \textbf {s3ql\_verify} command}{section.15.12}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.12.1}Synopsis}{58}{subsection.15.12.1}} +\newlabel{man/verify:synopsis}{{15.12.1}{58}{Synopsis}{subsection.15.12.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.12.2}Description}{58}{subsection.15.12.2}} +\newlabel{man/verify:description}{{15.12.2}{58}{Description}{subsection.15.12.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.12.3}Options}{58}{subsection.15.12.3}} +\newlabel{man/verify:options}{{15.12.3}{58}{Options}{subsection.15.12.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.12.4}Exit Codes}{59}{subsection.15.12.4}} +\newlabel{man/verify:exit-codes}{{15.12.4}{59}{Exit Codes}{subsection.15.12.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.12.5}See Also}{59}{subsection.15.12.5}} +\newlabel{man/verify:see-also}{{15.12.5}{59}{See Also}{subsection.15.12.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.13}The \textbf {pcp} command}{59}{section.15.13}} +\newlabel{man/pcp::doc}{{15.13}{59}{The \textbf {pcp} command}{section.15.13}{}} +\newlabel{man/pcp:the-command-command}{{15.13}{59}{The \textbf {pcp} command}{section.15.13}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.13.1}Synopsis}{59}{subsection.15.13.1}} +\newlabel{man/pcp:synopsis}{{15.13.1}{59}{Synopsis}{subsection.15.13.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.13.2}Description}{59}{subsection.15.13.2}} +\newlabel{man/pcp:description}{{15.13.2}{59}{Description}{subsection.15.13.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.13.3}Options}{60}{subsection.15.13.3}} +\newlabel{man/pcp:options}{{15.13.3}{60}{Options}{subsection.15.13.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.13.4}Exit Codes}{60}{subsection.15.13.4}} +\newlabel{man/pcp:exit-codes}{{15.13.4}{60}{Exit Codes}{subsection.15.13.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.13.5}See Also}{60}{subsection.15.13.5}} +\newlabel{man/pcp:see-also}{{15.13.5}{60}{See Also}{subsection.15.13.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.14}The \textbf {expire\_backups} command}{60}{section.15.14}} +\newlabel{man/expire_backups::doc}{{15.14}{60}{The \textbf {expire\_backups} command}{section.15.14}{}} +\newlabel{man/expire_backups:the-command-command}{{15.14}{60}{The \textbf {expire\_backups} command}{section.15.14}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.14.1}Synopsis}{60}{subsection.15.14.1}} +\newlabel{man/expire_backups:synopsis}{{15.14.1}{60}{Synopsis}{subsection.15.14.1}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.14.2}Description}{60}{subsection.15.14.2}} +\newlabel{man/expire_backups:description}{{15.14.2}{60}{Description}{subsection.15.14.2}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.14.3}Options}{61}{subsection.15.14.3}} +\newlabel{man/expire_backups:options}{{15.14.3}{61}{Options}{subsection.15.14.3}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.14.4}Exit Codes}{62}{subsection.15.14.4}} +\newlabel{man/expire_backups:exit-codes}{{15.14.4}{62}{Exit Codes}{subsection.15.14.4}{}} +\@writefile{toc}{\contentsline {subsection}{\numberline {15.14.5}See Also}{62}{subsection.15.14.5}} +\newlabel{man/expire_backups:see-also}{{15.14.5}{62}{See Also}{subsection.15.14.5}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {16}Further Resources / Getting Help}{63}{chapter.16}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{resources::doc}{{16}{63}{Further Resources / Getting Help}{chapter.16}{}} +\newlabel{resources:resources}{{16}{63}{Further Resources / Getting Help}{chapter.16}{}} +\newlabel{resources:further-resources-getting-help}{{16}{63}{Further Resources / Getting Help}{chapter.16}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {17}Implementation Details}{65}{chapter.17}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{impl_details::doc}{{17}{65}{Implementation Details}{chapter.17}{}} +\newlabel{impl_details:impl-details}{{17}{65}{Implementation Details}{chapter.17}{}} +\newlabel{impl_details:implementation-details}{{17}{65}{Implementation Details}{chapter.17}{}} +\@writefile{toc}{\contentsline {section}{\numberline {17.1}Metadata Storage}{65}{section.17.1}} +\newlabel{impl_details:metadata-storage}{{17.1}{65}{Metadata Storage}{section.17.1}{}} +\@writefile{toc}{\contentsline {section}{\numberline {17.2}Data Storage}{65}{section.17.2}} +\newlabel{impl_details:data-storage}{{17.2}{65}{Data Storage}{section.17.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {17.3}Data De-Duplication}{65}{section.17.3}} +\newlabel{impl_details:data-de-duplication}{{17.3}{65}{Data De-Duplication}{section.17.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {17.4}Caching}{66}{section.17.4}} +\newlabel{impl_details:caching}{{17.4}{66}{Caching}{section.17.4}{}} +\@writefile{toc}{\contentsline {section}{\numberline {17.5}Eventual Consistency Handling}{66}{section.17.5}} +\newlabel{impl_details:eventual-consistency-handling}{{17.5}{66}{Eventual Consistency Handling}{section.17.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {17.6}Encryption}{66}{section.17.6}} +\newlabel{impl_details:encryption}{{17.6}{66}{Encryption}{section.17.6}{}} diff --git a/doc/latex/manual.idx b/doc/latex/manual.idx new file mode 100644 index 0000000..f02afb0 --- /dev/null +++ b/doc/latex/manual.idx @@ -0,0 +1,36 @@ +\indexentry{gs\_backend command line option!no-ssl|hyperpage}{7} +\indexentry{no-ssl!gs\_backend command line option|hyperpage}{7} +\indexentry{gs\_backend command line option!ssl-ca-path=\textless{}path\textgreater{}|hyperpage}{8} +\indexentry{ssl-ca-path=\textless{}path\textgreater{}!gs\_backend command line option|hyperpage}{8} +\indexentry{gs\_backend command line option!tcp-timeout|hyperpage}{8} +\indexentry{tcp-timeout!gs\_backend command line option|hyperpage}{8} +\indexentry{s3\_backend command line option!no-ssl|hyperpage}{8} +\indexentry{no-ssl!s3\_backend command line option|hyperpage}{8} +\indexentry{s3\_backend command line option!ssl-ca-path=\textless{}path\textgreater{}|hyperpage}{8} +\indexentry{ssl-ca-path=\textless{}path\textgreater{}!s3\_backend command line option|hyperpage}{8} +\indexentry{s3\_backend command line option!tcp-timeout|hyperpage}{8} +\indexentry{tcp-timeout!s3\_backend command line option|hyperpage}{8} +\indexentry{s3\_backend command line option!sse|hyperpage}{8} +\indexentry{sse!s3\_backend command line option|hyperpage}{8} +\indexentry{s3\_backend command line option!ia|hyperpage}{8} +\indexentry{ia!s3\_backend command line option|hyperpage}{8} +\indexentry{s3\_backend command line option!rrs|hyperpage}{8} +\indexentry{rrs!s3\_backend command line option|hyperpage}{8} +\indexentry{swift\_backend command line option!no-ssl|hyperpage}{9} +\indexentry{no-ssl!swift\_backend command line option|hyperpage}{9} +\indexentry{swift\_backend command line option!ssl-ca-path=\textless{}path\textgreater{}|hyperpage}{9} +\indexentry{ssl-ca-path=\textless{}path\textgreater{}!swift\_backend command line option|hyperpage}{9} +\indexentry{swift\_backend command line option!tcp-timeout|hyperpage}{9} +\indexentry{tcp-timeout!swift\_backend command line option|hyperpage}{9} +\indexentry{swift\_backend command line option!disable-expect100|hyperpage}{9} +\indexentry{disable-expect100!swift\_backend command line option|hyperpage}{9} +\indexentry{s3c\_backend command line option!no-ssl|hyperpage}{10} +\indexentry{no-ssl!s3c\_backend command line option|hyperpage}{10} +\indexentry{s3c\_backend command line option!ssl-ca-path=\textless{}path\textgreater{}|hyperpage}{10} +\indexentry{ssl-ca-path=\textless{}path\textgreater{}!s3c\_backend command line option|hyperpage}{10} +\indexentry{s3c\_backend command line option!tcp-timeout|hyperpage}{10} +\indexentry{tcp-timeout!s3c\_backend command line option|hyperpage}{10} +\indexentry{s3c\_backend command line option!disable-expect100|hyperpage}{10} +\indexentry{disable-expect100!s3c\_backend command line option|hyperpage}{10} +\indexentry{s3c\_backend command line option!dumb-copy|hyperpage}{10} +\indexentry{dumb-copy!s3c\_backend command line option|hyperpage}{10} diff --git a/doc/latex/manual.out b/doc/latex/manual.out new file mode 100644 index 0000000..42512bb --- /dev/null +++ b/doc/latex/manual.out @@ -0,0 +1,80 @@ +\BOOKMARK [0][-]{chapter.1}{About S3QL}{}% 1 +\BOOKMARK [1][-]{section.1.1}{Features}{chapter.1}% 2 +\BOOKMARK [1][-]{section.1.2}{Development Status}{chapter.1}% 3 +\BOOKMARK [1][-]{section.1.3}{Supported Platforms}{chapter.1}% 4 +\BOOKMARK [1][-]{section.1.4}{Contributing}{chapter.1}% 5 +\BOOKMARK [0][-]{chapter.2}{Installation}{}% 6 +\BOOKMARK [1][-]{section.2.1}{Dependencies}{chapter.2}% 7 +\BOOKMARK [1][-]{section.2.2}{Installing S3QL}{chapter.2}% 8 +\BOOKMARK [1][-]{section.2.3}{Development Version}{chapter.2}% 9 +\BOOKMARK [1][-]{section.2.4}{Running tests requiring remote servers}{chapter.2}% 10 +\BOOKMARK [0][-]{chapter.3}{Storage Backends}{}% 11 +\BOOKMARK [1][-]{section.3.1}{Google Storage}{chapter.3}% 12 +\BOOKMARK [1][-]{section.3.2}{Amazon S3}{chapter.3}% 13 +\BOOKMARK [1][-]{section.3.3}{OpenStack/Swift}{chapter.3}% 14 +\BOOKMARK [1][-]{section.3.4}{Rackspace CloudFiles}{chapter.3}% 15 +\BOOKMARK [1][-]{section.3.5}{S3 compatible}{chapter.3}% 16 +\BOOKMARK [1][-]{section.3.6}{Local}{chapter.3}% 17 +\BOOKMARK [0][-]{chapter.4}{Important Rules to Avoid Losing Data}{}% 18 +\BOOKMARK [1][-]{section.4.1}{Rules in a Nutshell}{chapter.4}% 19 +\BOOKMARK [1][-]{section.4.2}{Consistency Window List}{chapter.4}% 20 +\BOOKMARK [1][-]{section.4.3}{Data Consistency}{chapter.4}% 21 +\BOOKMARK [1][-]{section.4.4}{Data Durability}{chapter.4}% 22 +\BOOKMARK [0][-]{chapter.5}{File System Creation}{}% 23 +\BOOKMARK [0][-]{chapter.6}{Managing File Systems}{}% 24 +\BOOKMARK [1][-]{section.6.1}{Changing the Passphrase}{chapter.6}% 25 +\BOOKMARK [1][-]{section.6.2}{Upgrading the file system}{chapter.6}% 26 +\BOOKMARK [1][-]{section.6.3}{Deleting a file system}{chapter.6}% 27 +\BOOKMARK [1][-]{section.6.4}{Restoring Metadata Backups}{chapter.6}% 28 +\BOOKMARK [0][-]{chapter.7}{Mounting}{}% 29 +\BOOKMARK [1][-]{section.7.1}{Permission Checking}{chapter.7}% 30 +\BOOKMARK [1][-]{section.7.2}{Compression Algorithms}{chapter.7}% 31 +\BOOKMARK [1][-]{section.7.3}{Notes about Caching}{chapter.7}% 32 +\BOOKMARK [1][-]{section.7.4}{Failure Modes}{chapter.7}% 33 +\BOOKMARK [1][-]{section.7.5}{Automatic Mounting}{chapter.7}% 34 +\BOOKMARK [0][-]{chapter.8}{Advanced S3QL Features}{}% 35 +\BOOKMARK [1][-]{section.8.1}{Snapshotting and Copy-on-Write}{chapter.8}% 36 +\BOOKMARK [1][-]{section.8.2}{Getting Statistics}{chapter.8}% 37 +\BOOKMARK [1][-]{section.8.3}{Immutable Trees}{chapter.8}% 38 +\BOOKMARK [1][-]{section.8.4}{Fast Recursive Removal}{chapter.8}% 39 +\BOOKMARK [1][-]{section.8.5}{Runtime Configuration}{chapter.8}% 40 +\BOOKMARK [0][-]{chapter.9}{Unmounting}{}% 41 +\BOOKMARK [0][-]{chapter.10}{Checking for Errors}{}% 42 +\BOOKMARK [1][-]{section.10.1}{Checking and repairing internal file system errors}{chapter.10}% 43 +\BOOKMARK [1][-]{section.10.2}{Detecting and handling backend data corruption}{chapter.10}% 44 +\BOOKMARK [0][-]{chapter.11}{Storing Authentication Information}{}% 45 +\BOOKMARK [0][-]{chapter.12}{Contributed Programs}{}% 46 +\BOOKMARK [1][-]{section.12.1}{benchmark.py}{chapter.12}% 47 +\BOOKMARK [1][-]{section.12.2}{clone\137fs.py}{chapter.12}% 48 +\BOOKMARK [1][-]{section.12.3}{pcp.py}{chapter.12}% 49 +\BOOKMARK [1][-]{section.12.4}{s3ql\137backup.sh}{chapter.12}% 50 +\BOOKMARK [1][-]{section.12.5}{expire\137backups.py}{chapter.12}% 51 +\BOOKMARK [1][-]{section.12.6}{remove\137objects.py}{chapter.12}% 52 +\BOOKMARK [0][-]{chapter.13}{Tips \046 Tricks}{}% 53 +\BOOKMARK [1][-]{section.13.1}{SSH Backend}{chapter.13}% 54 +\BOOKMARK [1][-]{section.13.2}{Permanently mounted backup file system}{chapter.13}% 55 +\BOOKMARK [1][-]{section.13.3}{Improving copy performance}{chapter.13}% 56 +\BOOKMARK [0][-]{chapter.14}{Known Issues}{}% 57 +\BOOKMARK [0][-]{chapter.15}{Manpages}{}% 58 +\BOOKMARK [1][-]{section.15.1}{The mkfs.s3ql command}{chapter.15}% 59 +\BOOKMARK [1][-]{section.15.2}{The s3qladm command}{chapter.15}% 60 +\BOOKMARK [1][-]{section.15.3}{The mount.s3ql command}{chapter.15}% 61 +\BOOKMARK [1][-]{section.15.4}{The s3qlstat command}{chapter.15}% 62 +\BOOKMARK [1][-]{section.15.5}{The s3qlctrl command}{chapter.15}% 63 +\BOOKMARK [1][-]{section.15.6}{The s3qlcp command}{chapter.15}% 64 +\BOOKMARK [1][-]{section.15.7}{The s3qlrm command}{chapter.15}% 65 +\BOOKMARK [1][-]{section.15.8}{The s3qllock command}{chapter.15}% 66 +\BOOKMARK [1][-]{section.15.9}{The umount.s3ql command}{chapter.15}% 67 +\BOOKMARK [1][-]{section.15.10}{The fsck.s3ql command}{chapter.15}% 68 +\BOOKMARK [1][-]{section.15.11}{The s3ql\137oauth\137client command}{chapter.15}% 69 +\BOOKMARK [1][-]{section.15.12}{The s3ql\137verify command}{chapter.15}% 70 +\BOOKMARK [1][-]{section.15.13}{The pcp command}{chapter.15}% 71 +\BOOKMARK [1][-]{section.15.14}{The expire\137backups command}{chapter.15}% 72 +\BOOKMARK [0][-]{chapter.16}{Further Resources / Getting Help}{}% 73 +\BOOKMARK [0][-]{chapter.17}{Implementation Details}{}% 74 +\BOOKMARK [1][-]{section.17.1}{Metadata Storage}{chapter.17}% 75 +\BOOKMARK [1][-]{section.17.2}{Data Storage}{chapter.17}% 76 +\BOOKMARK [1][-]{section.17.3}{Data De-Duplication}{chapter.17}% 77 +\BOOKMARK [1][-]{section.17.4}{Caching}{chapter.17}% 78 +\BOOKMARK [1][-]{section.17.5}{Eventual Consistency Handling}{chapter.17}% 79 +\BOOKMARK [1][-]{section.17.6}{Encryption}{chapter.17}% 80 diff --git a/doc/latex/manual.tex b/doc/latex/manual.tex new file mode 100644 index 0000000..7446c1f --- /dev/null +++ b/doc/latex/manual.tex @@ -0,0 +1,4168 @@ +% Generated by Sphinx. +\def\sphinxdocclass{report} +\documentclass[letterpaper,10pt,english]{sphinxmanual} +\usepackage[utf8]{inputenc} +\DeclareUnicodeCharacter{00A0}{\nobreakspace} +\usepackage{cmap} +\usepackage[T1]{fontenc} +\usepackage{babel} +\usepackage{times} +\usepackage[Bjarne]{fncychap} +\usepackage{longtable} +\usepackage{sphinx} +\usepackage{multirow} + + +\title{S3QL Documentation} +\date{October 28, 2016} +\release{2.21} +\author{Nikolaus Rath} +\newcommand{\sphinxlogo}{} +\renewcommand{\releasename}{Release} +\makeindex + +\makeatletter +\def\PYG@reset{\let\PYG@it=\relax \let\PYG@bf=\relax% + \let\PYG@ul=\relax \let\PYG@tc=\relax% + \let\PYG@bc=\relax \let\PYG@ff=\relax} +\def\PYG@tok#1{\csname PYG@tok@#1\endcsname} +\def\PYG@toks#1+{\ifx\relax#1\empty\else% + \PYG@tok{#1}\expandafter\PYG@toks\fi} +\def\PYG@do#1{\PYG@bc{\PYG@tc{\PYG@ul{% + \PYG@it{\PYG@bf{\PYG@ff{#1}}}}}}} +\def\PYG#1#2{\PYG@reset\PYG@toks#1+\relax+\PYG@do{#2}} + +\expandafter\def\csname PYG@tok@kt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.60}{##1}}} +\expandafter\def\csname PYG@tok@se\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@kr\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@sc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} +\expandafter\def\csname PYG@tok@nt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@gs\endcsname{\let\PYG@bf=\textbf} +\expandafter\def\csname PYG@tok@cs\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.80,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@err\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.67,0.67}{\strut ##1}}} +\expandafter\def\csname PYG@tok@gu\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}} +\expandafter\def\csname PYG@tok@kp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.53}{##1}}} +\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.33,0.53}{##1}}} +\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} +\expandafter\def\csname PYG@tok@sr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,1.00}{\strut ##1}}} +\expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@c1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@vg\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.87,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@mf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@gh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}} +\expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.80}{##1}}} +\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}} +\expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit} +\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.67,0.40,0.00}{##1}}} +\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} +\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@sh\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.33,0.47,0.60}{##1}}} +\expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} +\expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@nl\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.60,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.73,0.00,0.40}{##1}}} +\expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.20}{##1}}} +\expandafter\def\csname PYG@tok@mo\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.27,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.33,0.33,0.33}{##1}}} +\expandafter\def\csname PYG@tok@mi\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} +\expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}} +\expandafter\def\csname PYG@tok@vc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.40,0.60}{##1}}} +\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} +\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@si\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.93,0.93,0.93}{\strut ##1}}} +\expandafter\def\csname PYG@tok@no\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.40}{##1}}} +\expandafter\def\csname PYG@tok@mb\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@ow\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@sd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.27,0.13}{##1}}} +\expandafter\def\csname PYG@tok@kc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}} +\expandafter\def\csname PYG@tok@s\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@il\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} +\expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@gp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}} +\expandafter\def\csname PYG@tok@ne\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@c\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@bp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} +\expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.73}{##1}}} + +\def\PYGZbs{\char`\\} +\def\PYGZus{\char`\_} +\def\PYGZob{\char`\{} +\def\PYGZcb{\char`\}} +\def\PYGZca{\char`\^} +\def\PYGZam{\char`\&} +\def\PYGZlt{\char`\<} +\def\PYGZgt{\char`\>} +\def\PYGZsh{\char`\#} +\def\PYGZpc{\char`\%} +\def\PYGZdl{\char`\$} +\def\PYGZhy{\char`\-} +\def\PYGZsq{\char`\'} +\def\PYGZdq{\char`\"} +\def\PYGZti{\char`\~} +% for compatibility with earlier versions +\def\PYGZat{@} +\def\PYGZlb{[} +\def\PYGZrb{]} +\makeatother + +\renewcommand\PYGZsq{\textquotesingle} + +\begin{document} + +\maketitle +\tableofcontents +\phantomsection\label{index::doc} + + + +\chapter{About S3QL} +\label{about::doc}\label{about:s3ql-user-s-guide}\label{about:about-s3ql} +S3QL is a file system that stores all its data online using storage +services like \href{http://code.google.com/apis/storage/}{Google Storage}, \href{http://aws.amazon.com/s3}{Amazon S3}, or \href{http://openstack.org/projects/storage/}{OpenStack}. S3QL +effectively provides a hard disk of dynamic, infinite capacity that +can be accessed from any computer with internet access. + +S3QL is a standard conforming, full featured UNIX file system that is +conceptually indistinguishable from any local file system. +Furthermore, S3QL has additional features like compression, +encryption, data de-duplication, immutable trees and snapshotting +which make it especially suitable for online backup and archival. + +S3QL is designed to favor simplicity and elegance over performance and +feature-creep. Care has been taken to make the source code as +readable and serviceable as possible. Solid error detection and error +handling have been included from the very first line, and S3QL comes +with extensive automated test cases for all its components. + + +\section{Features} +\label{about:openstack}\label{about:features}\begin{itemize} +\item {} +\textbf{Transparency.} Conceptually, S3QL is indistinguishable from a +local file system. For example, it supports hardlinks, symlinks, +standard unix permissions, extended attributes and file +sizes up to 2 TB. + +\item {} +\textbf{Dynamic Size.} The size of an S3QL file system grows and shrinks +dynamically as required. + +\item {} +\textbf{Compression.} Before storage, all data may compressed with the +LZMA, bzip2 or deflate (gzip) algorithm. + +\item {} +\textbf{Encryption.} After compression (but before upload), all data can be +AES encrypted with a 256 bit key. An additional SHA256 HMAC checksum +is used to protect the data against manipulation. + +\item {} +\textbf{Data De-duplication.} If several files have identical contents, +the redundant data will be stored only once. This works across all +files stored in the file system, and also if only some parts of the +files are identical while other parts differ. + +\item {} +\textbf{Immutable Trees.} Directory trees can be made immutable, so that +their contents can no longer be changed in any way whatsoever. This +can be used to ensure that backups can not be modified after they +have been made. + +\item {} +\textbf{Copy-on-Write/Snapshotting.} S3QL can replicate entire directory +trees without using any additional storage space. Only if one of the +copies is modified, the part of the data that has been modified will +take up additional storage space. This can be used to create +intelligent snapshots that preserve the state of a directory at +different points in time using a minimum amount of space. + +\item {} +\textbf{High Performance independent of network latency.} All operations +that do not write or read file contents (like creating directories +or moving, renaming, and changing permissions of files and +directories) are very fast because they are carried out without any +network transactions. + +S3QL achieves this by saving the entire file and directory structure +in a database. This database is locally cached and the remote +copy updated asynchronously. + +\item {} +\textbf{Support for low bandwidth connections.} S3QL splits file contents +into smaller blocks and caches blocks locally. This minimizes both +the number of network transactions required for reading and writing +data, and the amount of data that has to be transferred when only +parts of a file are read or written. + +\end{itemize} + + +\section{Development Status} +\label{about:development-status} +S3QL is considered stable and suitable for production use. Starting +with version 2.17.1, S3QL uses semantic versioning. This means that +backwards-incompatible versions (e.g., versions that require an +upgrade of the file system revision) will be reflected in an increase +of the major version number. + + +\section{Supported Platforms} +\label{about:supported-platforms} +S3QL is developed and tested under Linux. Users have also reported +running S3QL successfully on OS-X, FreeBSD and NetBSD. We try to +maintain compatibility with these systems, but (due to lack of +pre-release testers) we cannot guarantee that every release will run +on all non-Linux systems. Please report any bugs you find, and we will +try to fix them. + + +\section{Contributing} +\label{about:contributing} +The S3QL source code is available both on \href{https://github.com/s3ql/main}{GitHub} and \href{https://bitbucket.org/nikratio/s3ql/}{BitBucket}. + + +\chapter{Installation} +\label{installation::doc}\label{installation:installation}\label{installation:github} +S3QL depends on several other programs and libraries that have to be +installed first. The best method to satisfy these dependencies depends +on your distribution. + +The following instructions are for S3QL 2.21 and should be +applicable to any system. The \href{https://bitbucket.org/nikratio/s3ql/wiki/Home}{S3QL Wiki} contains \href{https://bitbucket.org/nikratio/s3ql/wiki/Installation}{additional +help} help +for specific distributions and operating systems. Note, however, that +S3QL wiki is editable by anyone. The information there has thus not +been vetted by the S3QL maintainers, and may be wrong, out-of-date, or +even dangerous. Generally, you should only follow steps from the Wiki +that you fully understand yourself, and fall back on the instructions +below when in doubt. + + +\section{Dependencies} +\label{installation:dependencies} +The following is a list of the programs and libraries required for +running S3QL. Generally, you should first check if your distribution +already provides a suitable packages and only install from source if +that is not the case. +\begin{itemize} +\item {} +Kernel: Linux 2.6.9 or newer or FreeBSD with \href{http://www.freshports.org/sysutils/fusefs-kmod/}{FUSE4BSD}. Starting with +kernel 2.6.26 you will get significantly better write performance, +so under Linux you should actually use \emph{2.6.26 or newer whenever +possible}. + +\item {} +The \href{http://psmisc.sf.net/}{psmisc} utilities. + +\item {} +\href{http://www.sqlite.org/}{SQLite} version 3.7.0 or newer. SQLite +has to be installed as a \emph{shared library} with development headers. + +\item {} +\href{http://www.python.org/}{Python} 3.3.0 or newer. Make sure to also +install the development headers. + +\item {} +The following Python modules: +\begin{itemize} +\item {} +\href{https://pypi.python.org/pypi/setuptools}{setuptools}, version 1.0 or newer. + +\item {} +\href{https://www.dlitz.net/software/pycrypto/}{pycrypto} + +\item {} +\href{https://pypi.python.org/pypi/defusedxml/}{defusedxml} + +\item {} +\href{https://pypi.python.org/pypi/requests/}{requests} (optional, +required for OAuth2 authentication with Google Storage) + +\item {} +\href{https://github.com/systemd/python-systemd}{systemd} (optional, +for enabling systemd support). + +\item {} +\href{https://github.com/rogerbinns/apsw}{apsw}, version 3.7.0 or +newer. + +\item {} +\href{https://bitbucket.org/nikratio/python-llfuse/}{llfuse}, any +version between 1.0 (inclusive) and 2.0 (exclusive) + +\item {} +\href{https://bitbucket.org/nikratio/python-dugong/}{dugong}, any +version between 3.4 (inclusive) and 4.0 (exclusive) + +\item {} +\href{http://pytest.org/}{pytest}, version 2.3.3 or newer (optional, to run unit tests) + +\item {} +\href{https://github.com/eisensheng/pytest-catchlog}{pytest-catchlog} +(optional, to run unit tests) + +\end{itemize} + +To check if a specific module \code{\textless{}module\textgreater{}} is installed, execute +\code{python3 -c 'import \emph{\textless{}module\textgreater{}}; +print(\emph{\textless{}module\textgreater{}}.\_\_version\_\_)'}. This will result in an +\code{ImportError} if the module is not installed, and will print the +installed version if the module is installed. + +\end{itemize} + + +\section{Installing S3QL} +\label{installation:inst-s3ql}\label{installation:installing-s3ql} +To build and install S3QL itself, proceed as follows: +\begin{enumerate} +\item {} +Download S3QL from \href{https://bitbucket.org/nikratio/s3ql/downloads}{https://bitbucket.org/nikratio/s3ql/downloads} + +\item {} +Unpack it into a folder of your choice + +\item {} +Run \code{python3 setup.py build\_ext -{-}inplace} to build S3QL. + +\item {} +Run \code{python3 -m pytest tests/} to run a self-test. If this fails, ask +for help on the \href{http://groups.google.com/group/s3ql}{mailing list} or report a bug in the +\href{https://bitbucket.org/nikratio/s3ql/issues}{issue tracker}. + +\end{enumerate} + +Now you have three options: +\begin{itemize} +\item {} +You can run the S3QL commands from the \code{bin/} directory. + +\item {} +You can install S3QL system-wide for all users. To do that, you +have to run \code{sudo python3 setup.py install}. + +\item {} +You can install S3QL into \code{\textasciitilde{}/.local} by executing \code{python3 +setup.py install -{-}user}. In this case you should make sure that +\code{\textasciitilde{}/.local/bin} is in your \code{\$PATH} environment variable. + +\end{itemize} + + +\section{Development Version} +\label{installation:development-version} +If you have checked out the unstable development version from the +Mercurial repository, a bit more effort is required. You'll also need: +\begin{itemize} +\item {} +Version 0.24 or newer of the \href{http://www.cython.org/}{Cython} compiler. + +\item {} +Version 1.2b1 or newer of the \href{http://sphinx.pocoo.org/}{Sphinx} document processor. + +\end{itemize} + +With these additional dependencies installed, S3QL can be build and +tested with + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{python3 setup.py build\PYGZus{}cython} +\PYG{l}{python3 setup.py build\PYGZus{}ext \PYGZhy{}\PYGZhy{}inplace} +\PYG{l}{python3 \PYGZhy{}m pytest tests/} +\end{Verbatim} + +Note that when building from the Mercurial or Git repository, building +and testing is done with several additional checks. This may cause +compilation and/or tests to fail even though there are no problems +with functionality. For example, any use of functions that are +scheduled for deprecation in future Python version will cause tests to +fail. If you would rather just check for functionality, you can delete +the \code{MANIFEST.in} file. In that case, the build system will +behave as it does for a regular release. + +The HTML and PDF documentation can be generated with + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{python3 setup.py build\PYGZus{}sphinx} +\end{Verbatim} + +and S3QL can be installed as usual with + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{python3 setup.py install }\PYG{g+ge}{[\PYGZhy{}\PYGZhy{}user]} +\end{Verbatim} + + +\section{Running tests requiring remote servers} +\label{installation:running-tests-requiring-remote-servers} +By default, tests requiring a connection to a remote storage backend +are skipped. If you would like to run these tests too (which is always +a good idea), you have to create additional entries in your +\code{\textasciitilde{}/.s3ql/authinfo2} file that tell S3QL what server and credentials to +use for these tests. These entries have the following form: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{g+ge}{[\PYGZlt{}BACKEND\PYGZgt{}\PYGZhy{}test]} +\PYG{l}{backend\PYGZhy{}login: }\PYG{n+nv}{\PYGZlt{}user\PYGZgt{}} +\PYG{l}{backend\PYGZhy{}password: }\PYG{n+nv}{\PYGZlt{}password\PYGZgt{}} +\PYG{l}{test\PYGZhy{}fs: }\PYG{n+nv}{\PYGZlt{}storage\PYGZhy{}url\PYGZgt{}} +\end{Verbatim} + +Here \emph{\textless{}BACKEND\textgreater{}} specifies the backend that you want to test +(e.g. \emph{s3}, \emph{s3c}, \emph{gs}, or \emph{swift}), \emph{\textless{}user\textgreater{}} and \emph{\textless{}password\textgreater{}} are +the backend authentication credentials, and \emph{\textless{}storage-url\textgreater{}} specifies +the full storage URL that will be used for testing. \textbf{Any existing +S3QL file system in this storage URL will be destroyed during +testing}. + +For example, to run tests that need connection to a Google Storage +server, you would add something like + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{g+ge}{[gs\PYGZhy{}test]} +\PYG{l}{backend\PYGZhy{}login: GOOGIGWLONT238MD7HZ4} +\PYG{l}{backend\PYGZhy{}password: rmEbstjscoeunt1249oes1298gauidbs3hl} +\PYG{l}{test\PYGZhy{}fs: gs://joes\PYGZhy{}gs\PYGZhy{}bucket/s3ql\PYGZus{}tests/} +\end{Verbatim} + +On the next run of \code{runtest.py} (or \code{py.test} when using the +development version), the additional tests will be run. If the tests +are still skipped, you can get more information about why tests are +being skipped by passing the \code{-rs} argument to +\code{runtest.py}/\code{py.test}. + + +\chapter{Storage Backends} +\label{backends::doc}\label{backends:storage-backends}\label{backends:id1}\label{backends:sphinx} +S3QL supports different \emph{backends} to store data at different service +providers and using different protocols. A \emph{storage url} specifies a +backend together with some backend-specific information and uniquely +identifies an S3QL file system. The form of the storage url depends on +the backend and is described for every backend below. + +Furthermore, every S3QL commands that accepts a storage url also +accepts a \code{-{-}backend-options} parameter than can be used to +pass backend-specific options to the backend module. The available +options are documented with the respective backends below. + +All storage backends respect the \code{http\_proxy} (for plain HTTP +connections) and \code{https\_proxy} (for SSL connections) +environment variables. + +\begin{notice}{note}{Note:} +Storage backends are not necessarily compatible. Don't expect that +you can e.g. copy the data stored by the local backend into Amazon +S3 using some non-S3QL tool and then access it with S3QL's S3 +backend. If you want to copy file systems from one backend to +another, you need to use the \code{clone\_fs.py} script (from the +\code{contrib} directory in the S3QL tarball). +\end{notice} + + +\section{Google Storage} +\label{backends:google-storage} +\href{http://code.google.com/apis/storage/}{Google Storage} is an online +storage service offered by Google. To use the Google Storage backend, +you need to have (or sign up for) a Google account, and then \href{http://code.google.com/apis/storage/docs/signup.html}{activate +Google Storage} +for your account. The account is free, you will pay only for the +amount of storage and traffic that you actually use. There are two +ways to access Google storage: +\begin{enumerate} +\item {} +Use S3-like authentication. To do this, first \href{https://developers.google.com/storage/docs/migrating\#defaultproj}{set a default +project}. +Then use the \href{https://code.google.com/apis/console/\#:storage:legacy}{key management tool} to +retrieve your \emph{Google Storage developer access key} and \emph{Google +Storage developer secret} and use that as backend login and backend +password. + +\item {} +Use OAuth2 authentication. In this case you need to use \code{oauth2} +as the backend login, and a valid OAuth2 refresh token as the +backend password. To obtain a refresh token, you can use the +{\hyperref[man/oauth_client:oauth-client]{\emph{s3ql\_oauth\_client}}} program. It will instruct +you to open a specific URL in your browser, enter a code and +authenticate with your Google account. Once this procedure is +complete, {\hyperref[man/oauth_client:oauth-client]{\emph{s3ql\_oauth\_client}}} will print out +the refresh token. Note that you need to do this procedure only +once, the refresh token will remain valid until you explicitly +revoke it. + +\end{enumerate} + +To create a Google Storage bucket, you can use e.g. the \href{https://sandbox.google.com/storage/}{Google +Storage Manager}. The storage URL for accessing the bucket in S3QL is +then + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{gs://}\PYG{n+nv}{\PYGZlt{}bucketname\PYGZgt{}}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}prefix\PYGZgt{}} +\end{Verbatim} + +Here \emph{bucketname} is the name of the bucket, and \emph{prefix} can be an +arbitrary prefix that will be prepended to all object names used by +S3QL. This allows you to store several S3QL file systems in the same +Google Storage bucket. + +The Google Storage backend accepts the following backend options: +\index{gs\_backend command line option!no-ssl}\index{no-ssl!gs\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-gs_backend-arg-no-ssl}\pysigline{\bfcode{no-ssl}\code{}} +Disable encrypted (https) connections and use plain HTTP instead. + +\end{fulllineitems} + +\index{gs\_backend command line option!ssl-ca-path=\textless{}path\textgreater{}}\index{ssl-ca-path=\textless{}path\textgreater{}!gs\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-gs_backend-arg-ssl-ca-path}\pysigline{\bfcode{ssl-ca-path}\code{=\textless{}path\textgreater{}}} +Instead of using the system's default certificate store, validate +the server certificate against the specified CA +certificates. \code{\textless{}path\textgreater{}} may be either a file containing +multiple certificates, or a directory containing one certificate +per file. + +\end{fulllineitems} + +\index{gs\_backend command line option!tcp-timeout}\index{tcp-timeout!gs\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-gs_backend-arg-tcp-timeout}\pysigline{\bfcode{tcp-timeout}\code{}} +Specifies the timeout used for TCP connections. If no data can be +exchanged with the remote server for longer than this period, the +TCP connection is closed and re-established (default: 20 seconds). + +\end{fulllineitems} + + + +\section{Amazon S3} +\label{backends:amazon-s3}\label{backends:google-storage-manager} +\href{http://aws.amazon.com/s3}{Amazon S3} is the online storage service +offered by \href{http://aws.amazon.com/}{Amazon Web Services (AWS)}. To +use the S3 backend, you first need to sign up for an AWS account. The +account is free, you will pay only for the amount of storage and +traffic that you actually use. After that, you need to create a bucket +that will hold the S3QL file system, e.g. using the \href{https://console.aws.amazon.com/s3/home}{AWS Management +Console}. For best +performance, it is recommend to create the bucket in the +geographically closest storage region, but not the US Standard region +(see {\hyperref[durability:durability]{\emph{Important Rules to Avoid Losing Data}}} for the reason). + +The storage URL for accessing S3 buckets in S3QL has the form + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3://}\PYG{n+nv}{\PYGZlt{}bucketname\PYGZgt{}}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}prefix\PYGZgt{}} +\end{Verbatim} + +Here \emph{bucketname} is the name of the bucket, and \emph{prefix} can be an +arbitrary prefix that will be prepended to all object names used by +S3QL. This allows you to store several S3QL file systems in the same +S3 bucket. + +Note that the backend login and password for accessing S3 are not the +user id and password that you use to log into the Amazon Webpage, but +the \emph{AWS access key id} and \emph{AWS secret access key} shown under \href{https://aws-portal.amazon.com/gp/aws/developer/account/index.html?ie=UTF8\&action=access-key}{My +Account/Access Identifiers}. + +The Amazon S3 backend accepts the following backend options: +\index{s3\_backend command line option!no-ssl}\index{no-ssl!s3\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-s3_backend-arg-no-ssl}\pysigline{\bfcode{no-ssl}\code{}} +Disable encrypted (https) connections and use plain HTTP instead. + +\end{fulllineitems} + +\index{s3\_backend command line option!ssl-ca-path=\textless{}path\textgreater{}}\index{ssl-ca-path=\textless{}path\textgreater{}!s3\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-s3_backend-arg-ssl-ca-path}\pysigline{\bfcode{ssl-ca-path}\code{=\textless{}path\textgreater{}}} +Instead of using the system's default certificate store, validate +the server certificate against the specified CA +certificates. \code{\textless{}path\textgreater{}} may be either a file containing +multiple certificates, or a directory containing one certificate +per file. + +\end{fulllineitems} + +\index{s3\_backend command line option!tcp-timeout}\index{tcp-timeout!s3\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-s3_backend-arg-tcp-timeout}\pysigline{\bfcode{tcp-timeout}\code{}} +Specifies the timeout used for TCP connections. If no data can be +exchanged with the remote server for longer than this period, the +TCP connection is closed and re-established (default: 20 seconds). + +\end{fulllineitems} + +\index{s3\_backend command line option!sse}\index{sse!s3\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-s3_backend-arg-sse}\pysigline{\bfcode{sse}\code{}} +Enable server side encryption. Both costs \& benefits of S3 server +side encryption are probably rather small, and this option does +\emph{not} affect any client side encryption performed by S3QL itself. + +\end{fulllineitems} + +\index{s3\_backend command line option!ia}\index{ia!s3\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-s3_backend-arg-ia}\pysigline{\bfcode{ia}\code{}} +Use infrequent access storage class for new objects. + +\end{fulllineitems} + +\index{s3\_backend command line option!rrs}\index{rrs!s3\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-s3_backend-arg-rrs}\pysigline{\bfcode{rrs}\code{}} +Enable reduced redundancy storage for newly created objects +(overwrites the \emph{ia} option). + +When enabling this option, it is strongly recommended to +periodically run {\hyperref[fsck:s3ql-verify]{\emph{s3ql\_verify}}}, because objects +that are lost by the storage backend may cause subsequent data loss +even later in time due to the data de-duplication feature of S3QL (see +{\hyperref[durability:backend-reliability]{\emph{Data Durability}}} for details). + +\end{fulllineitems} + + + +\section{OpenStack/Swift} +\label{backends:openstack-swift}\label{backends:openstack-backend} +\href{http://www.openstack.org/}{OpenStack} is an open-source cloud server application suite. \href{http://openstack.org/projects/storage/}{Swift} is +the cloud storage module of OpenStack. Swift/OpenStack storage is +offered by many different companies. + +There are two different storage URL for the OpenStack backend that +make use of different authentication APIs. For legacy (v1) +authentication, the storage URL is + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{swift://}\PYG{n+nv}{\PYGZlt{}hostname\PYGZgt{}}\PYG{g+ge}{[:\PYGZlt{}port\PYGZgt{}]}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}container\PYGZgt{}}\PYG{g+ge}{[/\PYGZlt{}prefix\PYGZgt{}]} +\end{Verbatim} + +for keystore (v2) authentication, the storage URL is + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{swiftks://}\PYG{n+nv}{\PYGZlt{}hostname\PYGZgt{}}\PYG{g+ge}{[:\PYGZlt{}port\PYGZgt{}]}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}region\PYGZgt{}}\PYG{l}{:}\PYG{n+nv}{\PYGZlt{}container\PYGZgt{}}\PYG{g+ge}{[/\PYGZlt{}prefix\PYGZgt{}]} +\end{Verbatim} + +Note that when using keystore authentication, you can (and have to) +specify the storage region of the container as well. + +In both cases, \emph{hostname} name should be the name of the +authentication server. The storage container must already exist (most +OpenStack providers offer either a web frontend or a command line tool +for creating containers). \emph{prefix} can be an arbitrary prefix that +will be prepended to all object names used by S3QL, which can be used +to store multiple S3QL file systems in the same container. + +When using legacy authentication, the backend login and password +correspond to the OpenStack username and API Access Key. When using +keystore authentication, the backend password is your regular +OpenStack password and the backend login combines you OpenStack +username and tenant name in the form \code{\textless{}tenant\textgreater{}:\textless{}user\textgreater{}}. If no tenant +is required, the OpenStack username alone may be used as backend +login. + +The OpenStack backend accepts the following backend options: +\index{swift\_backend command line option!no-ssl}\index{no-ssl!swift\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-swift_backend-arg-no-ssl}\pysigline{\bfcode{no-ssl}\code{}} +Use plain HTTP to connect to the authentication server. This option +does not directly affect the connection to the storage +server. Whether HTTPS or plain HTTP is used to connect to the +storage server is determined by the authentication server. + +\end{fulllineitems} + +\index{swift\_backend command line option!ssl-ca-path=\textless{}path\textgreater{}}\index{ssl-ca-path=\textless{}path\textgreater{}!swift\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-swift_backend-arg-ssl-ca-path}\pysigline{\bfcode{ssl-ca-path}\code{=\textless{}path\textgreater{}}} +Instead of using the system's default certificate store, validate +the server certificate against the specified CA +certificates. \code{\textless{}path\textgreater{}} may be either a file containing +multiple certificates, or a directory containing one certificate +per file. + +\end{fulllineitems} + +\index{swift\_backend command line option!tcp-timeout}\index{tcp-timeout!swift\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-swift_backend-arg-tcp-timeout}\pysigline{\bfcode{tcp-timeout}\code{}} +Specifies the timeout used for TCP connections. If no data can be +exchanged with the remote server for longer than this period, the +TCP connection is closed and re-established (default: 20 seconds). + +\end{fulllineitems} + +\index{swift\_backend command line option!disable-expect100}\index{disable-expect100!swift\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-swift_backend-arg-disable-expect100}\pysigline{\bfcode{disable-expect100}\code{}} +If this option is specified, S3QL does not use the \code{Expect: +continue} header (cf. \href{http://tools.ietf.org/html/rfc2616\#section-8.2.3}{RFC2616, section 8.2.3}) when uploading +data to the server. This can be used to work around broken storage +servers that don't fully support HTTP 1.1, but may decrease +performance as object data will be transmitted to the server more +than once in some circumstances. + +\end{fulllineitems} + + + +\section{Rackspace CloudFiles} +\label{backends:swift}\label{backends:rackspace-cloudfiles} +\href{http://www.rackspace.com/}{Rackspace} CloudFiles uses \href{http://www.openstack.org/}{OpenStack} internally, so it is possible to +just use the OpenStack/Swift backend (see above) with +\code{auth.api.rackspacecloud.com} as the host name. For convenince, +there is also a special \code{rackspace} backend that uses a storage URL +of the form + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{rackspace://}\PYG{n+nv}{\PYGZlt{}region\PYGZgt{}}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}container\PYGZgt{}}\PYG{g+ge}{[/\PYGZlt{}prefix\PYGZgt{}]} +\end{Verbatim} + +The storage container must already exist in the selected +region. \emph{prefix} can be an arbitrary prefix that will be prepended to +all object names used by S3QL and can be used to store several S3QL +file systems in the same container. + +You can create a storage container for S3QL using the \href{https://mycloud.rackspace.com/}{Cloud Control +Panel} (click on \emph{Files} in the +topmost menu bar). + +The Rackspace backend accepts the same backend options as the +{\hyperref[backends:openstack-backend]{\emph{OpenStack backend}}}. + +\begin{notice}{note}{Note:} +As of January 2012, Rackspace does not give any durability or +consistency guarantees (see {\hyperref[durability:durability]{\emph{Important Rules to Avoid Losing Data}}} for why this is +important). However, Rackspace support agents seem prone to claim +very high guarantees. Unless explicitly backed by their terms of +service, any such statement should thus be viewed with +suspicion. S3QL developers have also \href{http://www.rath.org/Tales\%20from\%20the\%20Rackspace\%20Support}{repeatedly experienced} +similar issues with the credibility and competence of the Rackspace +support. +\end{notice} + + +\section{S3 compatible} +\label{backends:rackspace}\label{backends:s3-compatible} +The S3 compatible backend allows S3QL to access any storage service +that uses the same protocol as Amazon S3. The storage URL has the form + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3c://}\PYG{n+nv}{\PYGZlt{}hostname\PYGZgt{}}\PYG{l}{:}\PYG{n+nv}{\PYGZlt{}port\PYGZgt{}}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}bucketname\PYGZgt{}}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}prefix\PYGZgt{}} +\end{Verbatim} + +Here \emph{bucketname} is the name of an (existing) bucket, and \emph{prefix} +can be an arbitrary prefix that will be prepended to all object names +used by S3QL. This allows you to store several S3QL file systems in +the same bucket. + +The S3 compatible backend accepts the following backend options: +\index{s3c\_backend command line option!no-ssl}\index{no-ssl!s3c\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-s3c_backend-arg-no-ssl}\pysigline{\bfcode{no-ssl}\code{}} +Disable encrypted (https) connections and use plain HTTP instead. + +\end{fulllineitems} + +\index{s3c\_backend command line option!ssl-ca-path=\textless{}path\textgreater{}}\index{ssl-ca-path=\textless{}path\textgreater{}!s3c\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-s3c_backend-arg-ssl-ca-path}\pysigline{\bfcode{ssl-ca-path}\code{=\textless{}path\textgreater{}}} +Instead of using the system's default certificate store, validate +the server certificate against the specified CA +certificates. \code{\textless{}path\textgreater{}} may be either a file containing +multiple certificates, or a directory containing one certificate +per file. + +\end{fulllineitems} + +\index{s3c\_backend command line option!tcp-timeout}\index{tcp-timeout!s3c\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-s3c_backend-arg-tcp-timeout}\pysigline{\bfcode{tcp-timeout}\code{}} +Specifies the timeout used for TCP connections. If no data can be +exchanged with the remote server for longer than this period, the +TCP connection is closed and re-established (default: 20 seconds). + +\end{fulllineitems} + +\index{s3c\_backend command line option!disable-expect100}\index{disable-expect100!s3c\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-s3c_backend-arg-disable-expect100}\pysigline{\bfcode{disable-expect100}\code{}} +If this option is specified, S3QL does not use the \code{Expect: +continue} header (cf. \href{http://tools.ietf.org/html/rfc2616\#section-8.2.3}{RFC2616, section 8.2.3}) when uploading +data to the server. This can be used to work around broken storage +servers that don't fully support HTTP 1.1, but may decrease +performance as object data will be transmitted to the server more +than once in some circumstances. + +\end{fulllineitems} + +\index{s3c\_backend command line option!dumb-copy}\index{dumb-copy!s3c\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-s3c_backend-arg-dumb-copy}\pysigline{\bfcode{dumb-copy}\code{}} +If this option is specified, S3QL assumes that a COPY request to +the storage server has succeeded as soon as the server returns a +\code{200 OK} status. The \href{http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html}{S3 COPY API} specifies that the +storage server may still return an error in the request body (see +the \href{https://doc.s3.amazonaws.com/proposals/copy.html}{copy proposal} for the rationale), so this +option should only be used if you are certain that your storage +server only returns \code{200 OK} when the copy operation has been +completely and successfully carried out. Using this option may be +neccessary if your storage server does not return a valid response +body for a succesfull copy operation. + +\end{fulllineitems} + + + +\section{Local} +\label{backends:id6}\label{backends:local} +S3QL is also able to store its data on the local file system. This can +be used to backup data on external media, or to access external +services that S3QL can not talk to directly (e.g., it is possible to +store data over SSH by first mounting the remote system using \href{http://fuse.sourceforge.net/sshfs.html}{sshfs} +and then using the local backend to store the data in the sshfs +mountpoint). + +The storage URL for local storage is + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{local://}\PYG{n+nv}{\PYGZlt{}path\PYGZgt{}} +\end{Verbatim} + +Note that you have to write three consecutive slashes to specify an +absolute path, e.g. \code{local:///var/archive}. Also, relative paths will +automatically be converted to absolute paths before the authentication +file (see {\hyperref[authinfo:authinfo]{\emph{Storing Authentication Information}}}) is read, i.e. if you are in the +\code{/home/john} directory and try to mount \code{local://s3ql}, the +corresponding section in the authentication file must match the +storage url \code{local:///home/john/s3ql}. + +The local backend does not accept any backend options. + + +\chapter{Important Rules to Avoid Losing Data} +\label{durability::doc}\label{durability:sshfs}\label{durability:durability}\label{durability:important-rules-to-avoid-losing-data} +Most S3QL backends store data in distributed storage systems. These +systems differ from a traditional, local hard disk in several +important ways. In order to avoid losing data, this section should be +read very carefully. + + +\section{Rules in a Nutshell} +\label{durability:rules-in-a-nutshell} +To avoid losing your data, obey the following rules: +\begin{enumerate} +\item {} +Know what durability you can expect from your chosen storage +provider. The durability describes how likely it is that a stored +object becomes damaged over time. Such data corruption can never be +prevented completely, techniques like geographic replication and +RAID storage just reduce the likelihood of it to happen (i.e., +increase the durability). + +\item {} +When choosing a backend and storage provider, keep in mind that +when using S3QL, the effective durability of the file system data +will be reduced because of S3QL's data de-duplication feature. + +\item {} +Determine your storage service's consistency window. The +consistency window that is important for S3QL is the smaller of the +times for which: +\begin{itemize} +\item {} +a newly created object may not yet be included in the list of +stored objects + +\item {} +an attempt to read a newly created object may fail with the +storage service reporting that the object does not exist + +\end{itemize} + +If \emph{one} of the above times is zero, we say that as far as S3QL is +concerned the storage service has \emph{immediate} consistency. + +If your storage provider claims that \emph{neither} of the above can +ever happen, while at the same time promising high durability, you +should choose a respectable provider instead. + +\item {} +When mounting the same file system on different computers (or on +the same computer but with different \code{-{-}cachedir} directories), +the time that passes between the first and second of invocation of +\textbf{mount.s3ql} must be at least as long as your storage +service's consistency window. If your storage service offers +immediate consistency, you do not need to wait at all. + +\item {} +Before running \textbf{fsck.s3ql} or \textbf{s3qladm}, the file system +must have been left untouched for the length of the consistency +window. If your storage service offers immediate consistency, you +do not need to wait at all. + +\end{enumerate} + +The rest of this section explains the above rules and the reasons for +them in more detail. It also contains a list of the consistency +windows for a number of larger storage providers. + + +\section{Consistency Window List} +\label{durability:consistency-window-list} +The following is a list of the consistency windows (as far as S3QL is +concerned) for a number of storage providers. This list doesn't come +with any guarantees and may be outdated. If your storage provider is +not included, or if you need more reliable information, check with +your storage provider. + +\begin{tabulary}{\linewidth}{|L|L|} +\hline +\textsf{\relax +Storage Provider +} & \textsf{\relax +Consistency Window +}\\ +\hline +Amazon S3 in the US standard region + & +No guarantees +\\ +\hline +Amazon S3 in other regions + & +Immediate +\\ +\hline +Google Storage + & +Immediate +\\ +\hline\end{tabulary} + + + +\section{Data Consistency} +\label{durability:data-consistency} +In contrast to the typical hard disk, most storage providers do not +guarantee \emph{immediate consistency} of written data. This means that: +\begin{itemize} +\item {} +after an object has been stored, requests to read this object may +still fail or return the prior contents for a little while. + +\item {} +after an object has been deleted, attempts to read it may still +return the (old) data for some time, and it may still remain in the +list of stored objects for some time. + +\item {} +after a new object has been created, it may still not be included +when retrieving the list of stored objects for some time. + +\end{itemize} + +Of course, none of this is acceptable for a file system, and S3QL +generally handles any of the above situations internally so that it +always provides a fully consistent file system to the user. However, +there are some situations where an S3QL user nevertheless needs to be +aware of the peculiarities of his chosen storage service. + +Suppose that you mount the file system, store some new data, delete +some old data and unmount it. If you then mount the file system again +right away on another computer, there is no guarantee that S3QL will +see any of the changes that the first S3QL process has made. At least +in theory it is therefore possible that on the second mount, S3QL does +not see any of the changes that you have done and presents you an ``old +version'' of the file system without them. Even worse, if you notice +the problem and unmount the file system, S3QL will upload the old +status (which S3QL necessarily has to consider as current) and thereby +permanently override the newer version (even though this change may +not become immediately visible either). S3QL uses several techniques +to reduce the likelihood of this to happen (see {\hyperref[impl_details:impl-details]{\emph{Implementation Details}}} +for more information on this), but without support from the storage +service, the possibility cannot be eliminated completely. + +The same problem of course also applies when checking the file system. +If the storage service provides S3QL with only partially updated data, +S3QL has no way to find out if this a real consistency problem that +needs to be fixed or if it is only a temporary problem that will +resolve itself automatically (because there are still changes that +have not become visible yet). + +This is where the so called \emph{consistency window} comes in. The +consistency window is the maximum time (after writing or deleting the +object) for which any of the above ``outdated responses'' may be +received. If the consistency window is zero, i.e. all changes are +immediately effective, the storage service is said to have \emph{immediate +consistency}. If the window is infinite, i.e. there is no upper bound +on the time it may take for changes to become effect, the storage +service is said to be \emph{eventually consistent}. Note that often there +are different consistency windows for the different operations. For +example, Google Storage offers immediate consistency when reading +data, but only eventual consistency for the list of stored objects. + +To prevent the problem of S3QL working with an outdated copy of the +file system data, it is therefore sufficient to simply wait for the +consistency window to pass before mounting the file system again (or +running a file system check). The length of the consistency window +changes from storage service to storage service, and if your service +is not included in the list below, you should check the web page or +ask the technical support of your storage provider. The window that is +important for S3QL is the smaller of the times for which +\begin{itemize} +\item {} +a newly created object may not yet be included in the list of +stored objects + +\item {} +an attempt to read a newly created object may fail with the +storage service reporting that the object does not exist + +\end{itemize} + +Unfortunately, many storage providers are hesitant to guarantee +anything but eventual consistency, i.e. the length of the consistency +window is potentially infinite. In that case you simply have to pick a +length that you consider ``safe enough''. For example, even though +Amazon is only guaranteeing eventual consistency, the ordinary +consistency window for data stored in S3 is just a few seconds, and +only in exceptional circumstances (i.e., core network outages) it may +rise up to hours (\href{http://forums.aws.amazon.com/message.jspa?messageID=38471\#38471}{source}). + + +\section{Data Durability} +\label{durability:backend-reliability}\label{durability:data-durability} +The durability of a storage service a measure of the average +probability of a storage object to become corrupted over time. The +lower the chance of data loss, the higher the durability. Storage +services like Amazon S3 claim to achieve a durability of up to +99.999999999\% over a year, i.e. if you store 100000000 objects for 100 +years, you can expect that at the end of that time one object will be +corrupted or lost. + +S3QL is designed to reduce redundancy and store data in the smallest +possible form. Therefore, S3QL is generally not able to compensate for +any such losses, and when choosing a storage service you should +carefully review if the offered durability matches your requirements. +When doing this, there are two factors that should be kept in mind. + +Firstly, even though S3QL is not able to compensate for storage +service failures, it is able to detect them: when trying to access +data that has been lost or corrupted by the storage service, an IO +error will be returned and the mount point will become inaccessible to +ensure that the problem is noticed. + +Secondly, the consequences of a data loss by the storage service can +be significantly more severe than you may expect because of S3QL's +data de-duplication feature: a data loss in the storage service at +time \emph{x} may cause data that is written \emph{after} time \emph{x} to be lost as +well. Consider the following scenario: +\begin{enumerate} +\item {} +You store an important file in the S3QL file system. + +\item {} +The storage service loses the data blocks of this file. As long as you +do not access the file or run \textbf{fsck.s3ql}, S3QL is not +aware that the data has been lost by the storage service. + +\item {} +You save an additional copy of the important file in a different +location on the same S3QL file system. + +\item {} +S3QL detects that the contents of the new file are identical to the +data blocks that have been stored earlier. Since at this point S3QL +is not aware that these blocks have been lost by the storage service, it +does not save another copy of the file contents in the storage service but +relies on the (presumably) existing blocks instead. + +\item {} +Therefore, even though you saved another copy, you still do not +have a backup of the important file (since both copies refer to the +same data blocks that have been lost by the storage service). + +\end{enumerate} + +For some storage services, \textbf{fsck.s3ql} can mitigate this +effect. When \textbf{fsck.s3ql} runs, it asks the storage service +for a list of all stored objects. If objects are missing, it can then +mark the damaged files and prevent the problem from spreading forwards +in time. Figuratively speaking, this establishes a ``checkpoint'': data +loss that occurred before running \textbf{fsck.s3ql} can not affect +any file system operations that are performed after the check. +Unfortunately, many storage services only ``discover'' that objects are +missing or broken when the object actually needs to be retrieved. In +this case, \textbf{fsck.s3ql} will not learn anything by just +querying the list of objects. + +This effect can be mitigated to some degree by using the +\textbf{s3ql\_verify} command in additon to +\textbf{fsck.s3ql}. \textbf{s3ql\_verify} asks the storage service +to look up every stored object and may therefore take much longer than +running \textbf{fsck.s3ql}, but can also offer a much stronger +assurance that no data has been lost by the storage service. To +``recover'' from damaged storage objects in the backend, the damaged +objects found by \textbf{s3ql\_verify} have to be explicitly deleted +(so that a successive \textbf{fsck.s3ql} is able detect them as +missing, correct the file system metadata, and move any affected files +to \code{lost+found}). This procedure is currently not automated, so +it is generally a good idea to choose a storage service where the +expected data durability is high enough so that the possibility of a +lost object (and thus the need to run any full checks) can be +neglected over long periods of time. + + +\chapter{File System Creation} +\label{mkfs::doc}\label{mkfs:file-system-creation} +A S3QL file system is created with the \textbf{mkfs.s3ql} command. It has the +following syntax: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{mkfs.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} +\end{Verbatim} + +This command accepts the following options: +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: +\code{\textasciitilde{}/.s3ql)} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}backend-options \textless{}options\textgreater{}] +Backend specific options (separate by commas). See +backend documentation for available options. +\item [-{-}version] +just print program version and exit +\item [-L \textless{}name\textgreater{}] +Filesystem label +\item [-{-}max-obj-size \textless{}size\textgreater{}] +Maximum size of storage objects in KiB. Files bigger +than this will be spread over multiple objects in the +storage backend. Default: 10240 KiB. +\item [-{-}plain] +Create unencrypted file system. +\item [-{-}force] +Overwrite any existing data. +\end{optionlist} +\end{quote} + +Unless you have specified the \code{-{-}plain} option, +\textbf{mkfs.s3ql} will ask you to enter an encryption +password. This password will \emph{not} be read from an authentication file +specified with the \code{-{-}authfile} option to prevent accidental +creation of an encrypted file system. + +Note that: +\begin{itemize} +\item {} +All data that is stored under the given storage url is assumed to +managed exclusively by S3QL. Trying to manually save additional +objects (or remove or manipulate existing objects) will lead to file +system corruption, and \textbf{fsck.s3ql} may delete objects that +do not belong to the file system. + +\item {} +With most storage backends, slashes in the storage url prefix do not +have special meaning. For example, the storage urls +\code{s3://mybucket/myprefix/} and \code{s3://mybucket/myprefix} are +distinct. In the first case, the prefix is \code{myprefix/}, while in +the second it is \code{myprefix}. + +\item {} +S3QL file systems can not be ``stacked'', i.e. you cannot have one +file system stored at \code{s3://bucketname/outerprefix} and a second +one at \code{s3://bucketname/outerprefix/innerprefix}. + +\end{itemize} + + +\chapter{Managing File Systems} +\label{adm::doc}\label{adm:managing-file-systems} +The \code{s3qladm} command performs various operations on \emph{unmounted} S3QL +file systems. The file system \emph{must not be mounted} when using +\code{s3qladm} or things will go wrong badly. + +The syntax is + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qladm }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}action\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage\PYGZhy{}url\PYGZgt{}} +\end{Verbatim} + +where \code{action} may be either of \textbf{passphrase}, +\textbf{upgrade}, \textbf{clear} or \textbf{download-metadata}. + +The \textbf{s3qladm} accepts the following general options, no +matter what specific action is being invoked: +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}log \textless{}target\textgreater{}] +Destination for log messages. Specify \code{none} for +standard output or \code{syslog} for the system logging +daemon. Anything else will be interpreted as a file +name. Log files will be rotated when they reach 1 MiB, +and at most 5 old log files will be kept. Default: +\code{None} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +\item [-{-}backend-options \textless{}options\textgreater{}] +Backend specific options (separate by commas). See +backend documentation for available options. +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: +\code{\textasciitilde{}/.s3ql)} +\item [-{-}version] +just print program version and exit +\end{optionlist} +\end{quote} + +Hint: run \code{s3qladm \textless{}action\textgreater{} -{-}help} to get help on the additional arguments +that the different actions take. + + +\section{Changing the Passphrase} +\label{adm:changing-the-passphrase} +To change the passphrase of a file system, use the \code{passphrase} +subcommand: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qladm passphrase }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} +\end{Verbatim} + + +\section{Upgrading the file system} +\label{adm:upgrading-the-file-system} +If you have installed a new version of S3QL, it may sometimes be +necessary to upgrade the file system metadata as well. Note that in +this case the file system can no longer be accessed with older +versions of S3QL after the upgrade. + +During the upgrade you have to make sure that the command is not +interrupted, and that no one else tries to mount, check or upgrade the +file system at the same time. + +To upgrade a file system from the previous to the current revision, +execute + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qladm upgrade }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} +\end{Verbatim} + + +\section{Deleting a file system} +\label{adm:deleting-a-file-system} +A file system can be deleted with: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qladm clear }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} +\end{Verbatim} + +This physically deletes all the data and file system structures. + + +\section{Restoring Metadata Backups} +\label{adm:restoring-metadata-backups} +If the most-recent copy of the file system metadata has been damaged +irreparably, it is possible to restore one of the automatically +created backup copies. + +The command + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qladm download\PYGZhy{}metadata }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} +\end{Verbatim} + +will give you a list of the available metadata backups and allow you +to download them. This will create two new files in the current +directory, ending in \code{.db} and \code{.params}. To actually use the +downloaded backup, you need to move these files into the \code{\textasciitilde{}/.s3ql/} +directory and run \code{fsck.s3ql}. + +\begin{notice}{warning}{Warning:} +You should probably not use this functionality without having asked +for help on the mailing list first (see {\hyperref[resources:resources]{\emph{Further Resources / Getting Help}}}). +\end{notice} + + +\chapter{Mounting} +\label{mount::doc}\label{mount:mounting} +A S3QL file system is mounted with the \textbf{mount.s3ql} +command. It has the following syntax: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{mount.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}} +\end{Verbatim} + +\begin{notice}{note}{Note:} +S3QL is not a network file system like \href{http://en.wikipedia.org/wiki/Network\_File\_System\_\%28protocol\%29}{NFS} +or \href{http://en.wikipedia.org/wiki/CIFS}{CIFS}. It can only be +mounted on one computer at a time. +\end{notice} + +This command accepts the following options: +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}log \textless{}target\textgreater{}] +Destination for log messages. Specify \code{none} for +standard output or \code{syslog} for the system logging +daemon. Anything else will be interpreted as a file +name. Log files will be rotated when they reach 1 MiB, +and at most 5 old log files will be kept. Default: +\code{\textasciitilde{}/.s3ql/mount.log} +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: +\code{\textasciitilde{}/.s3ql)} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}backend-options \textless{}options\textgreater{}] +Backend specific options (separate by commas). See +backend documentation for available options. +\item [-{-}version] +just print program version and exit +\item [-{-}cachesize \textless{}size\textgreater{}] +Cache size in KiB (default: autodetect). +\item [-{-}max-cache-entries \textless{}num\textgreater{}] +Maximum number of entries in cache (default: +autodetect). Each cache entry requires one file +descriptor, so if you increase this number you have to +make sure that your process file descriptor limit (as +set with \code{ulimit -n}) is high enough (at least the +number of cache entries + 100). +\item [-{-}allow-other] +Normally, only the user who called \code{mount.s3ql} can +access the mount point. This user then also has full +access to it, independent of individual file +permissions. If the \code{-{-}allow-other} option is +specified, other users can access the mount point as +well and individual file permissions are taken into +account for all users. +\item [-{-}allow-root] +Like \code{-{-}allow-other}, but restrict access to the +mounting user and the root user. +\item [-{-}fg] +Do not daemonize, stay in foreground +\item [-{-}upstart] +Stay in foreground and raise SIGSTOP once mountpoint +is up. +\item [-{-}compress \textless{}algorithm-lvl\textgreater{}] +Compression algorithm and compression level to use +when storing new data. \emph{algorithm} may be any of +\code{lzma}, \code{bzip2}, \code{zlib}, or none. \emph{lvl} may be any +integer from 0 (fastest) to 9 (slowest). Default: +\code{lzma-6} +\item [-{-}metadata-upload-interval \textless{}seconds\textgreater{}] +Interval in seconds between complete metadata uploads. +Set to 0 to disable. Default: 24h. +\item [-{-}threads \textless{}no\textgreater{}] +Number of parallel upload threads to use (default: +auto). +\item [-{-}nfs] +Enable some optimizations for exporting the file +system over NFS. (default: False) +\end{optionlist} +\end{quote} + + +\section{Permission Checking} +\label{mount:permission-checking} +If the file system is mounted with neither the \code{allow-root} +nor \code{allow-other} option, the mounting user has full +permissions on the S3QL file system (he is effectively root). If one +(or both) of the options is used, standard unix permission checks +apply, i.e. only the real root user has full access and all other +users (including the mounting user) are subject to permission checks. + + +\section{Compression Algorithms} +\label{mount:compression-algorithms} +S3QL supports three compression algorithms, LZMA, Bzip2 and zlib (with +LZMA being the default). The compression algorithm can be specified +freely whenever the file system is mounted, since it affects only the +compression of new data blocks. + +Roughly speaking, LZMA is slower but achieves better compression +ratios than Bzip2, while Bzip2 in turn is slower but achieves better +compression ratios than zlib. + +For maximum file system performance, the best algorithm therefore +depends on your network connection speed: the compression algorithm +should be fast enough to saturate your network connection. + +To find the optimal algorithm and number of parallel compression +threads for your system, S3QL ships with a program called +\code{benchmark.py} in the \code{contrib} directory. You should run this program +on a file that has a size that is roughly equal to the block size of +your file system and has similar contents. It will then determine the +compression speeds for the different algorithms and the upload speeds +for the specified backend and recommend the best algorithm that is +fast enough to saturate your network connection. + +Obviously you should make sure that there is little other system load +when you run \code{benchmark.py} (i.e., don't compile software or encode +videos at the same time). + + +\section{Notes about Caching} +\label{mount:notes-about-caching} +S3QL maintains a local cache of the file system data to speed up +access. The cache is block based, so it is possible that only parts of +a file are in the cache. + + +\subsection{Maximum Number of Cache Entries} +\label{mount:maximum-number-of-cache-entries} +The maximum size of the cache can be configured with the +\code{-{-}cachesize} option. In addition to that, the maximum number +of objects in the cache is limited by the +\code{-{-}max-cache-entries} option, so it is possible that the cache +does not grow up to the maximum cache size because the maximum number +of cache elements has been reached. The reason for this limit is that +each cache entry requires one open file descriptor, and Linux +distributions usually limit the total number of file descriptors per +process to about a thousand. + +If you specify a value for \code{-{-}max-cache-entries}, you should +therefore make sure to also configure your system to increase the +maximum number of open file handles. This can be done temporarily with +the \textbf{ulimit -n} command. The method to permanently change this limit +system-wide depends on your distribution. + + +\subsection{Cache Flushing and Expiration} +\label{mount:cache-flushing-and-expiration} +S3QL flushes changed blocks in the cache to the backend whenever a block +has not been accessed for at least 10 seconds. Note that when a block is +flushed, it still remains in the cache. + +Cache expiration (i.e., removal of blocks from the cache) is only done +when the maximum cache size is reached. S3QL always expires the least +recently used blocks first. + + +\section{Failure Modes} +\label{mount:failure-modes} +Once an S3QL file system has been mounted, there is a multitude of +problems that can occur when communicating with the remote +server. Generally, \textbf{mount.s3ql} always tries to keep the file +system as accessible as possible under the circumstances. That means +that if network connectivity is lost, data can still be written as +long as there is space in the local cache. Attempts to read data not +already present in the cache, however, will block until connection is +re-established. If any sort of data corruption is detected, the file +system will switch to read-only mode. Attempting to read files that +are affected by the corruption will return an input/output error +(\emph{errno} set to \code{EIO}). + +In case of other unexpected or fatal problems, \textbf{mount.s3ql} +terminates, but does not unmount the file system. Any attempt to +access the mountpoint will result in a ``Transport endpoint not +connected'' error (\emph{errno} set to \code{ESHUTDOWN}). This ensures that a +mountpoint whose \textbf{mount.s3ql} process has terminated can not +be confused with a mountpoint containing an empty file system (which +would be fatal if e.g. the mountpoint is automatically mirrored). When +this has happened, the mountpoint can be cleared by using the +\textbf{fusermount} command (provided by FUSE) with the \code{-u} +parameter. + +\textbf{mount.s3ql} will automatically try to re-establish the +connection to the server if network connectivity is lost, and retry +sending a request when the connection is established but the remote +server signals a temporary problem. These attempts will be made at +increasing intervals for a period up to 24 hours, with retry intervals +starting at 20 ms and increasing up to 5 minutes. After 24 hours, +\textbf{mount.s3ql} will give up and terminate, leaving the +mountpoint inaccessible as described above. + +Generally, \textbf{mount.s3ql} will also emit log messages for any +unusual conditions that it encounters. The destination for these +messages can be set with the \code{-{-}log} parameter. It is highly +recommended to periodically check these logs, for example with a tool +like \href{http://sourceforge.net/projects/logcheck/}{logcheck}. Many potential issues that \textbf{mount.s3ql} may +encounter do not justify restricting access to the file system, but +should nevertheless be investigated if they occur. Checking the log +messages is the only way to find out about them. + + +\section{Automatic Mounting} +\label{mount:logcheck}\label{mount:automatic-mounting} +If you want to mount and umount an S3QL file system automatically at +system startup and shutdown, you should do so with a dedicated S3QL +init job (instead of using \code{/etc/fstab}. When using systemd, +\textbf{mount.s3ql} can be run as a service of type \code{notify}. + +\begin{notice}{note}{Note:} +In principle, it is also possible to automatically mount an S3QL +file system with an appropriate entry in \code{/etc/fstab}. However, +this is not recommended for several reasons: +\begin{itemize} +\item {} +file systems mounted in \code{/etc/fstab} will be unmounted with the +\textbf{umount} command, so your system will not wait until all data has +been uploaded but shutdown (or restart) immediately (this is a +FUSE limitation, see \href{https://bitbucket.org/nikratio/s3ql/issue/1/blocking-fusermount-and-umount}{issue \#1}). + +\item {} +There is no way to tell the system that mounting S3QL requires a +Python interpreter to be available, so it may attempt to run +\textbf{mount.s3ql} before it has mounted the volume containing +the Python interpreter. + +\item {} +There is no standard way to tell the system that internet +connection has to be up before the S3QL file system can be +mounted. + +\end{itemize} +\end{notice} + + +\chapter{Advanced S3QL Features} +\label{special::doc}\label{special:advanced-s3ql-features} + +\section{Snapshotting and Copy-on-Write} +\label{special:s3qlcp}\label{special:snapshotting-and-copy-on-write} +The command \code{s3qlcp} can be used to duplicate a directory tree without +physically copying the file contents. This is made possible by the +data de-duplication feature of S3QL. + +The syntax of \code{s3qlcp} is: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qlcp }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}src\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}target\PYGZgt{}} +\end{Verbatim} + +This will replicate the contents of the directory \code{\textless{}src\textgreater{}} in the +directory \code{\textless{}target\textgreater{}}. \code{\textless{}src\textgreater{}} has to be an existing directory and +\code{\textless{}target\textgreater{}} must not exist. Moreover, both directories have to be +within the same S3QL file system. + +The replication will not take any additional space. Only if one of +directories is modified later on, the modified data will take +additional storage space. + +\code{s3qlcp} can only be called by the user that mounted the file system +and (if the file system was mounted with \code{-{-}allow-other} or \code{-{-}allow-root}) +the root user. + +Note that: +\begin{itemize} +\item {} +After the replication, both source and target directory will still +be completely ordinary directories. You can regard \code{\textless{}src\textgreater{}} as a +snapshot of \code{\textless{}target\textgreater{}} or vice versa. However, the most common +usage of \code{s3qlcp} is to regularly duplicate the same source +directory, say \code{documents}, to different target directories. For a +e.g. monthly replication, the target directories would typically be +named something like \code{documents\_January} for the replication in +January, \code{documents\_February} for the replication in February etc. +In this case it is clear that the target directories should be +regarded as snapshots of the source directory. + +\item {} +Exactly the same effect could be achieved by an ordinary copy +program like \code{cp -a}. However, this procedure would be orders of +magnitude slower, because \code{cp} would have to read every file +completely (so that S3QL had to fetch all the data over the network +from the backend) before writing them into the destination folder. + +\end{itemize} + + +\subsection{Snapshotting vs Hardlinking} +\label{special:snapshotting-vs-hardlinking} +Snapshot support in S3QL is inspired by the hardlinking feature that +is offered by programs like \href{http://www.samba.org/rsync}{rsync} or +\href{http://savannah.nongnu.org/projects/storebackup}{storeBackup}. +These programs can create a hardlink instead of copying a file if an +identical file already exists in the backup. However, using hardlinks +has two large disadvantages: +\begin{itemize} +\item {} +backups and restores always have to be made with a special program +that takes care of the hardlinking. The backup must not be touched +by any other programs (they may make changes that inadvertently +affect other hardlinked files) + +\item {} +special care needs to be taken to handle files which are already +hardlinked (the restore program needs to know that the hardlink was +not just introduced by the backup program to safe space) + +\end{itemize} + +S3QL snapshots do not have these problems, and they can be used with +any backup program. + + +\section{Getting Statistics} +\label{special:s3qlstat}\label{special:getting-statistics} +You can get more information about a mounted S3QL file system with the +\code{s3qlstat} command. It has the following syntax: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qlstat }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}} +\end{Verbatim} + +This will print out something like this + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{Directory entries: 1488068} +\PYG{l}{Inodes: 1482991} +\PYG{l}{Data blocks: 87948} +\PYG{l}{Total data size: 400 GiB} +\PYG{l}{After de\PYGZhy{}duplication: 51 GiB (12.98\PYGZpc{} of total)} +\PYG{l}{After compression: 43 GiB (10.85\PYGZpc{} of total, 83.60\PYGZpc{} of de\PYGZhy{}duplicated)} +\PYG{l}{Database size: 172 MiB (uncompressed)} +\PYG{l}{(some values do not take into account not\PYGZhy{}yet\PYGZhy{}uploaded dirty blocks in cache)} +\end{Verbatim} + +Probably the most interesting numbers are the total size of your data, +the total size after duplication, and the final size after +de-duplication and compression. + +\code{s3qlstat} can only be called by the user that mounted the file system +and (if the file system was mounted with \code{-{-}allow-other} or \code{-{-}allow-root}) +the root user. + +For a full list of available options, run \code{s3qlstat -{-}help}. + + +\section{Immutable Trees} +\label{special:s3qllock}\label{special:immutable-trees} +The command \textbf{s3qllock} can be used to make a directory tree +immutable. Immutable trees can no longer be changed in any way +whatsoever. You can not add new files or directories and you can not +change or delete existing files and directories. The only way to get +rid of an immutable tree is to use the \textbf{s3qlrm} command (see +below). + +For example, to make the directory tree beneath the directory +\code{2010-04-21} immutable, execute + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qllock 2010\PYGZhy{}04\PYGZhy{}21} +\end{Verbatim} + +Immutability is a feature designed for backups. Traditionally, backups +have been made on external tape drives. Once a backup was made, the +tape drive was removed and locked somewhere in a shelf. This has the +great advantage that the contents of the backup are now permanently +fixed. Nothing (short of physical destruction) can change or delete +files in the backup. + +In contrast, when backing up into an online storage system like S3QL, +all backups are available every time the file system is mounted. +Nothing prevents a file in an old backup from being changed again +later on. In the worst case, this may make your entire backup system +worthless. Imagine that your system gets infected by a nasty virus +that simply deletes all files it can find -- if the virus is active +while the backup file system is mounted, the virus will destroy all +your old backups as well! + +Even if the possibility of a malicious virus or trojan horse is +excluded, being able to change a backup after it has been made is +generally not a good idea. A common S3QL use case is to keep the file +system mounted at all times and periodically create backups with +\textbf{rsync -a}. This allows every user to recover her files from a +backup without having to call the system administrator. However, this +also allows every user to accidentally change or delete files \emph{in} one +of the old backups. + +Making a backup immutable protects you against all these problems. +Unless you happen to run into a virus that was specifically programmed +to attack S3QL file systems, backups can be neither deleted nor +changed after they have been made immutable. + + +\section{Fast Recursive Removal} +\label{special:s3qlrm}\label{special:fast-recursive-removal} +The \code{s3qlrm} command can be used to recursively delete files and +directories on an S3QL file system. Although \code{s3qlrm} is faster than +using e.g. \code{rm -r}, the main reason for its existence is that it +allows you to delete immutable trees as well. The syntax is rather +simple: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qlrm }\PYG{n+nv}{\PYGZlt{}directory\PYGZgt{}} +\end{Verbatim} + +Be warned that there is no additional confirmation. The directory will +be removed entirely and immediately. + + +\section{Runtime Configuration} +\label{special:runtime-configuration}\label{special:s3qlctrl} +The \code{s3qlctrl} can be used to control a mounted S3QL file system. Its +syntax is + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qlctrl }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}action\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}}\PYG{l}{ ...} +\end{Verbatim} + +\code{\textless{}mountpoint\textgreater{}} must be the location of a mounted S3QL file system. +For a list of valid options, run \code{s3qlctrl -{-}help}. \code{\textless{}action\textgreater{}} +may be either of: +\begin{quote} +\begin{quote}\begin{description} +\item[{flushcache}] \leavevmode +Flush file system cache. The command blocks until the cache has +been flushed. + +\item[{log}] \leavevmode +Change log level. + +\item[{cachesize}] \leavevmode +Change file system cache size. + +\item[{upload-meta}] \leavevmode +Trigger a metadata upload. + +\end{description}\end{quote} +\end{quote} + + +\chapter{Unmounting} +\label{umount::doc}\label{umount:unmounting} +To unmount an S3QL file system, use the command: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{umount.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}} +\end{Verbatim} + +This will block until all data has been written to the backend. + +Only the user who mounted the file system with \textbf{mount.s3ql} +is able to unmount it again. If you are root and want to unmount an +S3QL file system mounted by an ordinary user, you have to use the +\textbf{fusermount -u} or \textbf{umount} command instead. Note +that these commands do not block until all data has been uploaded, so +if you use them instead of \code{umount.s3ql} then you should manually wait +for the \code{mount.s3ql} process to terminate before shutting down the +system. + +The \textbf{umount.s3ql} command accepts the following options: +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}version] +just print program version and exit +\item [-{-}lazy, -z] +Lazy umount. Detaches the file system immediately, +even if there are still open files. The data will be +uploaded in the background once all open files have +been closed. +\end{optionlist} +\end{quote} + +If, for some reason, the \code{umount.sql} command does not work, the file +system can also be unmounted with \code{fusermount -u -z}. Note that this +command will return immediately and the file system may continue to +upload data in the background for a while longer. + + +\chapter{Checking for Errors} +\label{fsck::doc}\label{fsck:checking-for-errors} +It is recommended to periodically run the \textbf{fsck.s3ql} and +\textbf{s3ql\_verify} commands (in this order) to ensure that the +file system is consistent, and that there has been no data corruption +or data loss in the storage backend. + +\textbf{fsck.s3ql} is intended to detect and correct problems with +the internal file system structure, caused by e.g. a file system crash +or a bug in S3QL. It assumes that the storage backend can be fully +trusted, i.e. if the backend reports that a specific storage object +exists, \textbf{fsck.s3ql} takes that as proof that the data is +present and intact. + +In contrast to that, the \textbf{s3ql\_verify} command is intended to +check the consistency of the storage backend. It assumes that the +internal file system data is correct, and verifies that all data can +actually be retrieved from the backend. Running \textbf{s3ql\_verify} +may therefore take much longer than running \textbf{fsck.s3ql}. + + +\section{Checking and repairing internal file system errors} +\label{fsck:checking-and-repairing-internal-file-system-errors} +\textbf{fsck.s3ql} checks that the internal file system structure is +consistent and attempts to correct any problems it finds. If an S3QL +file system has not been unmounted correcly for any reason, you need +to run \textbf{fsck.s3ql} before you can mount the file system +again. + +The \textbf{fsck.s3ql} command has the following syntax: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{fsck.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} +\end{Verbatim} + +This command accepts the following options: +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}log \textless{}target\textgreater{}] +Destination for log messages. Specify \code{none} for +standard output or \code{syslog} for the system logging +daemon. Anything else will be interpreted as a file +name. Log files will be rotated when they reach 1 MiB, +and at most 5 old log files will be kept. Default: +\code{\textasciitilde{}/.s3ql/fsck.log} +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: +\code{\textasciitilde{}/.s3ql)} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}backend-options \textless{}options\textgreater{}] +Backend specific options (separate by commas). See +backend documentation for available options. +\item [-{-}version] +just print program version and exit +\item [-{-}batch] +If user input is required, exit without prompting. +\item [-{-}force] +Force checking even if file system is marked clean. +\end{optionlist} +\end{quote} + + +\section{Detecting and handling backend data corruption} +\label{fsck:detecting-and-handling-backend-data-corruption}\label{fsck:s3ql-verify} +The \textbf{s3ql\_verify} command verifies all data in the file +system. In contrast to \textbf{fsck.s3ql}, \textbf{s3ql\_verify} +does not trust the object listing returned by the backend, but +actually attempts to retrieve every object. By default, +\textbf{s3ql\_verify} will attempt to retrieve just the metadata for +every object (for e.g. the S3-compatible or Google Storage backends +this corresponds to a \code{HEAD} request for each object), which is +generally sufficient to determine if the object still exists. When +specifying the \code{-{-}data} option, \textbf{s3ql\_verify} will +instead read every object entirely. To determine how much data will be +transmitted in total when using \code{-{-}data}, look at the \emph{After +compression} row in the {\hyperref[special:s3qlstat]{\emph{s3qlstat}}} output. + +\textbf{s3ql\_verify} is not able to correct any data corruption that +it finds. Instead, a list of the corrupted and/or missing objects is +written to a file and the decision about the proper course of action +is left to the user. If you have administrative access to the backend +server, you may want to investigate the cause of the corruption or +check if the missing/corrupted objects can be restored from +backups. If you believe that the missing/corrupted objects are indeed +lost irrevocably, you can use the {\hyperref[contrib:remove-objects]{\emph{remove\_objects.py}}} script (from +the \code{contrib} directory of the S3QL distribution) to explicitly +delete the objects from the storage backend. After that, you should +run \textbf{fsck.s3ql}. Since the (now explicitly deleted) objects +should now no longer be included in the object index reported by the +backend, \textbf{fsck.s3ql} will identify the objects as missing, +update the internal file system structures accordingly, and move the +affected files into the \code{lost+found} directory. + +The \textbf{s3ql\_verify} command has the following syntax: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3ql\PYGZus{}verify }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} +\end{Verbatim} + +This command accepts the following options: +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}version] +just print program version and exit +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: +\code{\textasciitilde{}/.s3ql)} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +\item [-{-}backend-options \textless{}options\textgreater{}] +Backend specific options (separate by commas). See +backend documentation for available options. +\item [-{-}missing-file \textless{}name\textgreater{}] +File to store keys of missing objects. +\item [-{-}corrupted-file \textless{}name\textgreater{}] +File to store keys of corrupted objects. +\item [-{-}data] +Read every object completely, instead of checking just +the metadata. +\item [-{-}parallel PARALLEL] +Number of connections to use in parallel. +\item [-{-}start-with \textless{}n\textgreater{}] +Skip over first \textless{}n\textgreater{} objects and with verifying object +\textless{}n\textgreater{}+1. +\end{optionlist} +\end{quote} + + +\chapter{Storing Authentication Information} +\label{authinfo::doc}\label{authinfo:authinfo}\label{authinfo:storing-authentication-information} +Normally, S3QL reads username and password for the backend as well as +an encryption passphrase for the file system from the terminal. Most +commands also accept an \code{-{-}authfile} parameter that can be +used to read this information from a file instead. + +The authentication file consists of sections, led by a \code{{[}section{]}} +header and followed by \code{name: value} entries. The section headers +themselves are not used by S3QL but have to be unique within the file. + +In each section, the following entries can be defined: +\begin{quote}\begin{description} +\item[{storage-url}] \leavevmode +Specifies the storage url to which this section applies. If a +storage url starts with the value of this entry, the section is +considered applicable. + +\item[{backend-login}] \leavevmode +Specifies the username to use for authentication with the backend. + +\item[{backend-password}] \leavevmode +Specifies the password to use for authentication with the backend. + +\item[{fs-passphrase}] \leavevmode +Specifies the passphrase to use to decrypt the file system (if +it is encrypted). + +\end{description}\end{quote} + +When reading the authentication file, S3QL considers every applicable +section in order and uses the last value that it found for each entry. +For example, consider the following authentication file: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{g+ge}{[s3]} +\PYG{l}{storage\PYGZhy{}url: s3://} +\PYG{l}{backend\PYGZhy{}login: joe} +\PYG{l}{backend\PYGZhy{}password: notquitesecret} + +\PYG{g+ge}{[fs1]} +\PYG{l}{storage\PYGZhy{}url: s3://joes\PYGZhy{}first\PYGZhy{}bucket} +\PYG{l}{fs\PYGZhy{}passphrase: neitheristhis} + +\PYG{g+ge}{[fs2]} +\PYG{l}{storage\PYGZhy{}url: s3://joes\PYGZhy{}second\PYGZhy{}bucket} +\PYG{l}{fs\PYGZhy{}passphrase: swordfish} + +\PYG{g+ge}{[fs3]} +\PYG{l}{storage\PYGZhy{}url: s3://joes\PYGZhy{}second\PYGZhy{}bucket/with\PYGZhy{}prefix} +\PYG{l}{backend\PYGZhy{}login: bill} +\PYG{l}{backend\PYGZhy{}password: bi23ll} +\PYG{l}{fs\PYGZhy{}passphrase: ll23bi} +\end{Verbatim} + +With this authentication file, S3QL would try to log in as ``joe'' +whenever the s3 backend is used, except when accessing a storage url +that begins with ``s3://joes-second-bucket/with-prefix''. In that case, +the last section becomes active and S3QL would use the ``bill'' +credentials. Furthermore, file system encryption passphrases will be used +for storage urls that start with ``s3://joes-first-bucket'' or +``s3://joes-second-bucket''. + +The authentication file is parsed by the \href{http://docs.python.org/library/configparser.html}{Python ConfigParser +module}. + + +\chapter{Contributed Programs} +\label{contrib::doc}\label{contrib:contributed-programs} +S3QL comes with a few contributed programs that are not part of the +core distribution (and are therefore not installed automatically by +default), but which may nevertheless be useful. These programs are in +the \code{contrib} directory of the source distribution or in +\code{/usr/share/doc/s3ql/contrib} if you installed S3QL from a package. + + +\section{benchmark.py} +\label{contrib:benchmark-py} +This program measures S3QL write performance, uplink bandwidth and +compression speed to determine the limiting factor. It also gives +recommendation for compression algorithm and number of upload threads +to achieve maximum performance. + + +\section{clone\_fs.py} +\label{contrib:clone-fs-py} +This program physically clones an S3QL file system from one backend +into another, without recompressing or reencrypting. It can be used to +migrate S3 buckets to a different storage region or storage class +(standard or reduced redundancy). + + +\section{pcp.py} +\label{contrib:pcp}\label{contrib:pcp-py} +\code{pcp.py} is a wrapper program that starts several rsync processes to +copy directory trees in parallel. This is important because +transferring files in parallel significantly enhances performance when +copying data from an S3QL file system (see {\hyperref[tips:copy-performance]{\emph{Improving copy performance}}} for +details). + +To recursively copy the directory \code{/mnt/home-backup} into +\code{/home/joe} using 8 parallel processes and preserving permissions, +you would execute + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{pcp.py \PYGZhy{}a \PYGZhy{}\PYGZhy{}processes=8 /mnt/home\PYGZhy{}backup/ /home/joe} +\end{Verbatim} + + +\section{s3ql\_backup.sh} +\label{contrib:s3ql-backup-sh} +This is an example script that demonstrates how to set up a simple but +powerful backup solution using S3QL and \href{http://samba.org/rsync}{rsync}. + +The \code{s3ql\_backup.sh} script automates the following steps: +\begin{enumerate} +\item {} +Mount the file system + +\item {} +Replicate the previous backup with {\hyperref[special:s3qlcp]{\emph{s3qlcp}}} + +\item {} +Update the new copy with the data from the backup source using rsync + +\item {} +Make the new backup immutable with {\hyperref[special:s3qllock]{\emph{s3qllock}}} + +\item {} +Delete old backups that are no longer needed + +\item {} +Unmount the file system + +\end{enumerate} + +The backups are stored in directories of the form +\code{YYYY-MM-DD\_HH:mm:SS} and the {\hyperref[contrib:expire-backups-py]{expire\_backups.py}} command is used to +delete old backups. + + +\section{expire\_backups.py} +\label{contrib:expire-backups-py} +\textbf{expire\_backups.py} is a program to intelligently remove old +backups that are no longer needed. + +To define what backups you want to keep for how long, you define a +number of \emph{age ranges}. \textbf{expire\_backups} ensures that you +will have at least one backup in each age range at all times. It will +keep exactly as many backups as are required for that and delete any +backups that become redundant. + +Age ranges are specified by giving a list of range boundaries in terms +of backup cycles. Every time you create a new backup, the existing +backups age by one cycle. + +Example: when \textbf{expire\_backups} is called with the age range +definition \code{1 3 7 14 31}, it will guarantee that you always have the +following backups available: +\begin{enumerate} +\item {} +A backup that is 0 to 1 cycles old (i.e, the most recent backup) + +\item {} +A backup that is 1 to 3 cycles old + +\item {} +A backup that is 3 to 7 cycles old + +\item {} +A backup that is 7 to 14 cycles old + +\item {} +A backup that is 14 to 31 cycles old + +\end{enumerate} + +\begin{notice}{note}{Note:} +If you do backups in fixed intervals, then one cycle will be +equivalent to the backup interval. The advantage of specifying the +age ranges in terms of backup cycles rather than days or weeks is +that it allows you to gracefully handle irregular backup intervals. +Imagine that for some reason you do not turn on your computer for +one month. Now all your backups are at least a month old, and if you +had specified the above backup strategy in terms of absolute ages, +they would all be deleted! Specifying age ranges in terms of backup +cycles avoids these sort of problems. +\end{notice} + +\textbf{expire\_backups} usage is simple. It requires backups to be +stored in directories of the form \code{year-month-day\_hour:minute:seconds} +(\code{YYYY-MM-DD\_HH:mm:ss}) and works on all backups in the current +directory. So for the above backup strategy, the correct invocation +would be: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{expire\PYGZus{}backups.py 1 3 7 14 31} +\end{Verbatim} + +When storing your backups on an S3QL file system, you probably want to +specify the \code{-{-}use-s3qlrm} option as well. This tells +\textbf{expire\_backups} to use the {\hyperref[special:s3qlrm]{\emph{s3qlrm}}} command to +delete directories. + +\textbf{expire\_backups} uses a ``state file'' to keep track which +backups are how many cycles old (since this cannot be inferred from +the dates contained in the directory names). The standard name for +this state file is \code{.expire\_backups.dat}. If this file gets +damaged or deleted, \textbf{expire\_backups} no longer knows the ages +of the backups and refuses to work. In this case you can use the +\code{-{-}reconstruct-state} option to try to reconstruct the state +from the backup dates. However, the accuracy of this reconstruction +depends strongly on how rigorous you have been with making backups (it +is only completely correct if the time between subsequent backups has +always been exactly the same), so it's generally a good idea not to +tamper with the state file. + +For a full list of available options, run \textbf{expire\_backups.py +--help}. + + +\section{remove\_objects.py} +\label{contrib:remove-objects}\label{contrib:remove-objects-py} +\textbf{remove\_objects.py} is a program to remove a list of objects +from a storage backend. Since it acts on the backend-level, the +backend need not contain an S3QL file system. + + +\chapter{Tips \& Tricks} +\label{tips::doc}\label{tips:tips-tricks} + +\section{SSH Backend} +\label{tips:ssh-tipp}\label{tips:ssh-backend} +By combining S3QL's local backend with \href{http://fuse.sourceforge.net/sshfs.html}{sshfs}, it is possible to store an +S3QL file system on arbitrary SSH servers: first mount the remote +target directory into the local filesystem, + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{sshfs user@my.server.com:/mnt/s3ql /mnt/sshfs} +\end{Verbatim} + +and then give the mountpoint to S3QL as a local destination: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{mount.s3ql local:///mnt/sshfs/myfsdata /mnt/s3ql} +\end{Verbatim} + + +\section{Permanently mounted backup file system} +\label{tips:permanently-mounted-backup-file-system} +If you use S3QL as a backup file system, it can be useful to mount the +file system permanently (rather than just mounting it for a backup and +unmounting it afterwards). Especially if your file system becomes +large, this saves you long mount- and unmount times if you only want +to restore a single file. + +If you decide to do so, you should make sure to +\begin{itemize} +\item {} +Use {\hyperref[special:s3qllock]{\emph{s3qllock}}} to ensure that backups are immutable +after they have been made. + +\item {} +Call {\hyperref[special:s3qlctrl]{\emph{s3qlctrl upload-meta}}} right after a every +backup to make sure that the newest metadata is stored safely (if +you do backups often enough, this may also allow you to set the +\code{-{-}metadata-upload-interval} option of \textbf{mount.s3ql} +to zero). + +\end{itemize} + + +\section{Improving copy performance} +\label{tips:copy-performance}\label{tips:improving-copy-performance} +\begin{notice}{note}{Note:} +The following applies only when copying data \textbf{from} an S3QL file +system, \textbf{not} when copying data \textbf{to} an S3QL file system. +\end{notice} + +If you want to copy a lot of smaller files \emph{from} an S3QL file system +(e.g. for a system restore) you will probably notice that the +performance is rather bad. + +The reason for this is intrinsic to the way S3QL works. Whenever you +read a file, S3QL first has to retrieve this file over the network +from the backend. This takes a minimum amount of time (the network +latency), no matter how big or small the file is. So when you copy +lots of small files, 99\% of the time is actually spend waiting for +network data. + +Theoretically, this problem is easy to solve: you just have to copy +several files at the same time. In practice, however, almost all unix +utilities (\code{cp}, \code{rsync}, \code{tar} and friends) insist on copying +data one file at a time. This makes a lot of sense when copying data +on the local hard disk, but in case of S3QL this is really +unfortunate. + +The best workaround that has been found so far is to copy files by +starting several rsync processes at once and use exclusion rules to +make sure that they work on different sets of files. + +For example, the following script will start 3 rsync instances. The +first instance handles all filenames starting with a-f, the second the +filenames from g-l and the third covers the rest. The \code{+ */} rule +ensures that every instance looks into all directories. + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{c}{\PYGZsh{}!/bin/bash} + +\PYG{l}{RSYNC\PYGZus{}ARGS=\PYGZdq{}\PYGZhy{}aHv /mnt/s3ql/ /home/restore/\PYGZdq{}} + +\PYG{l}{rsync \PYGZhy{}f \PYGZdq{}+ */\PYGZdq{} \PYGZhy{}f \PYGZdq{}\PYGZhy{}! }\PYG{g+ge}{[a\PYGZhy{}f]}\PYG{l}{*\PYGZdq{} \PYGZdl{}RSYNC\PYGZus{}ARGS \PYGZam{}} +\PYG{l}{rsync \PYGZhy{}f \PYGZdq{}+ */\PYGZdq{} \PYGZhy{}f \PYGZdq{}\PYGZhy{}! }\PYG{g+ge}{[g\PYGZhy{}l]}\PYG{l}{*\PYGZdq{} \PYGZdl{}RSYNC\PYGZus{}ARGS \PYGZam{}} +\PYG{l}{rsync \PYGZhy{}f \PYGZdq{}+ */\PYGZdq{} \PYGZhy{}f \PYGZdq{}\PYGZhy{} }\PYG{g+ge}{[a\PYGZhy{}l]}\PYG{l}{*\PYGZdq{} \PYGZdl{}RSYNC\PYGZus{}ARGS \PYGZam{}} + +\PYG{l}{wait} +\end{Verbatim} + +The optimum number of parallel processes depends on your network +connection and the size of the files that you want to transfer. +However, starting about 10 processes seems to be a good compromise +that increases performance dramatically in almost all situations. + +S3QL comes with a script named \code{pcp.py} in the \code{contrib} directory +that can be used to transfer files in parallel without having to write +an explicit script first. See the description of {\hyperref[contrib:pcp]{\emph{pcp.py}}} for +details. + + +\chapter{Known Issues} +\label{issues::doc}\label{issues:known-issues}\begin{itemize} +\item {} +S3QL de-duplicates data blocks based solely only on SHA256 +checksums, without doing a byte-by-byte comparison of the blocks. +Since it is possible for two data blocks to have the same checksum +despite having different contents, this can lead to problems. If two +such blocks are stored in an S3QL file system, the data in one block +will be lost and replaced by the data in the other block. However, +the chances of this occuring for any two blocks are about 1 in 10\textasciicircum{}77 +(2\textasciicircum{}256). For a file system that holds a total of 10\textasciicircum{}34 blocks, the +chances of a collision increase to about 1 in 10\textasciicircum{}9. Storing more +than 10\textasciicircum{}34 blocks (or about 10\textasciicircum{}25 TB with an (extremely small) block +size of 4 kB) is therefore not recommended. Being exceptionally +unlucky may also be a disadvantage. + +\item {} +S3QL does not support Access Control Lists (ACLs). This is due to a +bug in the FUSE library and will therefore hopefully be fixed at +some point. See \href{https://bitbucket.org/nikratio/s3ql/issue/16/support-access-control-lists-acls}{issue \#16} +for more details. + +\item {} +As of Linux kernel 3.5 S3QL file systems do not implement the ``write +protect'' bit on directories. In other words, even if a directory has +the write protect bit set, the owner of the directory can delete any +files and (empty) subdirectories inside it. This is a bug in the +FUSE kernel module +(cf. \href{https://github.com/libfuse/libfuse/issues/23}{https://github.com/libfuse/libfuse/issues/23}) and needs to be +fixed in the kernel. Unfortunately it does not look as if this is +going to be fixed anytime soon (as of 2016/2/28). + +\item {} +S3QL is rather slow when an application tries to write data in +unreasonably small chunks. If a 1 MiB file is copied in chunks of 1 +KB, this will take more than 10 times as long as when it's copied +with the (recommended) chunk size of 128 KiB. + +This is a limitation of the FUSE library (which does not yet support +write caching) which will hopefully be addressed in some future FUSE +version. + +Most applications, including e.g. GNU \code{cp} and \code{rsync}, use +reasonably large buffers and are therefore not affected by this +problem and perform very efficient on S3QL file systems. + +However, if you encounter unexpectedly slow performance with a +specific program, this might be due to the program using very small +write buffers. Although this is not really a bug in the program, +it might be worth to ask the program's authors for help. + +\item {} +S3QL always updates file and directory access times as if the \code{relatime} +mount option has been specified: the access time (``atime'') is only updated +if it is currently earlier than either the status change time +(``ctime'') or modification time (``mtime''). + +\item {} +S3QL directories always have an \code{st\_nlink} value of 1. This may confuse +programs that rely on directories having \code{st\_nlink} values of \emph{(2 + +number of sub directories)}. + +Note that this is not a bug in S3QL. Including sub directories in +the \code{st\_nlink} value is a Unix convention, but by no means a +requirement. If an application blindly relies on this convention +being followed, then this is a bug in the application. + +A prominent example are early versions of GNU find, which required +the \code{-{-}noleaf} option to work correctly on S3QL file systems. This +bug has already been fixed in recent find versions. + +\item {} +The \code{umount} and \code{fusermount -u} commands will \emph{not} block until all +data has been uploaded to the backend. (this is a FUSE limitation +that will hopefully be removed in the future, see \href{https://bitbucket.org/nikratio/s3ql/issue/1/blocking-fusermount-and-umount}{issue \#1}). If +you use either command to unmount an S3QL file system, you have to +take care to explicitly wait for the \code{mount.s3ql} process to +terminate before you shut down or restart the system. Therefore it +is generally not a good idea to mount an S3QL file system in +\code{/etc/fstab} (you should use a dedicated init script instead). + +\item {} +S3QL relies on the backends not to run out of space. This is a given +for big storage providers like Amazon S3 or Google Storage, but you +may stumble upon this if you use your own server or smaller providers. + +If there is no space left in the backend, attempts to write more +data into the S3QL file system will fail and the file system will be +in an inconsistent state and require a file system check (and you +should make sure to make space available in the backend before +running the check). + +Unfortunately, there is no way to handle insufficient space in the +backend without leaving the file system inconsistent. Since +S3QL first writes data into the cache, it can no longer return an +error when it later turns out that the cache can not be committed to +the backend. + +\item {} +When using python-dugong versions 3.3 or earlier, S3QL supports only +CONNECT-style proxying, which may cause issues with some proxy +servers when using plain HTTP. Upgrading to python-dugong 3.4 or +newer removes this limitation. + +\end{itemize} + + +\chapter{Manpages} +\label{man/index::doc}\label{man/index:manpages} +The man pages are installed with S3QL on your system and can be viewed +with the \textbf{man} command. For reference, they are also included +here in the User's Guide. + + +\section{The \textbf{mkfs.s3ql} command} +\label{man/mkfs::doc}\label{man/mkfs:the-command-command} + +\subsection{Synopsis} +\label{man/mkfs:synopsis} +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{mkfs.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} +\end{Verbatim} + + +\subsection{Description} +\label{man/mkfs:description} +The \textbf{mkfs.s3ql} command creates a new file system in the location +specified by \emph{storage url}. The storage url depends on the backend +that is used. The S3QL User's Guide should be consulted for a +description of the available backends. + +Unless you have specified the \code{-{-}plain} option, \code{mkfs.s3ql} will ask +you to enter an encryption password. This password will \emph{not} be read +from an authentication file specified with the \code{-{-}authfile} +option to prevent accidental creation of an encrypted file system. + + +\subsection{Options} +\label{man/mkfs:options} +The \textbf{mkfs.s3ql} command accepts the following options. +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: +\code{\textasciitilde{}/.s3ql)} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}backend-options \textless{}options\textgreater{}] +Backend specific options (separate by commas). See +backend documentation for available options. +\item [-{-}version] +just print program version and exit +\item [-L \textless{}name\textgreater{}] +Filesystem label +\item [-{-}max-obj-size \textless{}size\textgreater{}] +Maximum size of storage objects in KiB. Files bigger +than this will be spread over multiple objects in the +storage backend. Default: 10240 KiB. +\item [-{-}plain] +Create unencrypted file system. +\item [-{-}force] +Overwrite any existing data. +\end{optionlist} +\end{quote} + + +\subsection{Exit Codes} +\label{man/mkfs:exit-codes} +\textbf{mkfs.s3ql} may terminate with the following exit codes: +\begin{quote}\begin{description} +\item[{0}] \leavevmode +Everything went well. + +\item[{1}] \leavevmode +An unexpected error occured. This may indicate a bug in the +program. + +\item[{2}] \leavevmode +Invalid command line argument. + +\item[{3}] \leavevmode +Invalid backend option. + +\item[{11}] \leavevmode +No such backend. + +\item[{12}] \leavevmode +Authentication file has insecure permissions. + +\item[{13}] \leavevmode +Unable to parse proxy settings. + +\item[{14}] \leavevmode +Invalid credentials (Authentication failed). + +\item[{15}] \leavevmode +No permission to access backend (Authorization denied). + +\item[{16}] \leavevmode +Invalid storage URL, specified location does not exist in backend. + +\item[{19}] \leavevmode +Unable to connect to backend, can't resolve hostname. + +\item[{45}] \leavevmode +Unable to access cache directory. + +\end{description}\end{quote} + + +\subsection{See Also} +\label{man/mkfs:see-also} +The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. + +The full S3QL documentation should also be installed somewhere on your +system, common locations are \code{/usr/share/doc/s3ql} or +\code{/usr/local/doc/s3ql}. + + +\section{The \textbf{s3qladm} command} +\label{man/adm::doc}\label{man/adm:the-command-command} + +\subsection{Synopsis} +\label{man/adm:synopsis} +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qladm }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}action\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} +\end{Verbatim} + +where \code{action} may be either of \textbf{passphrase}, +\textbf{upgrade}, \textbf{delete} or \textbf{download-metadata}. + + +\subsection{Description} +\label{man/adm:description} +The \textbf{s3qladm} command performs various operations on \emph{unmounted} S3QL +file systems. The file system \emph{must not be mounted} when using +\textbf{s3qladm} or things will go wrong badly. + +The storage url depends on the backend that is used. The S3QL User's +Guide should be consulted for a description of the available backends. + + +\subsection{Options} +\label{man/adm:options} +The \textbf{s3qladm} command accepts the following options. +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}log \textless{}target\textgreater{}] +Destination for log messages. Specify \code{none} for +standard output or \code{syslog} for the system logging +daemon. Anything else will be interpreted as a file +name. Log files will be rotated when they reach 1 MiB, +and at most 5 old log files will be kept. Default: +\code{None} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +\item [-{-}backend-options \textless{}options\textgreater{}] +Backend specific options (separate by commas). See +backend documentation for available options. +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: +\code{\textasciitilde{}/.s3ql)} +\item [-{-}version] +just print program version and exit +\end{optionlist} +\end{quote} + +Hint: run \code{s3qladm \textless{}action\textgreater{} -{-}help} to get help on the additional arguments +that the different actions take. + + +\subsection{Actions} +\label{man/adm:actions} +The following actions may be specified: +\begin{description} +\item[{passphrase}] \leavevmode +Changes the encryption passphrase of the file system. + +\item[{upgrade}] \leavevmode +Upgrade the file system to the newest revision. + +\item[{delete}] \leavevmode +Delete the file system with all the stored data. + +\item[{download-metadata}] \leavevmode +Interactively download backups of the file system metadata. + +\end{description} + + +\subsection{Exit Codes} +\label{man/adm:exit-codes} +\textbf{s3qladm} may terminate with the following exit codes: +\begin{quote}\begin{description} +\item[{0}] \leavevmode +Everything went well. + +\item[{1}] \leavevmode +An unexpected error occured. This may indicate a bug in the +program. + +\item[{2}] \leavevmode +Invalid command line argument. + +\item[{3}] \leavevmode +Invalid backend option. + +\item[{10}] \leavevmode +Could not open log file for writing. + +\item[{11}] \leavevmode +No such backend. + +\item[{12}] \leavevmode +Authentication file has insecure permissions. + +\item[{13}] \leavevmode +Unable to parse proxy settings. + +\item[{14}] \leavevmode +Invalid credentials (Authentication failed). + +\item[{15}] \leavevmode +No permission to access backend (Authorization denied). + +\item[{16}] \leavevmode +Invalid storage URL, specified location does not exist in backend. + +\item[{17}] \leavevmode +Wrong file system passphrase. + +\item[{18}] \leavevmode +No S3QL file system found at given storage URL. + +\item[{19}] \leavevmode +Unable to connect to backend, can't resolve hostname. + +\item[{45}] \leavevmode +Unable to access cache directory. + +\end{description}\end{quote} + + +\subsection{See Also} +\label{man/adm:see-also} +The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. + +The full S3QL documentation should also be installed somewhere on your +system, common locations are \code{/usr/share/doc/s3ql} or +\code{/usr/local/doc/s3ql}. + + +\section{The \textbf{mount.s3ql} command} +\label{man/mount::doc}\label{man/mount:the-command-command} + +\subsection{Synopsis} +\label{man/mount:synopsis} +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{mount.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mount point\PYGZgt{}} +\end{Verbatim} + + +\subsection{Description} +\label{man/mount:description} +The \textbf{mount.s3ql} command mounts the S3QL file system stored in \emph{storage +url} in the directory \emph{mount point}. The storage url depends on the +backend that is used. The S3QL User's Guide should be consulted for a +description of the available backends. + + +\subsection{Options} +\label{man/mount:options} +The \textbf{mount.s3ql} command accepts the following options. +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}log \textless{}target\textgreater{}] +Destination for log messages. Specify \code{none} for +standard output or \code{syslog} for the system logging +daemon. Anything else will be interpreted as a file +name. Log files will be rotated when they reach 1 MiB, +and at most 5 old log files will be kept. Default: +\code{\textasciitilde{}/.s3ql/mount.log} +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: +\code{\textasciitilde{}/.s3ql)} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}backend-options \textless{}options\textgreater{}] +Backend specific options (separate by commas). See +backend documentation for available options. +\item [-{-}version] +just print program version and exit +\item [-{-}cachesize \textless{}size\textgreater{}] +Cache size in KiB (default: autodetect). +\item [-{-}max-cache-entries \textless{}num\textgreater{}] +Maximum number of entries in cache (default: +autodetect). Each cache entry requires one file +descriptor, so if you increase this number you have to +make sure that your process file descriptor limit (as +set with \code{ulimit -n}) is high enough (at least the +number of cache entries + 100). +\item [-{-}allow-other] +Normally, only the user who called \code{mount.s3ql} can +access the mount point. This user then also has full +access to it, independent of individual file +permissions. If the \code{-{-}allow-other} option is +specified, other users can access the mount point as +well and individual file permissions are taken into +account for all users. +\item [-{-}allow-root] +Like \code{-{-}allow-other}, but restrict access to the +mounting user and the root user. +\item [-{-}fg] +Do not daemonize, stay in foreground +\item [-{-}upstart] +Stay in foreground and raise SIGSTOP once mountpoint +is up. +\item [-{-}compress \textless{}algorithm-lvl\textgreater{}] +Compression algorithm and compression level to use +when storing new data. \emph{algorithm} may be any of +\code{lzma}, \code{bzip2}, \code{zlib}, or none. \emph{lvl} may be any +integer from 0 (fastest) to 9 (slowest). Default: +\code{lzma-6} +\item [-{-}metadata-upload-interval \textless{}seconds\textgreater{}] +Interval in seconds between complete metadata uploads. +Set to 0 to disable. Default: 24h. +\item [-{-}threads \textless{}no\textgreater{}] +Number of parallel upload threads to use (default: +auto). +\item [-{-}nfs] +Enable some optimizations for exporting the file +system over NFS. (default: False) +\end{optionlist} +\end{quote} + + +\subsection{Exit Codes} +\label{man/mount:exit-codes} +\textbf{mount.s3ql} may terminate with the following exit codes: +\begin{quote}\begin{description} +\item[{0}] \leavevmode +Everything went well. + +\item[{1}] \leavevmode +An unexpected error occured. This may indicate a bug in the +program. + +\item[{2}] \leavevmode +Invalid command line argument. + +\item[{3}] \leavevmode +Invalid backend option. + +\item[{10}] \leavevmode +Could not open log file for writing. + +\item[{11}] \leavevmode +No such backend. + +\item[{12}] \leavevmode +Authentication file has insecure permissions. + +\item[{13}] \leavevmode +Unable to parse proxy settings. + +\item[{14}] \leavevmode +Invalid credentials (Authentication failed). + +\item[{15}] \leavevmode +No permission to access backend (Authorization denied). + +\item[{16}] \leavevmode +Invalid storage URL, specified location does not exist in backend. + +\item[{17}] \leavevmode +Wrong file system passphrase. + +\item[{18}] \leavevmode +No S3QL file system found at given storage URL. + +\item[{19}] \leavevmode +Unable to connect to backend, can't resolve hostname. + +\item[{30}] \leavevmode +File system was not unmounted cleanly. + +\item[{31}] \leavevmode +File system appears to be mounted elsewhere. + +\item[{32}] \leavevmode +Unsupported file system revision (too old). + +\item[{33}] \leavevmode +Unsupported file system revision (too new). + +\item[{34}] \leavevmode +Insufficient free nodes, need to run \textbf{fsck.s3ql}. + +\item[{35}] \leavevmode +Attempted to mount read-only, this is not supported. + +\item[{36}] \leavevmode +Mountpoint does not exist. + +\item[{37}] \leavevmode +Not enough available file descriptors. + +\item[{39}] \leavevmode +Unable to bind file system to mountpoint. + +\item[{45}] \leavevmode +Unable to access cache directory. + +\end{description}\end{quote} + + +\subsection{See Also} +\label{man/mount:see-also} +The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. + +The full S3QL documentation should also be installed somewhere on your +system, common locations are \code{/usr/share/doc/s3ql} or +\code{/usr/local/doc/s3ql}. + + +\section{The \textbf{s3qlstat} command} +\label{man/stat::doc}\label{man/stat:the-command-command} + +\subsection{Synopsis} +\label{man/stat:synopsis} +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qlstat }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}} +\end{Verbatim} + + +\subsection{Description} +\label{man/stat:description} +The \textbf{s3qlstat} command prints statistics about the S3QL file system mounted +at \code{mountpoint}. + +\textbf{s3qlstat} can only be called by the user that mounted the file system +and (if the file system was mounted with \code{-{-}allow-other} or +\code{-{-}allow-root}) the root user. + + +\subsection{Options} +\label{man/stat:options} +The \textbf{s3qlstat} command accepts the following options: +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}version] +just print program version and exit +\item [-{-}raw] +Do not pretty-print numbers +\end{optionlist} +\end{quote} + + +\subsection{Exit Codes} +\label{man/stat:exit-codes} +\textbf{s3qlstat} may terminate with the following exit codes: +\begin{quote}\begin{description} +\item[{0}] \leavevmode +Everything went well. + +\item[{1}] \leavevmode +An unexpected error occured. This may indicate a bug in the +program. + +\item[{2}] \leavevmode +Invalid command line argument. + +\end{description}\end{quote} + + +\subsection{See Also} +\label{man/stat:see-also} +The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. + +The full S3QL documentation should also be installed somewhere on your +system, common locations are \code{/usr/share/doc/s3ql} or +\code{/usr/local/doc/s3ql}. + + +\section{The \textbf{s3qlctrl} command} +\label{man/ctrl::doc}\label{man/ctrl:the-command-command} + +\subsection{Synopsis} +\label{man/ctrl:synopsis} +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qlctrl }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}action\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}}\PYG{l}{ ...} +\end{Verbatim} + +where \code{action} may be either of \textbf{flushcache}, +\textbf{upload-meta}, \textbf{cachesize} or +\textbf{log-metadata}. + + +\subsection{Description} +\label{man/ctrl:description} +The \textbf{s3qlctrl} command performs various actions on the S3QL file system mounted +in \code{mountpoint}. + +\textbf{s3qlctrl} can only be called by the user that mounted the file system +and (if the file system was mounted with \code{-{-}allow-other} or +\code{-{-}allow-root}) the root user. + +The following actions may be specified: +\begin{description} +\item[{flushcache}] \leavevmode +Uploads all changed file data to the backend. + +\item[{upload-meta}] \leavevmode +Upload metadata to the backend. All file system operations will +block while a snapshot of the metadata is prepared for upload. + +\item[{cachesize}] \leavevmode +Changes the cache size of the file system. This action requires an +additional argument that specifies the new cache size in KiB, so the +complete command line is: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qlctrl }\PYG{g+ge}{[options]}\PYG{l}{ cachesize }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}new\PYGZhy{}cache\PYGZhy{}size\PYGZgt{}} +\end{Verbatim} + +\item[{log}] \leavevmode +Change the amount of information that is logged into +\code{\textasciitilde{}/.s3ql/mount.log} file. The complete syntax is: + +\begin{Verbatim}[commandchars=\\\{\}] +s3qlctrl [options] log \PYGZlt{}mountpoint\PYGZgt{} \PYGZlt{}level\PYGZgt{} [\PYGZlt{}module\PYGZgt{} [\PYGZlt{}module\PYGZgt{} ...]] +\end{Verbatim} + +here \code{level} is the desired new log level and may be either of +\emph{debug}, \emph{info} or \emph{warn}. One or more \code{module} may only be +specified with the \emph{debug} level and allow to restrict the debug +output to just the listed modules. + +\end{description} + + +\subsection{Options} +\label{man/ctrl:options} +The \textbf{s3qlctrl} command also accepts the following options, no matter +what specific action is being invoked: +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}version] +just print program version and exit +\end{optionlist} +\end{quote} + +Hint: run \code{s3qlctrl \textless{}action\textgreater{} -{-}help} to get help on the additional arguments +that the different actions take. + + +\subsection{Exit Codes} +\label{man/ctrl:exit-codes} +\textbf{s3qlctrl} may terminate with the following exit codes: +\begin{quote}\begin{description} +\item[{0}] \leavevmode +Everything went well. + +\item[{1}] \leavevmode +An unexpected error occured. This may indicate a bug in the +program. + +\item[{2}] \leavevmode +Invalid command line argument. + +\end{description}\end{quote} + + +\subsection{See Also} +\label{man/ctrl:see-also} +The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. + +The full S3QL documentation should also be installed somewhere on your +system, common locations are \code{/usr/share/doc/s3ql} or +\code{/usr/local/doc/s3ql}. + + +\section{The \textbf{s3qlcp} command} +\label{man/cp::doc}\label{man/cp:the-command-command} + +\subsection{Synopsis} +\label{man/cp:synopsis} +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qlcp }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}source\PYGZhy{}dir\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}dest\PYGZhy{}dir\PYGZgt{}} +\end{Verbatim} + + +\subsection{Description} +\label{man/cp:description} +The \textbf{s3qlcp} command duplicates the directory tree \code{source-dir} +into \code{dest-dir} without physically copying the file contents. +Both source and destination must lie inside the same S3QL file system. + +The replication will not take any additional space. Only if one of +directories is modified later on, the modified data will take +additional storage space. + +\code{s3qlcp} can only be called by the user that mounted the file system +and (if the file system was mounted with \code{-{-}allow-other} or \code{-{-}allow-root}) +the root user. + +Note that: +\begin{itemize} +\item {} +After the replication, both source and target directory will still +be completely ordinary directories. You can regard \code{\textless{}src\textgreater{}} as a +snapshot of \code{\textless{}target\textgreater{}} or vice versa. However, the most common +usage of \code{s3qlcp} is to regularly duplicate the same source +directory, say \code{documents}, to different target directories. For a +e.g. monthly replication, the target directories would typically be +named something like \code{documents\_January} for the replication in +January, \code{documents\_February} for the replication in February etc. +In this case it is clear that the target directories should be +regarded as snapshots of the source directory. + +\item {} +Exactly the same effect could be achieved by an ordinary copy +program like \code{cp -a}. However, this procedure would be orders of +magnitude slower, because \code{cp} would have to read every file +completely (so that S3QL had to fetch all the data over the network +from the backend) before writing them into the destination folder. + +\end{itemize} + + +\subsubsection{Snapshotting vs Hardlinking} +\label{man/cp:snapshotting-vs-hardlinking} +Snapshot support in S3QL is inspired by the hardlinking feature that +is offered by programs like \href{http://www.samba.org/rsync}{rsync} or +\href{http://savannah.nongnu.org/projects/storebackup}{storeBackup}. +These programs can create a hardlink instead of copying a file if an +identical file already exists in the backup. However, using hardlinks +has two large disadvantages: +\begin{itemize} +\item {} +backups and restores always have to be made with a special program +that takes care of the hardlinking. The backup must not be touched +by any other programs (they may make changes that inadvertently +affect other hardlinked files) + +\item {} +special care needs to be taken to handle files which are already +hardlinked (the restore program needs to know that the hardlink was +not just introduced by the backup program to safe space) + +\end{itemize} + +S3QL snapshots do not have these problems, and they can be used with +any backup program. + + +\subsection{Options} +\label{man/cp:options} +The \textbf{s3qlcp} command accepts the following options: +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}version] +just print program version and exit +\end{optionlist} +\end{quote} + + +\subsection{Exit Codes} +\label{man/cp:exit-codes} +\textbf{s3qlcp} may terminate with the following exit codes: +\begin{quote}\begin{description} +\item[{0}] \leavevmode +Everything went well. + +\item[{1}] \leavevmode +An unexpected error occured. This may indicate a bug in the +program. + +\item[{2}] \leavevmode +Invalid command line argument. + +\end{description}\end{quote} + + +\subsection{See Also} +\label{man/cp:see-also} +The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. + +The full S3QL documentation should also be installed somewhere on your +system, common locations are \code{/usr/share/doc/s3ql} or +\code{/usr/local/doc/s3ql}. + + +\section{The \textbf{s3qlrm} command} +\label{man/rm::doc}\label{man/rm:the-command-command} + +\subsection{Synopsis} +\label{man/rm:synopsis} +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qlrm }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}directory\PYGZgt{}} +\end{Verbatim} + + +\subsection{Description} +\label{man/rm:description} +The \textbf{s3qlrm} command recursively deletes files and directories on an +S3QL file system. Although \textbf{s3qlrm} is faster than using e.g. +\textbf{rm -r{}`}, the main reason for its existence is that it allows +you to delete immutable trees (which can be created with +\textbf{s3qllock}) as well. + +Be warned that there is no additional confirmation. The directory will +be removed entirely and immediately. + +\textbf{s3qlrm} can only be called by the user that mounted the file system +and (if the file system was mounted with \code{-{-}allow-other} or +\code{-{-}allow-root}) the root user. + + +\subsection{Options} +\label{man/rm:options} +The \textbf{s3qlrm} command accepts the following options: +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}version] +just print program version and exit +\end{optionlist} +\end{quote} + + +\subsection{Exit Codes} +\label{man/rm:exit-codes} +\textbf{s3qlrm} may terminate with the following exit codes: +\begin{quote}\begin{description} +\item[{0}] \leavevmode +Everything went well. + +\item[{1}] \leavevmode +An unexpected error occured. This may indicate a bug in the +program. + +\item[{2}] \leavevmode +Invalid command line argument. + +\end{description}\end{quote} + + +\subsection{See Also} +\label{man/rm:see-also} +The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. + +The full S3QL documentation should also be installed somewhere on your +system, common locations are \code{/usr/share/doc/s3ql} or +\code{/usr/local/doc/s3ql}. + + +\section{The \textbf{s3qllock} command} +\label{man/lock::doc}\label{man/lock:the-command-command} + +\subsection{Synopsis} +\label{man/lock:synopsis} +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3qllock }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}directory\PYGZgt{}} +\end{Verbatim} + + +\subsection{Description} +\label{man/lock:description} +The \textbf{s3qllock} command makes a directory tree in an S3QL file +system immutable. Immutable trees can no longer be changed in any way +whatsoever. You can not add new files or directories and you can not +change or delete existing files and directories. The only way to get +rid of an immutable tree is to use the \textbf{s3qlrm} command. + +\textbf{s3qllock} can only be called by the user that mounted the file system +and (if the file system was mounted with \code{-{-}allow-other} or +\code{-{-}allow-root}) the root user. + + +\subsection{Rationale} +\label{man/lock:rationale} +Immutability is a feature designed for backups. Traditionally, backups +have been made on external tape drives. Once a backup was made, the +tape drive was removed and locked somewhere in a shelf. This has the +great advantage that the contents of the backup are now permanently +fixed. Nothing (short of physical destruction) can change or delete +files in the backup. + +In contrast, when backing up into an online storage system like S3QL, +all backups are available every time the file system is mounted. +Nothing prevents a file in an old backup from being changed again +later on. In the worst case, this may make your entire backup system +worthless. Imagine that your system gets infected by a nasty virus +that simply deletes all files it can find -- if the virus is active +while the backup file system is mounted, the virus will destroy all +your old backups as well! + +Even if the possibility of a malicious virus or trojan horse is +excluded, being able to change a backup after it has been made is +generally not a good idea. A common S3QL use case is to keep the file +system mounted at all times and periodically create backups with +\textbf{rsync -a}. This allows every user to recover her files from a +backup without having to call the system administrator. However, this +also allows every user to accidentally change or delete files \emph{in} one +of the old backups. + +Making a backup immutable protects you against all these problems. +Unless you happen to run into a virus that was specifically programmed +to attack S3QL file systems, backups can be neither deleted nor +changed after they have been made immutable. + + +\subsection{Options} +\label{man/lock:options} +The \textbf{s3qllock} command accepts the following options: +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}version] +just print program version and exit +\end{optionlist} +\end{quote} + + +\subsection{Exit Codes} +\label{man/lock:exit-codes} +\textbf{s3qllock} may terminate with the following exit codes: +\begin{quote}\begin{description} +\item[{0}] \leavevmode +Everything went well. + +\item[{1}] \leavevmode +An unexpected error occured. This may indicate a bug in the +program. + +\item[{2}] \leavevmode +Invalid command line argument. + +\end{description}\end{quote} + + +\subsection{See Also} +\label{man/lock:see-also} +The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. + +The full S3QL documentation should also be installed somewhere on your +system, common locations are \code{/usr/share/doc/s3ql} or +\code{/usr/local/doc/s3ql}. + + +\section{The \textbf{umount.s3ql} command} +\label{man/umount::doc}\label{man/umount:the-command-command} + +\subsection{Synopsis} +\label{man/umount:synopsis} +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{umount.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mount point\PYGZgt{}} +\end{Verbatim} + + +\subsection{Description} +\label{man/umount:description} +The \textbf{umount.s3ql} command unmounts the S3QL file system mounted in the +directory \emph{mount point} and blocks until all data has been uploaded to +the storage backend. + +Only the user who mounted the file system with \textbf{mount.s3ql} +is able to unmount it with \textbf{umount.s3ql}. If you are root and want to +unmount an S3QL file system mounted by an ordinary user, you have to +use the \textbf{fusermount -u} or \textbf{umount} command instead. +Note that these commands do not block until all data has been +uploaded, so if you use them instead of \textbf{umount.s3ql} then +you should manually wait for the \textbf{mount.s3ql} process to +terminate before shutting down the system. + + +\subsection{Options} +\label{man/umount:options} +The \textbf{umount.s3ql} command accepts the following options. +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}version] +just print program version and exit +\item [-{-}lazy, -z] +Lazy umount. Detaches the file system immediately, +even if there are still open files. The data will be +uploaded in the background once all open files have +been closed. +\end{optionlist} +\end{quote} + + +\subsection{Exit Codes} +\label{man/umount:exit-codes} +\textbf{umount.s3ql} may terminate with the following exit codes: +\begin{quote}\begin{description} +\item[{0}] \leavevmode +Everything went well. + +\item[{1}] \leavevmode +An unexpected error occured. This may indicate a bug in the +program. + +\item[{2}] \leavevmode +Invalid command line argument. + +\end{description}\end{quote} + + +\subsection{See Also} +\label{man/umount:see-also} +The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. + +The full S3QL documentation should also be installed somewhere on your +system, common locations are \code{/usr/share/doc/s3ql} or +\code{/usr/local/doc/s3ql}. + + +\section{The \textbf{fsck.s3ql} command} +\label{man/fsck::doc}\label{man/fsck:the-command-command} + +\subsection{Synopsis} +\label{man/fsck:synopsis} +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{fsck.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} +\end{Verbatim} + + +\subsection{Description} +\label{man/fsck:description} +The \textbf{fsck.s3ql} command checks the file system in the location specified +by \emph{storage url} for errors and attempts to repair any problems. The +storage url depends on the backend that is used. The S3QL User's Guide +should be consulted for a description of the available backends. + + +\subsection{Options} +\label{man/fsck:options} +The \textbf{fsck.s3ql} command accepts the following options. +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}log \textless{}target\textgreater{}] +Destination for log messages. Specify \code{none} for +standard output or \code{syslog} for the system logging +daemon. Anything else will be interpreted as a file +name. Log files will be rotated when they reach 1 MiB, +and at most 5 old log files will be kept. Default: +\code{\textasciitilde{}/.s3ql/fsck.log} +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: +\code{\textasciitilde{}/.s3ql)} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}backend-options \textless{}options\textgreater{}] +Backend specific options (separate by commas). See +backend documentation for available options. +\item [-{-}version] +just print program version and exit +\item [-{-}batch] +If user input is required, exit without prompting. +\item [-{-}force] +Force checking even if file system is marked clean. +\end{optionlist} +\end{quote} + + +\subsection{Exit Codes} +\label{man/fsck:exit-codes} +If \textbf{fsck.s3ql} found any file system errors (no matter if they were +corrected or not), the exit code will be 128 plus one of the codes +listed below. If no errors were found, the following exit codes are +used as-is: +\begin{quote}\begin{description} +\item[{0}] \leavevmode +Everything went well. + +\item[{1}] \leavevmode +An unexpected error occured. This may indicate a bug in the +program. + +\item[{2}] \leavevmode +Invalid command line argument. + +\item[{3}] \leavevmode +Invalid backend option. + +\item[{10}] \leavevmode +Could not open log file for writing. + +\item[{11}] \leavevmode +No such backend. + +\item[{12}] \leavevmode +Authentication file has insecure permissions. + +\item[{13}] \leavevmode +Unable to parse proxy settings. + +\item[{14}] \leavevmode +Invalid credentials (Authentication failed). + +\item[{15}] \leavevmode +No permission to access backend (Authorization denied). + +\item[{16}] \leavevmode +Invalid storage URL, specified location does not exist in backend. + +\item[{17}] \leavevmode +Wrong file system passphrase. + +\item[{18}] \leavevmode +No S3QL file system found at given storage URL. + +\item[{19}] \leavevmode +Unable to connect to backend, can't resolve hostname. + +\item[{32}] \leavevmode +Unsupported file system revision (too old). + +\item[{33}] \leavevmode +Unsupported file system revision (too new). + +\item[{40}] \leavevmode +Cannot check mounted file system. + +\item[{41}] \leavevmode +User input required, but running in batch mode. + +\item[{42}] \leavevmode +File system check aborted by user. + +\item[{43}] \leavevmode +Local metadata is corrupted. + +\item[{44}] \leavevmode +Uncorrectable errors found. + +\item[{45}] \leavevmode +Unable to access cache directory. + +\item[{128}] \leavevmode +This error code will be \emph{added} to one of the codes above if any +file system errors have been found (no matter if they were +corrected or not). + +\end{description}\end{quote} + + +\subsection{See Also} +\label{man/fsck:see-also} +The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. + +The full S3QL documentation should also be installed somewhere on your +system, common locations are \code{/usr/share/doc/s3ql} or +\code{/usr/local/doc/s3ql}. + + +\section{The \textbf{s3ql\_oauth\_client} command} +\label{man/oauth_client::doc}\label{man/oauth_client:the-command-command}\label{man/oauth_client:oauth-client} + +\subsection{Synopsis} +\label{man/oauth_client:synopsis} +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3ql\PYGZus{}oauth\PYGZus{}client }\PYG{g+ge}{[options]} +\end{Verbatim} + + +\subsection{Description} +\label{man/oauth_client:description} +The \textbf{s3ql\_oauth\_client} command may be used to obtain OAuth2 authentication +tokens for use with Google Storage. It requests ``user code'' from +Google which has to be pasted into the browser to complete the +authentication process interactively. Once authentication in the +browser has been completed, \textbf{s3ql\_oauth\_client} displays the OAuth2 refresh +token. + +When combined with the special username \code{oauth2}, the refresh token +can be used as a backend passphrase when using the Google Storage S3QL +backend. + + +\subsection{Options} +\label{man/oauth_client:options} +The \textbf{s3ql\_oauth\_client} command accepts the following options: +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}version] +just print program version and exit +\end{optionlist} +\end{quote} + + +\subsection{Exit Codes} +\label{man/oauth_client:exit-codes} +\textbf{s3ql\_oauth\_client} may terminate with the following exit codes: +\begin{quote}\begin{description} +\item[{0}] \leavevmode +Everything went well. + +\item[{1}] \leavevmode +An unexpected error occured. This may indicate a bug in the +program. + +\item[{2}] \leavevmode +Invalid command line argument. + +\end{description}\end{quote} + + +\subsection{See Also} +\label{man/oauth_client:see-also} +The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. + +The full S3QL documentation should also be installed somewhere on your +system, common locations are \code{/usr/share/doc/s3ql} or +\code{/usr/local/doc/s3ql}. + + +\section{The \textbf{s3ql\_verify} command} +\label{man/verify::doc}\label{man/verify:the-command-command} + +\subsection{Synopsis} +\label{man/verify:synopsis} +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{s3ql\PYGZus{}verify }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} +\end{Verbatim} + + +\subsection{Description} +\label{man/verify:description} +The \textbf{s3ql\_verify} command verifies all data in the file system. In +contrast to \textbf{fsck.s3ql}, \textbf{s3ql\_verify} does not trust the object +listing returned by the backend, but actually attempts to retrieve +every object. It therefore takes a lot longer. + +The format of \code{\textless{}storage url\textgreater{}} depends on the backend that is +used. The S3QL User's Guide should be consulted for a description of +the available backends. + + +\subsection{Options} +\label{man/verify:options} +The \textbf{s3ql\_verify} command accepts the following options. +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}quiet] +be really quiet +\item [-{-}version] +just print program version and exit +\item [-{-}cachedir \textless{}path\textgreater{}] +Store cached data in this directory (default: +\code{\textasciitilde{}/.s3ql)} +\item [-{-}authfile \textless{}path\textgreater{}] +Read authentication credentials from this file +(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +\item [-{-}backend-options \textless{}options\textgreater{}] +Backend specific options (separate by commas). See +backend documentation for available options. +\item [-{-}missing-file \textless{}name\textgreater{}] +File to store keys of missing objects. +\item [-{-}corrupted-file \textless{}name\textgreater{}] +File to store keys of corrupted objects. +\item [-{-}data] +Read every object completely, instead of checking just +the metadata. +\item [-{-}parallel PARALLEL] +Number of connections to use in parallel. +\item [-{-}start-with \textless{}n\textgreater{}] +Skip over first \textless{}n\textgreater{} objects and with verifying object +\textless{}n\textgreater{}+1. +\end{optionlist} +\end{quote} + + +\subsection{Exit Codes} +\label{man/verify:exit-codes} +\textbf{s3ql\_verify} may terminate with the following exit codes: +\begin{quote}\begin{description} +\item[{0}] \leavevmode +Everything went well. + +\item[{1}] \leavevmode +An unexpected error occured. This may indicate a bug in the +program. + +\item[{2}] \leavevmode +Invalid command line argument. + +\item[{3}] \leavevmode +Invalid backend option. + +\item[{10}] \leavevmode +Could not open log file for writing. + +\item[{11}] \leavevmode +No such backend. + +\item[{12}] \leavevmode +Authentication file has insecure permissions. + +\item[{13}] \leavevmode +Unable to parse proxy settings. + +\item[{14}] \leavevmode +Invalid credentials (Authentication failed). + +\item[{15}] \leavevmode +No permission to access backend (Authorization denied). + +\item[{16}] \leavevmode +Invalid storage URL, specified location does not exist in backend. + +\item[{17}] \leavevmode +Wrong file system passphrase. + +\item[{18}] \leavevmode +No S3QL file system found at given storage URL. + +\item[{19}] \leavevmode +Unable to connect to backend, can't resolve hostname. + +\item[{32}] \leavevmode +Unsupported file system revision (too old). + +\item[{33}] \leavevmode +Unsupported file system revision (too new). + +\item[{45}] \leavevmode +Unable to access cache directory. + +\item[{46}] \leavevmode +The file system data was verified, and some objects were found +to be missing or corrupted. + +\end{description}\end{quote} + + +\subsection{See Also} +\label{man/verify:see-also} +The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. + +The full S3QL documentation should also be installed somewhere on your +system, common locations are \code{/usr/share/doc/s3ql} or +\code{/usr/local/doc/s3ql}. + + +\section{The \textbf{pcp} command} +\label{man/pcp::doc}\label{man/pcp:the-command-command} + +\subsection{Synopsis} +\label{man/pcp:synopsis} +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{pcp }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}source\PYGZgt{}}\PYG{l}{ }\PYG{g+ge}{[\PYGZlt{}source\PYGZgt{} ...]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}destination\PYGZgt{}} +\end{Verbatim} + + +\subsection{Description} +\label{man/pcp:description} +The \textbf{pcp} command is a is a wrapper that starts several +\textbf{sync} processes to copy directory trees in parallel. This is +allows much better copying performance on file system that have +relatively high latency when retrieving individual files like S3QL. + +\textbf{Note}: Using this program only improves performance when copying +\emph{from} an S3QL file system. When copying \emph{to} an S3QL file system, +using \textbf{pcp} is more likely to \emph{decrease} performance. + + +\subsection{Options} +\label{man/pcp:options} +The \textbf{pcp} command accepts the following options: +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}quiet] +be really quiet +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}version] +just print program version and exit +\item [-a] +Pass -aHAX option to rsync. +\item [-{-}processes \textless{}no\textgreater{}] +Number of rsync processes to use (default: 10). +\end{optionlist} +\end{quote} + + +\subsection{Exit Codes} +\label{man/pcp:exit-codes} +\textbf{pcp} may terminate with the following exit codes: +\begin{quote}\begin{description} +\item[{0}] \leavevmode +Everything went well. + +\item[{1}] \leavevmode +An unexpected error occured. This may indicate a bug in the +program. + +\item[{2}] \leavevmode +Invalid command line argument. + +\end{description}\end{quote} + + +\subsection{See Also} +\label{man/pcp:see-also} +\textbf{pcp} is shipped as part of S3QL, \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. + + +\section{The \textbf{expire\_backups} command} +\label{man/expire_backups::doc}\label{man/expire_backups:the-command-command} + +\subsection{Synopsis} +\label{man/expire_backups:synopsis} +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{expire\PYGZus{}backups }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}age\PYGZgt{}}\PYG{l}{ }\PYG{g+ge}{[\PYGZlt{}age\PYGZgt{} ...]} +\end{Verbatim} + + +\subsection{Description} +\label{man/expire_backups:description} +The \textbf{expire\_backups} command intelligently remove old backups that are no +longer needed. + +To define what backups you want to keep for how long, you define a +number of \emph{age ranges}. \textbf{expire\_backups} ensures that you +will have at least one backup in each age range at all times. It will +keep exactly as many backups as are required for that and delete any +backups that become redundant. + +Age ranges are specified by giving a list of range boundaries in terms +of backup cycles. Every time you create a new backup, the existing +backups age by one cycle. + +Example: when \textbf{expire\_backups} is called with the age range +definition \code{1 3 7 14 31}, it will guarantee that you always have the +following backups available: +\begin{enumerate} +\item {} +A backup that is 0 to 1 cycles old (i.e, the most recent backup) + +\item {} +A backup that is 1 to 3 cycles old + +\item {} +A backup that is 3 to 7 cycles old + +\item {} +A backup that is 7 to 14 cycles old + +\item {} +A backup that is 14 to 31 cycles old + +\end{enumerate} + +\begin{notice}{note}{Note:} +If you do backups in fixed intervals, then one cycle will be +equivalent to the backup interval. The advantage of specifying the +age ranges in terms of backup cycles rather than days or weeks is +that it allows you to gracefully handle irregular backup intervals. +Imagine that for some reason you do not turn on your computer for +one month. Now all your backups are at least a month old, and if you +had specified the above backup strategy in terms of absolute ages, +they would all be deleted! Specifying age ranges in terms of backup +cycles avoids these sort of problems. +\end{notice} + +\textbf{expire\_backups} usage is simple. It requires backups to be +stored in directories of the form \code{year-month-day\_hour:minute:seconds} +(\code{YYYY-MM-DD\_HH:mm:ss}) and works on all backups in the current +directory. So for the above backup strategy, the correct invocation +would be: + +\begin{Verbatim}[commandchars=\\\{\}] +\PYG{l}{expire\PYGZus{}backups.py 1 3 7 14 31} +\end{Verbatim} + +When storing your backups on an S3QL file system, you probably want to +specify the \code{-{-}use-s3qlrm} option as well. This tells +\textbf{expire\_backups} to use the {\hyperref[special:s3qlrm]{\emph{s3qlrm}}} command to +delete directories. + +\textbf{expire\_backups} uses a ``state file'' to keep track which +backups are how many cycles old (since this cannot be inferred from +the dates contained in the directory names). The standard name for +this state file is \code{.expire\_backups.dat}. If this file gets +damaged or deleted, \textbf{expire\_backups} no longer knows the ages +of the backups and refuses to work. In this case you can use the +\code{-{-}reconstruct-state} option to try to reconstruct the state +from the backup dates. However, the accuracy of this reconstruction +depends strongly on how rigorous you have been with making backups (it +is only completely correct if the time between subsequent backups has +always been exactly the same), so it's generally a good idea not to +tamper with the state file. + + +\subsection{Options} +\label{man/expire_backups:options} +The \textbf{expire\_backups} command accepts the following options: +\begin{quote} +\begin{optionlist}{3cm} +\item [-{-}quiet] +be really quiet +\item [-{-}debug-modules \textless{}modules\textgreater{}] +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\code{-{-}log} option. +\item [-{-}debug] +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \code{-{-}log} option. +\item [-{-}version] +just print program version and exit +\item [-{-}state \textless{}file\textgreater{}] +File to save state information in (default: +''.expire\_backups.dat'') +\item [-n] +Dry run. Just show which backups would be deleted. +\item [-{-}reconstruct-state] +Try to reconstruct a missing state file from backup +dates. +\item [-{-}use-s3qlrm] +Use \code{s3qlrm} command to delete backups. +\end{optionlist} +\end{quote} + + +\subsection{Exit Codes} +\label{man/expire_backups:exit-codes} +\textbf{expire\_backups} may terminate with the following exit codes: +\begin{quote}\begin{description} +\item[{0}] \leavevmode +Everything went well. + +\item[{1}] \leavevmode +An unexpected error occured. This may indicate a bug in the +program. + +\item[{2}] \leavevmode +Invalid command line argument. + +\end{description}\end{quote} + + +\subsection{See Also} +\label{man/expire_backups:see-also} +\textbf{expire\_backups} is shipped as part of S3QL, \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. + + +\chapter{Further Resources / Getting Help} +\label{resources::doc}\label{resources:resources}\label{resources:further-resources-getting-help} +If you have questions or problems with S3QL that you weren't able to +resolve with this manual, you might want to consider the following other resources: +\begin{itemize} +\item {} +The \href{https://bitbucket.org/nikratio/s3ql/wiki}{S3QL Wiki} + +\item {} +The \href{https://bitbucket.org/nikratio/s3ql/wiki/FAQ}{S3QL FAQ} + +\item {} +The \href{http://groups.google.com/group/s3ql}{S3QL Mailing List}. You +can subscribe by sending a mail to +\href{mailto:s3ql+subscribe@googlegroups.com}{s3ql+subscribe@googlegroups.com}. + +\end{itemize} + +Please report any bugs you may encounter in the \href{https://bitbucket.org/nikratio/s3ql/issues}{Issue Tracker}. + + +\chapter{Implementation Details} +\label{impl_details::doc}\label{impl_details:impl-details}\label{impl_details:implementation-details} +This section provides some background information on how S3QL works +internally. Reading this section is not necessary to use S3QL. + + +\section{Metadata Storage} +\label{impl_details:metadata-storage} +Like most unix filesystems, S3QL has a concept of inodes. + +The contents of directory inodes (aka the names and inodes of the +files and sub directories contained in a directory) are stored +directly in an \href{http://www.sqlite.org/}{SQLite} database. This database +is stored in a special storage object that is downloaded when the file +system is mounted and uploaded periodically in the background and when +the file system is unmounted. This has two implications: +\begin{enumerate} +\item {} +The entire file system tree can be read from the +database. Fetching/storing storage objects from/in the storage +backend is only required to access the contents of files (or, more +precisely, inodes). This makes most file system operations very +fast because no data has to be send over the network. + +\item {} +An S3QL filesystem can only be mounted on one computer at a time, +using a single \textbf{mount.s3ql} process. Otherwise changes made in +one mountpoint will invariably be overwritten when the second mount +point is unmounted. + +\end{enumerate} + +Sockets, FIFOs and character devices do not need any additional +storage, all information about them is contained in the database. + + +\section{Data Storage} +\label{impl_details:data-storage} +The contents of file inodes are split into individual blocks. The +maximum size of a block is specified when the file system is created +and cannot be changed afterwards. Every block is stored as an +individual object in the backend, and the mapping from inodes to +blocks and from blocks to objects is stored in the database. + +While the file system is mounted, blocks are cached locally. + +Blocks can also be compressed and encrypted before they are stored in +the storage backend. + +If some files have blocks with identical contents, the blocks will be +stored in the same backend object (i.e., the data is only stored +once). + + +\section{Data De-Duplication} +\label{impl_details:data-de-duplication} +Instead of uploading every block, S3QL first computes a checksum (a +SHA256 hash) to check if an identical blocks has already been stored +in an backend object. If that is the case, the new block will be +linked to the existing object instead of being uploaded. + +This procedure is invisible for the user and the contents of the block +can still be changed. If several blocks share a backend object and one +of the blocks is changed, the changed block is automatically stored in +a new object (so that the contents of the other block remain +unchanged). + + +\section{Caching} +\label{impl_details:caching} +When an application tries to read or write from a file, S3QL +determines the block that contains the required part of the file and +retrieves it from the backend or creates it if it does not yet exist. +The block is then held in the cache directory. It is committed to S3 +when it has not been accessed for more than a few seconds. Blocks are +removed from the cache only when the maximum cache size is reached. + +When the file system is unmounted, all modified blocks are written to +the backend and the cache is cleaned. + + +\section{Eventual Consistency Handling} +\label{impl_details:eventual-consistency-handling} +S3QL has to take into account that with some storage providers, +changes in objects do not propagate immediately. For example, when an +Amazon S3 object is uploaded and immediately downloaded again, the +downloaded data might not yet reflect the changes done in the upload +(see also +\href{http://developer.amazonwebservices.com/connect/message.jspa?messageID=38538}{http://developer.amazonwebservices.com/connect/message.jspa?messageID=38538}) + +For the data blocks this is not a problem because a data blocks always +get a new object ID when they are updated. + +For the metadata however, S3QL has to make sure that it always +downloads the most recent copy of the database when mounting the file +system. + +To that end, metadata versions are numbered, and the most recent +version number is stored as part of the object id of a very small +``marker'' object. When S3QL has downloaded the metadata it checks the +version number against the marker object and, if the two do not agree, +waits for the most recent metadata to become available. Once the +current metadata is available, the version number is increased and the +marker object updated. + + +\section{Encryption} +\label{impl_details:encryption} +When the file system is created, \textbf{mkfs.s3ql} generates a 256 bit +master key by reading from \code{/dev/random}. The master key is +encrypted with the passphrase that is entered by the user, and then +stored with the rest of the file system data. Since the passphrase is +only used to access the master key (which is used to encrypt the +actual file system data), the passphrase can easily be changed. + +Data is encrypted with a new session key for each object and each +upload. The session key is generated by appending a nonce to the +master key and then calculating the SHA256 hash. The nonce is +generated by concatenating the object id and the current UTC time as a +32 bit float. The precision of the time is given by the Python \href{http://docs.python.org/library/time.html\#time.time}{time()} function and +usually at least 1 millisecond. The SHA256 implementation is included +in the Python standard library. + +Once the session key has been calculated, a SHA256 HMAC is calculated +over the data that is to be uploaded. Afterwards, the data is +compressed (unless \code{-{-}compress none} was passed to +\textbf{mount.s3ql}) and the HMAC inserted at the beginning. Both HMAC +and compressed data are then encrypted using 256 bit AES in CTR +mode using \href{http://www.pycrypto.org/}{PyCrypto}. Finally, the nonce is +inserted in front of the encrypted data and HMAC, and the packet is +send to the backend as a new S3 object. + + + +\renewcommand{\indexname}{Index} +\printindex +\end{document} diff --git a/doc/latex/manual.toc b/doc/latex/manual.toc new file mode 100644 index 0000000..bdf5b44 --- /dev/null +++ b/doc/latex/manual.toc @@ -0,0 +1,157 @@ +\select@language {english} +\contentsline {chapter}{\numberline {1}About S3QL}{1}{chapter.1} +\contentsline {section}{\numberline {1.1}Features}{1}{section.1.1} +\contentsline {section}{\numberline {1.2}Development Status}{2}{section.1.2} +\contentsline {section}{\numberline {1.3}Supported Platforms}{2}{section.1.3} +\contentsline {section}{\numberline {1.4}Contributing}{2}{section.1.4} +\contentsline {chapter}{\numberline {2}Installation}{3}{chapter.2} +\contentsline {section}{\numberline {2.1}Dependencies}{3}{section.2.1} +\contentsline {section}{\numberline {2.2}Installing S3QL}{4}{section.2.2} +\contentsline {section}{\numberline {2.3}Development Version}{4}{section.2.3} +\contentsline {section}{\numberline {2.4}Running tests requiring remote servers}{4}{section.2.4} +\contentsline {chapter}{\numberline {3}Storage Backends}{7}{chapter.3} +\contentsline {section}{\numberline {3.1}Google Storage}{7}{section.3.1} +\contentsline {section}{\numberline {3.2}Amazon S3}{8}{section.3.2} +\contentsline {section}{\numberline {3.3}OpenStack/Swift}{9}{section.3.3} +\contentsline {section}{\numberline {3.4}Rackspace CloudFiles}{9}{section.3.4} +\contentsline {section}{\numberline {3.5}S3 compatible}{10}{section.3.5} +\contentsline {section}{\numberline {3.6}Local}{10}{section.3.6} +\contentsline {chapter}{\numberline {4}Important Rules to Avoid Losing Data}{13}{chapter.4} +\contentsline {section}{\numberline {4.1}Rules in a Nutshell}{13}{section.4.1} +\contentsline {section}{\numberline {4.2}Consistency Window List}{14}{section.4.2} +\contentsline {section}{\numberline {4.3}Data Consistency}{14}{section.4.3} +\contentsline {section}{\numberline {4.4}Data Durability}{15}{section.4.4} +\contentsline {chapter}{\numberline {5}File System Creation}{17}{chapter.5} +\contentsline {chapter}{\numberline {6}Managing File Systems}{19}{chapter.6} +\contentsline {section}{\numberline {6.1}Changing the Passphrase}{19}{section.6.1} +\contentsline {section}{\numberline {6.2}Upgrading the file system}{20}{section.6.2} +\contentsline {section}{\numberline {6.3}Deleting a file system}{20}{section.6.3} +\contentsline {section}{\numberline {6.4}Restoring Metadata Backups}{20}{section.6.4} +\contentsline {chapter}{\numberline {7}Mounting}{21}{chapter.7} +\contentsline {section}{\numberline {7.1}Permission Checking}{22}{section.7.1} +\contentsline {section}{\numberline {7.2}Compression Algorithms}{22}{section.7.2} +\contentsline {section}{\numberline {7.3}Notes about Caching}{22}{section.7.3} +\contentsline {subsection}{\numberline {7.3.1}Maximum Number of Cache Entries}{23}{subsection.7.3.1} +\contentsline {subsection}{\numberline {7.3.2}Cache Flushing and Expiration}{23}{subsection.7.3.2} +\contentsline {section}{\numberline {7.4}Failure Modes}{23}{section.7.4} +\contentsline {section}{\numberline {7.5}Automatic Mounting}{23}{section.7.5} +\contentsline {chapter}{\numberline {8}Advanced S3QL Features}{25}{chapter.8} +\contentsline {section}{\numberline {8.1}Snapshotting and Copy-on-Write}{25}{section.8.1} +\contentsline {subsection}{\numberline {8.1.1}Snapshotting vs Hardlinking}{25}{subsection.8.1.1} +\contentsline {section}{\numberline {8.2}Getting Statistics}{26}{section.8.2} +\contentsline {section}{\numberline {8.3}Immutable Trees}{26}{section.8.3} +\contentsline {section}{\numberline {8.4}Fast Recursive Removal}{27}{section.8.4} +\contentsline {section}{\numberline {8.5}Runtime Configuration}{27}{section.8.5} +\contentsline {chapter}{\numberline {9}Unmounting}{29}{chapter.9} +\contentsline {chapter}{\numberline {10}Checking for Errors}{31}{chapter.10} +\contentsline {section}{\numberline {10.1}Checking and repairing internal file system errors}{31}{section.10.1} +\contentsline {section}{\numberline {10.2}Detecting and handling backend data corruption}{32}{section.10.2} +\contentsline {chapter}{\numberline {11}Storing Authentication Information}{33}{chapter.11} +\contentsline {chapter}{\numberline {12}Contributed Programs}{35}{chapter.12} +\contentsline {section}{\numberline {12.1}benchmark.py}{35}{section.12.1} +\contentsline {section}{\numberline {12.2}clone\_fs.py}{35}{section.12.2} +\contentsline {section}{\numberline {12.3}pcp.py}{35}{section.12.3} +\contentsline {section}{\numberline {12.4}s3ql\_backup.sh}{35}{section.12.4} +\contentsline {section}{\numberline {12.5}expire\_backups.py}{36}{section.12.5} +\contentsline {section}{\numberline {12.6}remove\_objects.py}{37}{section.12.6} +\contentsline {chapter}{\numberline {13}Tips \& Tricks}{39}{chapter.13} +\contentsline {section}{\numberline {13.1}SSH Backend}{39}{section.13.1} +\contentsline {section}{\numberline {13.2}Permanently mounted backup file system}{39}{section.13.2} +\contentsline {section}{\numberline {13.3}Improving copy performance}{39}{section.13.3} +\contentsline {chapter}{\numberline {14}Known Issues}{41}{chapter.14} +\contentsline {chapter}{\numberline {15}Manpages}{43}{chapter.15} +\contentsline {section}{\numberline {15.1}The \textbf {mkfs.s3ql} command}{43}{section.15.1} +\contentsline {subsection}{\numberline {15.1.1}Synopsis}{43}{subsection.15.1.1} +\contentsline {subsection}{\numberline {15.1.2}Description}{43}{subsection.15.1.2} +\contentsline {subsection}{\numberline {15.1.3}Options}{43}{subsection.15.1.3} +\contentsline {subsection}{\numberline {15.1.4}Exit Codes}{44}{subsection.15.1.4} +\contentsline {subsection}{\numberline {15.1.5}See Also}{44}{subsection.15.1.5} +\contentsline {section}{\numberline {15.2}The \textbf {s3qladm} command}{44}{section.15.2} +\contentsline {subsection}{\numberline {15.2.1}Synopsis}{44}{subsection.15.2.1} +\contentsline {subsection}{\numberline {15.2.2}Description}{45}{subsection.15.2.2} +\contentsline {subsection}{\numberline {15.2.3}Options}{45}{subsection.15.2.3} +\contentsline {subsection}{\numberline {15.2.4}Actions}{45}{subsection.15.2.4} +\contentsline {subsection}{\numberline {15.2.5}Exit Codes}{45}{subsection.15.2.5} +\contentsline {subsection}{\numberline {15.2.6}See Also}{46}{subsection.15.2.6} +\contentsline {section}{\numberline {15.3}The \textbf {mount.s3ql} command}{46}{section.15.3} +\contentsline {subsection}{\numberline {15.3.1}Synopsis}{46}{subsection.15.3.1} +\contentsline {subsection}{\numberline {15.3.2}Description}{46}{subsection.15.3.2} +\contentsline {subsection}{\numberline {15.3.3}Options}{46}{subsection.15.3.3} +\contentsline {subsection}{\numberline {15.3.4}Exit Codes}{47}{subsection.15.3.4} +\contentsline {subsection}{\numberline {15.3.5}See Also}{48}{subsection.15.3.5} +\contentsline {section}{\numberline {15.4}The \textbf {s3qlstat} command}{48}{section.15.4} +\contentsline {subsection}{\numberline {15.4.1}Synopsis}{48}{subsection.15.4.1} +\contentsline {subsection}{\numberline {15.4.2}Description}{48}{subsection.15.4.2} +\contentsline {subsection}{\numberline {15.4.3}Options}{49}{subsection.15.4.3} +\contentsline {subsection}{\numberline {15.4.4}Exit Codes}{49}{subsection.15.4.4} +\contentsline {subsection}{\numberline {15.4.5}See Also}{49}{subsection.15.4.5} +\contentsline {section}{\numberline {15.5}The \textbf {s3qlctrl} command}{49}{section.15.5} +\contentsline {subsection}{\numberline {15.5.1}Synopsis}{49}{subsection.15.5.1} +\contentsline {subsection}{\numberline {15.5.2}Description}{49}{subsection.15.5.2} +\contentsline {subsection}{\numberline {15.5.3}Options}{50}{subsection.15.5.3} +\contentsline {subsection}{\numberline {15.5.4}Exit Codes}{50}{subsection.15.5.4} +\contentsline {subsection}{\numberline {15.5.5}See Also}{50}{subsection.15.5.5} +\contentsline {section}{\numberline {15.6}The \textbf {s3qlcp} command}{50}{section.15.6} +\contentsline {subsection}{\numberline {15.6.1}Synopsis}{50}{subsection.15.6.1} +\contentsline {subsection}{\numberline {15.6.2}Description}{51}{subsection.15.6.2} +\contentsline {subsubsection}{Snapshotting vs Hardlinking}{51}{subsubsection*.21} +\contentsline {subsection}{\numberline {15.6.3}Options}{51}{subsection.15.6.3} +\contentsline {subsection}{\numberline {15.6.4}Exit Codes}{51}{subsection.15.6.4} +\contentsline {subsection}{\numberline {15.6.5}See Also}{52}{subsection.15.6.5} +\contentsline {section}{\numberline {15.7}The \textbf {s3qlrm} command}{52}{section.15.7} +\contentsline {subsection}{\numberline {15.7.1}Synopsis}{52}{subsection.15.7.1} +\contentsline {subsection}{\numberline {15.7.2}Description}{52}{subsection.15.7.2} +\contentsline {subsection}{\numberline {15.7.3}Options}{52}{subsection.15.7.3} +\contentsline {subsection}{\numberline {15.7.4}Exit Codes}{52}{subsection.15.7.4} +\contentsline {subsection}{\numberline {15.7.5}See Also}{53}{subsection.15.7.5} +\contentsline {section}{\numberline {15.8}The \textbf {s3qllock} command}{53}{section.15.8} +\contentsline {subsection}{\numberline {15.8.1}Synopsis}{53}{subsection.15.8.1} +\contentsline {subsection}{\numberline {15.8.2}Description}{53}{subsection.15.8.2} +\contentsline {subsection}{\numberline {15.8.3}Rationale}{53}{subsection.15.8.3} +\contentsline {subsection}{\numberline {15.8.4}Options}{53}{subsection.15.8.4} +\contentsline {subsection}{\numberline {15.8.5}Exit Codes}{54}{subsection.15.8.5} +\contentsline {subsection}{\numberline {15.8.6}See Also}{54}{subsection.15.8.6} +\contentsline {section}{\numberline {15.9}The \textbf {umount.s3ql} command}{54}{section.15.9} +\contentsline {subsection}{\numberline {15.9.1}Synopsis}{54}{subsection.15.9.1} +\contentsline {subsection}{\numberline {15.9.2}Description}{54}{subsection.15.9.2} +\contentsline {subsection}{\numberline {15.9.3}Options}{54}{subsection.15.9.3} +\contentsline {subsection}{\numberline {15.9.4}Exit Codes}{55}{subsection.15.9.4} +\contentsline {subsection}{\numberline {15.9.5}See Also}{55}{subsection.15.9.5} +\contentsline {section}{\numberline {15.10}The \textbf {fsck.s3ql} command}{55}{section.15.10} +\contentsline {subsection}{\numberline {15.10.1}Synopsis}{55}{subsection.15.10.1} +\contentsline {subsection}{\numberline {15.10.2}Description}{55}{subsection.15.10.2} +\contentsline {subsection}{\numberline {15.10.3}Options}{55}{subsection.15.10.3} +\contentsline {subsection}{\numberline {15.10.4}Exit Codes}{56}{subsection.15.10.4} +\contentsline {subsection}{\numberline {15.10.5}See Also}{57}{subsection.15.10.5} +\contentsline {section}{\numberline {15.11}The \textbf {s3ql\_oauth\_client} command}{57}{section.15.11} +\contentsline {subsection}{\numberline {15.11.1}Synopsis}{57}{subsection.15.11.1} +\contentsline {subsection}{\numberline {15.11.2}Description}{57}{subsection.15.11.2} +\contentsline {subsection}{\numberline {15.11.3}Options}{57}{subsection.15.11.3} +\contentsline {subsection}{\numberline {15.11.4}Exit Codes}{57}{subsection.15.11.4} +\contentsline {subsection}{\numberline {15.11.5}See Also}{58}{subsection.15.11.5} +\contentsline {section}{\numberline {15.12}The \textbf {s3ql\_verify} command}{58}{section.15.12} +\contentsline {subsection}{\numberline {15.12.1}Synopsis}{58}{subsection.15.12.1} +\contentsline {subsection}{\numberline {15.12.2}Description}{58}{subsection.15.12.2} +\contentsline {subsection}{\numberline {15.12.3}Options}{58}{subsection.15.12.3} +\contentsline {subsection}{\numberline {15.12.4}Exit Codes}{59}{subsection.15.12.4} +\contentsline {subsection}{\numberline {15.12.5}See Also}{59}{subsection.15.12.5} +\contentsline {section}{\numberline {15.13}The \textbf {pcp} command}{59}{section.15.13} +\contentsline {subsection}{\numberline {15.13.1}Synopsis}{59}{subsection.15.13.1} +\contentsline {subsection}{\numberline {15.13.2}Description}{59}{subsection.15.13.2} +\contentsline {subsection}{\numberline {15.13.3}Options}{60}{subsection.15.13.3} +\contentsline {subsection}{\numberline {15.13.4}Exit Codes}{60}{subsection.15.13.4} +\contentsline {subsection}{\numberline {15.13.5}See Also}{60}{subsection.15.13.5} +\contentsline {section}{\numberline {15.14}The \textbf {expire\_backups} command}{60}{section.15.14} +\contentsline {subsection}{\numberline {15.14.1}Synopsis}{60}{subsection.15.14.1} +\contentsline {subsection}{\numberline {15.14.2}Description}{60}{subsection.15.14.2} +\contentsline {subsection}{\numberline {15.14.3}Options}{61}{subsection.15.14.3} +\contentsline {subsection}{\numberline {15.14.4}Exit Codes}{62}{subsection.15.14.4} +\contentsline {subsection}{\numberline {15.14.5}See Also}{62}{subsection.15.14.5} +\contentsline {chapter}{\numberline {16}Further Resources / Getting Help}{63}{chapter.16} +\contentsline {chapter}{\numberline {17}Implementation Details}{65}{chapter.17} +\contentsline {section}{\numberline {17.1}Metadata Storage}{65}{section.17.1} +\contentsline {section}{\numberline {17.2}Data Storage}{65}{section.17.2} +\contentsline {section}{\numberline {17.3}Data De-Duplication}{65}{section.17.3} +\contentsline {section}{\numberline {17.4}Caching}{66}{section.17.4} +\contentsline {section}{\numberline {17.5}Eventual Consistency Handling}{66}{section.17.5} +\contentsline {section}{\numberline {17.6}Encryption}{66}{section.17.6} diff --git a/doc/latex/python.ist b/doc/latex/python.ist new file mode 100644 index 0000000..9ffa0f9 --- /dev/null +++ b/doc/latex/python.ist @@ -0,0 +1,11 @@ +line_max 100 +headings_flag 1 +heading_prefix " \\bigletter " + +preamble "\\begin{theindex} +\\def\\bigletter#1{{\\Large\\sffamily#1}\\nopagebreak\\vspace{1mm}} + +" + +symhead_positive "{Symbols}" +numhead_positive "{Numbers}" diff --git a/doc/latex/sphinx.sty b/doc/latex/sphinx.sty new file mode 100644 index 0000000..ee0a923 --- /dev/null +++ b/doc/latex/sphinx.sty @@ -0,0 +1,524 @@ +% +% sphinx.sty +% +% Adapted from the old python.sty, mostly written by Fred Drake, +% by Georg Brandl. +% + +\NeedsTeXFormat{LaTeX2e}[1995/12/01] +\ProvidesPackage{sphinx}[2010/01/15 LaTeX package (Sphinx markup)] + +\@ifclassloaded{memoir}{}{\RequirePackage{fancyhdr}} + +\RequirePackage{textcomp} +\RequirePackage{fancybox} +\RequirePackage{titlesec} +\RequirePackage{tabulary} +\RequirePackage{amsmath} % for \text +\RequirePackage{makeidx} +\RequirePackage{framed} +\RequirePackage{ifthen} +\RequirePackage{color} +% For highlighted code. +\RequirePackage{fancyvrb} +% For table captions. +\RequirePackage{threeparttable} +% Handle footnotes in tables. +\RequirePackage{footnote} +\makesavenoteenv{tabulary} +% For floating figures in the text. +\RequirePackage{wrapfig} +% Separate paragraphs by space by default. +\RequirePackage{parskip} +% For parsed-literal blocks. +\RequirePackage{alltt} +% Display "real" single quotes in literal blocks. +\RequirePackage{upquote} + +% Redefine these colors to your liking in the preamble. +\definecolor{TitleColor}{rgb}{0.126,0.263,0.361} +\definecolor{InnerLinkColor}{rgb}{0.208,0.374,0.486} +\definecolor{OuterLinkColor}{rgb}{0.216,0.439,0.388} +% Redefine these colors to something not white if you want to have colored +% background and border for code examples. +\definecolor{VerbatimColor}{rgb}{1,1,1} +\definecolor{VerbatimBorderColor}{rgb}{1,1,1} + +% Uncomment these two lines to ignore the paper size and make the page +% size more like a typical published manual. +%\renewcommand{\paperheight}{9in} +%\renewcommand{\paperwidth}{8.5in} % typical squarish manual +%\renewcommand{\paperwidth}{7in} % O'Reilly ``Programmming Python'' + +% use pdfoutput for pTeX and dvipdfmx +\ifx\kanjiskip\undefined\else + \ifx\Gin@driver{dvipdfmx.def}\undefined\else + \newcount\pdfoutput\pdfoutput=0 + \fi +\fi + +% For graphicx, check if we are compiling under latex or pdflatex. +\ifx\pdftexversion\undefined + \usepackage{graphicx} +\else + \usepackage[pdftex]{graphicx} +\fi + +% for PDF output, use colors and maximal compression +\newif\ifsphinxpdfoutput\sphinxpdfoutputfalse +\ifx\pdfoutput\undefined\else\ifcase\pdfoutput + \let\py@NormalColor\relax + \let\py@TitleColor\relax +\else + \sphinxpdfoutputtrue + \input{pdfcolor} + \def\py@NormalColor{\color[rgb]{0.0,0.0,0.0}} + \def\py@TitleColor{\color{TitleColor}} + \pdfcompresslevel=9 +\fi\fi + +% XeLaTeX can do colors, too +\ifx\XeTeXrevision\undefined\else + \def\py@NormalColor{\color[rgb]{0.0,0.0,0.0}} + \def\py@TitleColor{\color{TitleColor}} +\fi + +% Increase printable page size (copied from fullpage.sty) +\topmargin 0pt +\advance \topmargin by -\headheight +\advance \topmargin by -\headsep + +% attempt to work a little better for A4 users +\textheight \paperheight +\advance\textheight by -2in + +\oddsidemargin 0pt +\evensidemargin 0pt +%\evensidemargin -.25in % for ``manual size'' documents +\marginparwidth 0.5in + +\textwidth \paperwidth +\advance\textwidth by -2in + + +% Style parameters and macros used by most documents here +\raggedbottom +\sloppy +\hbadness = 5000 % don't print trivial gripes + +\pagestyle{empty} % start this way + +% Use this to set the font family for headers and other decor: +\newcommand{\py@HeaderFamily}{\sffamily\bfseries} + +% Redefine the 'normal' header/footer style when using "fancyhdr" package: +\@ifundefined{fancyhf}{}{ + % Use \pagestyle{normal} as the primary pagestyle for text. + \fancypagestyle{normal}{ + \fancyhf{} + \fancyfoot[LE,RO]{{\py@HeaderFamily\thepage}} + \fancyfoot[LO]{{\py@HeaderFamily\nouppercase{\rightmark}}} + \fancyfoot[RE]{{\py@HeaderFamily\nouppercase{\leftmark}}} + \fancyhead[LE,RO]{{\py@HeaderFamily \@title, \py@release}} + \renewcommand{\headrulewidth}{0.4pt} + \renewcommand{\footrulewidth}{0.4pt} + % define chaptermark with \@chappos when \@chappos is available for Japanese + \ifx\@chappos\undefined\else + \def\chaptermark##1{\markboth{\@chapapp\space\thechapter\space\@chappos\space ##1}{}} + \fi + } + % Update the plain style so we get the page number & footer line, + % but not a chapter or section title. This is to keep the first + % page of a chapter and the blank page between chapters `clean.' + \fancypagestyle{plain}{ + \fancyhf{} + \fancyfoot[LE,RO]{{\py@HeaderFamily\thepage}} + \renewcommand{\headrulewidth}{0pt} + \renewcommand{\footrulewidth}{0.4pt} + } +} + +% Some custom font markup commands. +% +\newcommand{\strong}[1]{{\textbf{#1}}} +\newcommand{\code}[1]{\texttt{#1}} +\newcommand{\bfcode}[1]{\code{\bfseries#1}} +\newcommand{\email}[1]{\textsf{#1}} + +% Redefine the Verbatim environment to allow border and background colors. +% The original environment is still used for verbatims within tables. +\let\OriginalVerbatim=\Verbatim +\let\endOriginalVerbatim=\endVerbatim + +% Play with vspace to be able to keep the indentation. +\newlength\distancetoright +\def\mycolorbox#1{% + \setlength\distancetoright{\linewidth}% + \advance\distancetoright -\@totalleftmargin % + \fcolorbox{VerbatimBorderColor}{VerbatimColor}{% + \begin{minipage}{\distancetoright}% + #1 + \end{minipage}% + }% +} +\def\FrameCommand{\mycolorbox} + +\renewcommand{\Verbatim}[1][1]{% + % list starts new par, but we don't want it to be set apart vertically + \bgroup\parskip=0pt% + \smallskip% + % The list environement is needed to control perfectly the vertical + % space. + \list{}{% + \setlength\parskip{0pt}% + \setlength\itemsep{0ex}% + \setlength\topsep{0ex}% + \setlength\partopsep{0pt}% + \setlength\leftmargin{0pt}% + }% + \item\MakeFramed {\FrameRestore}% + \small% + \OriginalVerbatim[#1]% +} +\renewcommand{\endVerbatim}{% + \endOriginalVerbatim% + \endMakeFramed% + \endlist% + % close group to restore \parskip + \egroup% +} + + +% \moduleauthor{name}{email} +\newcommand{\moduleauthor}[2]{} + +% \sectionauthor{name}{email} +\newcommand{\sectionauthor}[2]{} + +% Augment the sectioning commands used to get our own font family in place, +% and reset some internal data items: +\titleformat{\section}{\Large\py@HeaderFamily}% + {\py@TitleColor\thesection}{0.5em}{\py@TitleColor}{\py@NormalColor} +\titleformat{\subsection}{\large\py@HeaderFamily}% + {\py@TitleColor\thesubsection}{0.5em}{\py@TitleColor}{\py@NormalColor} +\titleformat{\subsubsection}{\py@HeaderFamily}% + {\py@TitleColor\thesubsubsection}{0.5em}{\py@TitleColor}{\py@NormalColor} +\titleformat{\paragraph}{\small\py@HeaderFamily}% + {\py@TitleColor}{0em}{\py@TitleColor}{\py@NormalColor} + +% {fulllineitems} is the main environment for object descriptions. +% +\newcommand{\py@itemnewline}[1]{% + \@tempdima\linewidth% + \advance\@tempdima \leftmargin\makebox[\@tempdima][l]{#1}% +} + +\newenvironment{fulllineitems}{ + \begin{list}{}{\labelwidth \leftmargin \labelsep 0pt + \rightmargin 0pt \topsep -\parskip \partopsep \parskip + \itemsep -\parsep + \let\makelabel=\py@itemnewline} +}{\end{list}} + +% \optional is used for ``[, arg]``, i.e. desc_optional nodes. +\newcommand{\optional}[1]{% + {\textnormal{\Large[}}{#1}\hspace{0.5mm}{\textnormal{\Large]}}} + +\newlength{\py@argswidth} +\newcommand{\py@sigparams}[2]{% + \parbox[t]{\py@argswidth}{#1\code{)}#2}} +\newcommand{\pysigline}[1]{\item[#1]\nopagebreak} +\newcommand{\pysiglinewithargsret}[3]{% + \settowidth{\py@argswidth}{#1\code{(}}% + \addtolength{\py@argswidth}{-2\py@argswidth}% + \addtolength{\py@argswidth}{\linewidth}% + \item[#1\code{(}\py@sigparams{#2}{#3}]} + +% Production lists +% +\newenvironment{productionlist}{ +% \def\optional##1{{\Large[}##1{\Large]}} + \def\production##1##2{\\\code{##1}&::=&\code{##2}} + \def\productioncont##1{\\& &\code{##1}} + \parindent=2em + \indent + \setlength{\LTpre}{0pt} + \setlength{\LTpost}{0pt} + \begin{longtable}[l]{lcl} +}{% + \end{longtable} +} + +% Notices / Admonitions +% +\newlength{\py@noticelength} + +\newcommand{\py@heavybox}{ + \setlength{\fboxrule}{1pt} + \setlength{\fboxsep}{6pt} + \setlength{\py@noticelength}{\linewidth} + \addtolength{\py@noticelength}{-2\fboxsep} + \addtolength{\py@noticelength}{-2\fboxrule} + %\setlength{\shadowsize}{3pt} + \noindent\Sbox + \minipage{\py@noticelength} +} +\newcommand{\py@endheavybox}{ + \endminipage + \endSbox + \fbox{\TheSbox} +} + +\newcommand{\py@lightbox}{{% + \setlength\parskip{0pt}\par + \noindent\rule[0ex]{\linewidth}{0.5pt}% + \par\noindent\vspace{-0.5ex}% + }} +\newcommand{\py@endlightbox}{{% + \setlength{\parskip}{0pt}% + \par\noindent\rule[0.5ex]{\linewidth}{0.5pt}% + \par\vspace{-0.5ex}% + }} + +% Some are quite plain: +\newcommand{\py@noticestart@note}{\py@lightbox} +\newcommand{\py@noticeend@note}{\py@endlightbox} +\newcommand{\py@noticestart@hint}{\py@lightbox} +\newcommand{\py@noticeend@hint}{\py@endlightbox} +\newcommand{\py@noticestart@important}{\py@lightbox} +\newcommand{\py@noticeend@important}{\py@endlightbox} +\newcommand{\py@noticestart@tip}{\py@lightbox} +\newcommand{\py@noticeend@tip}{\py@endlightbox} + +% Others gets more visible distinction: +\newcommand{\py@noticestart@warning}{\py@heavybox} +\newcommand{\py@noticeend@warning}{\py@endheavybox} +\newcommand{\py@noticestart@caution}{\py@heavybox} +\newcommand{\py@noticeend@caution}{\py@endheavybox} +\newcommand{\py@noticestart@attention}{\py@heavybox} +\newcommand{\py@noticeend@attention}{\py@endheavybox} +\newcommand{\py@noticestart@danger}{\py@heavybox} +\newcommand{\py@noticeend@danger}{\py@endheavybox} +\newcommand{\py@noticestart@error}{\py@heavybox} +\newcommand{\py@noticeend@error}{\py@endheavybox} + +\newenvironment{notice}[2]{ + \def\py@noticetype{#1} + \csname py@noticestart@#1\endcsname + \strong{#2} +}{\csname py@noticeend@\py@noticetype\endcsname} + +% Allow the release number to be specified independently of the +% \date{}. This allows the date to reflect the document's date and +% release to specify the release that is documented. +% +\newcommand{\py@release}{} +\newcommand{\version}{} +\newcommand{\shortversion}{} +\newcommand{\releaseinfo}{} +\newcommand{\releasename}{Release} +\newcommand{\release}[1]{% + \renewcommand{\py@release}{\releasename\space\version}% + \renewcommand{\version}{#1}} +\newcommand{\setshortversion}[1]{% + \renewcommand{\shortversion}{#1}} +\newcommand{\setreleaseinfo}[1]{% + \renewcommand{\releaseinfo}{#1}} + +% Allow specification of the author's address separately from the +% author's name. This can be used to format them differently, which +% is a good thing. +% +\newcommand{\py@authoraddress}{} +\newcommand{\authoraddress}[1]{\renewcommand{\py@authoraddress}{#1}} + +% This sets up the fancy chapter headings that make the documents look +% at least a little better than the usual LaTeX output. +% +\@ifundefined{ChTitleVar}{}{ + \ChNameVar{\raggedleft\normalsize\py@HeaderFamily} + \ChNumVar{\raggedleft \bfseries\Large\py@HeaderFamily} + \ChTitleVar{\raggedleft \textrm{\Huge\py@HeaderFamily}} + % This creates chapter heads without the leading \vspace*{}: + \def\@makechapterhead#1{% + {\parindent \z@ \raggedright \normalfont + \ifnum \c@secnumdepth >\m@ne + \DOCH + \fi + \interlinepenalty\@M + \DOTI{#1} + } + } +} + +% Redefine description environment so that it is usable inside fulllineitems. +% +\renewcommand{\description}{% + \list{}{\labelwidth\z@% + \itemindent-\leftmargin% + \labelsep5pt% + \let\makelabel=\descriptionlabel}} + +% Definition lists; requested by AMK for HOWTO documents. Probably useful +% elsewhere as well, so keep in in the general style support. +% +\newenvironment{definitions}{% + \begin{description}% + \def\term##1{\item[##1]\mbox{}\\*[0mm]} +}{% + \end{description}% +} + +% Tell TeX about pathological hyphenation cases: +\hyphenation{Base-HTTP-Re-quest-Hand-ler} + + +% The following is stuff copied from docutils' latex writer. +% +\newcommand{\optionlistlabel}[1]{\bf #1 \hfill} +\newenvironment{optionlist}[1] +{\begin{list}{} + {\setlength{\labelwidth}{#1} + \setlength{\rightmargin}{1cm} + \setlength{\leftmargin}{\rightmargin} + \addtolength{\leftmargin}{\labelwidth} + \addtolength{\leftmargin}{\labelsep} + \renewcommand{\makelabel}{\optionlistlabel}} +}{\end{list}} + +\newlength{\lineblockindentation} +\setlength{\lineblockindentation}{2.5em} +\newenvironment{lineblock}[1] +{\begin{list}{} + {\setlength{\partopsep}{\parskip} + \addtolength{\partopsep}{\baselineskip} + \topsep0pt\itemsep0.15\baselineskip\parsep0pt + \leftmargin#1} + \raggedright} +{\end{list}} + +% Redefine includgraphics for avoiding images larger than the screen size +% If the size is not specified. +\let\py@Oldincludegraphics\includegraphics + +\newbox\image@box% +\newdimen\image@width% +\renewcommand\includegraphics[2][\@empty]{% + \ifx#1\@empty% + \setbox\image@box=\hbox{\py@Oldincludegraphics{#2}}% + \image@width\wd\image@box% + \ifdim \image@width>\linewidth% + \setbox\image@box=\hbox{\py@Oldincludegraphics[width=\linewidth]{#2}}% + \box\image@box% + \else% + \py@Oldincludegraphics{#2}% + \fi% + \else% + \py@Oldincludegraphics[#1]{#2}% + \fi% +} + +% to make pdf with correct encoded bookmarks in Japanese +% this should precede the hyperref package +\ifx\kanjiskip\undefined\else + \usepackage{atbegshi} + \ifx\ucs\undefined + \ifnum 42146=\euc"A4A2 + \AtBeginShipoutFirst{\special{pdf:tounicode EUC-UCS2}} + \else + \AtBeginShipoutFirst{\special{pdf:tounicode 90ms-RKSJ-UCS2}} + \fi + \else + \AtBeginShipoutFirst{\special{pdf:tounicode UTF8-UCS2}} + \fi +\fi + +% Include hyperref last. +\RequirePackage[colorlinks,breaklinks, + linkcolor=InnerLinkColor,filecolor=OuterLinkColor, + menucolor=OuterLinkColor,urlcolor=OuterLinkColor, + citecolor=InnerLinkColor]{hyperref} +% Fix anchor placement for figures with captions. +% (Note: we don't use a package option here; instead, we give an explicit +% \capstart for figures that actually have a caption.) +\RequirePackage{hypcap} + +% From docutils.writers.latex2e +\providecommand{\DUspan}[2]{% + {% group ("span") to limit the scope of styling commands + \@for\node@class@name:=#1\do{% + \ifcsname docutilsrole\node@class@name\endcsname% + \csname docutilsrole\node@class@name\endcsname% + \fi% + }% + {#2}% node content + }% close "span" +} + +\providecommand*{\DUprovidelength}[2]{ + \ifthenelse{\isundefined{#1}}{\newlength{#1}\setlength{#1}{#2}}{} +} + +\DUprovidelength{\DUlineblockindent}{2.5em} +\ifthenelse{\isundefined{\DUlineblock}}{ + \newenvironment{DUlineblock}[1]{% + \list{}{\setlength{\partopsep}{\parskip} + \addtolength{\partopsep}{\baselineskip} + \setlength{\topsep}{0pt} + \setlength{\itemsep}{0.15\baselineskip} + \setlength{\parsep}{0pt} + \setlength{\leftmargin}{#1}} + \raggedright + } + {\endlist} +}{} + + +% From footmisc.sty: allows footnotes in titles +\let\FN@sf@@footnote\footnote +\def\footnote{\ifx\protect\@typeset@protect + \expandafter\FN@sf@@footnote + \else + \expandafter\FN@sf@gobble@opt + \fi +} +\edef\FN@sf@gobble@opt{\noexpand\protect + \expandafter\noexpand\csname FN@sf@gobble@opt \endcsname} +\expandafter\def\csname FN@sf@gobble@opt \endcsname{% + \@ifnextchar[%] + \FN@sf@gobble@twobracket + \@gobble +} +\def\FN@sf@gobble@twobracket[#1]#2{} + +% adjust the margins for footer, +% this works with the jsclasses only (Japanese standard document classes) +\ifx\@jsc@uplatextrue\undefined\else + \hypersetup{setpagesize=false} + \setlength\footskip{2\baselineskip} + \addtolength{\textheight}{-2\baselineskip} +\fi + +% fix the double index and bibliography on the table of contents +% in jsclasses (Japanese standard document classes) +\ifx\@jsc@uplatextrue\undefined\else + \renewcommand{\theindex}{ + \cleardoublepage + \phantomsection + \py@OldTheindex + } + \renewcommand{\thebibliography}[1]{ + \cleardoublepage + \phantomsection + \py@OldThebibliography{1} + } +\fi + +% disable \@chappos in Appendix in pTeX +\ifx\kanjiskip\undefined\else + \let\py@OldAppendix=\appendix + \renewcommand{\appendix}{ + \py@OldAppendix + \gdef\@chappos{} + } +\fi diff --git a/doc/latex/sphinxhowto.cls b/doc/latex/sphinxhowto.cls new file mode 100644 index 0000000..26e63a7 --- /dev/null +++ b/doc/latex/sphinxhowto.cls @@ -0,0 +1,104 @@ +% +% sphinxhowto.cls for Sphinx (http://sphinx-doc.org/) +% + +\NeedsTeXFormat{LaTeX2e}[1995/12/01] +\ProvidesClass{sphinxhowto}[2009/06/02 Document class (Sphinx HOWTO)] + +% 'oneside' option overriding the 'twoside' default +\newif\if@oneside +\DeclareOption{oneside}{\@onesidetrue} +% Pass remaining document options to the parent class. +\DeclareOption*{\PassOptionsToClass{\CurrentOption}{\sphinxdocclass}} +\ProcessOptions\relax + +% Default to two-side document +\if@oneside +% nothing to do (oneside is the default) +\else +\PassOptionsToClass{twoside}{\sphinxdocclass} +\fi + +\LoadClass{\sphinxdocclass} + +% Set some sane defaults for section numbering depth and TOC depth. You can +% reset these counters in your preamble. +% +\setcounter{secnumdepth}{2} + +% Change the title page to look a bit better, and fit in with the fncychap +% ``Bjarne'' style a bit better. +% +\renewcommand{\maketitle}{ + \rule{\textwidth}{1pt} + \ifsphinxpdfoutput + \begingroup + % These \defs are required to deal with multi-line authors; it + % changes \\ to ', ' (comma-space), making it pass muster for + % generating document info in the PDF file. + \def\\{, } + \def\and{and } + \pdfinfo{ + /Author (\@author) + /Title (\@title) + } + \endgroup + \fi + \begin{flushright} + \sphinxlogo% + {\rm\Huge\py@HeaderFamily \@title} \par + {\em\large\py@HeaderFamily \py@release\releaseinfo} \par + \vspace{25pt} + {\Large\py@HeaderFamily + \begin{tabular}[t]{c} + \@author + \end{tabular}} \par + \vspace{25pt} + \@date \par + \py@authoraddress \par + \end{flushright} + \@thanks + \setcounter{footnote}{0} + \let\thanks\relax\let\maketitle\relax + %\gdef\@thanks{}\gdef\@author{}\gdef\@title{} +} + +\let\py@OldTableofcontents=\tableofcontents +\renewcommand{\tableofcontents}{ + \begingroup + \parskip = 0mm + \py@OldTableofcontents + \endgroup + \rule{\textwidth}{1pt} + \vspace{12pt} +} + +\@ifundefined{fancyhf}{ + \pagestyle{plain}}{ + \pagestyle{normal}} % start this way; change for +\pagenumbering{arabic} % ToC & chapters + +\thispagestyle{empty} + +% Fix the bibliography environment to add an entry to the Table of +% Contents. +% For an article document class this environment is a section, +% so no page break before it. +\let\py@OldThebibliography=\thebibliography +\renewcommand{\thebibliography}[1]{ + \phantomsection + \py@OldThebibliography{1} + \addcontentsline{toc}{section}{\bibname} +} + +% Same for the indices. +% The memoir class already does this, so we don't duplicate it in that case. +% +\@ifclassloaded{memoir}{}{ + \let\py@OldTheindex=\theindex + \renewcommand{\theindex}{ + \phantomsection + \py@OldTheindex + \addcontentsline{toc}{section}{\indexname} + } +} diff --git a/doc/latex/sphinxmanual.cls b/doc/latex/sphinxmanual.cls new file mode 100644 index 0000000..a6b9b39 --- /dev/null +++ b/doc/latex/sphinxmanual.cls @@ -0,0 +1,148 @@ +% +% sphinxmanual.cls for Sphinx (http://sphinx-doc.org/) +% + +\NeedsTeXFormat{LaTeX2e}[1995/12/01] +\ProvidesClass{sphinxmanual}[2009/06/02 Document class (Sphinx manual)] + +% chapters starting at odd pages (overridden by 'openany' document option) +\PassOptionsToClass{openright}{\sphinxdocclass} + +% 'oneside' option overriding the 'twoside' default +\newif\if@oneside +\DeclareOption{oneside}{\@onesidetrue} +% Pass remaining document options to the parent class. +\DeclareOption*{\PassOptionsToClass{\CurrentOption}{\sphinxdocclass}} +\ProcessOptions\relax + +% Defaults two-side document +\if@oneside +% nothing to do (oneside is the default) +\else +\PassOptionsToClass{twoside}{\sphinxdocclass} +\fi + +\LoadClass{\sphinxdocclass} + +% Set some sane defaults for section numbering depth and TOC depth. You can +% reset these counters in your preamble. +% +\setcounter{secnumdepth}{2} +\setcounter{tocdepth}{1} + +% Change the title page to look a bit better, and fit in with the fncychap +% ``Bjarne'' style a bit better. +% +\renewcommand{\maketitle}{% + \begin{titlepage}% + \let\footnotesize\small + \let\footnoterule\relax + \rule{\textwidth}{1pt}% + \ifsphinxpdfoutput + \begingroup + % These \defs are required to deal with multi-line authors; it + % changes \\ to ', ' (comma-space), making it pass muster for + % generating document info in the PDF file. + \def\\{, } + \def\and{and } + \pdfinfo{ + /Author (\@author) + /Title (\@title) + } + \endgroup + \fi + \begin{flushright}% + \sphinxlogo% + {\rm\Huge\py@HeaderFamily \@title \par}% + {\em\LARGE\py@HeaderFamily \py@release\releaseinfo \par} + \vfill + {\LARGE\py@HeaderFamily + \begin{tabular}[t]{c} + \@author + \end{tabular} + \par} + \vfill\vfill + {\large + \@date \par + \vfill + \py@authoraddress \par + }% + \end{flushright}%\par + \@thanks + \end{titlepage}% + \cleardoublepage% + \setcounter{footnote}{0}% + \let\thanks\relax\let\maketitle\relax + %\gdef\@thanks{}\gdef\@author{}\gdef\@title{} +} + + +% Catch the end of the {abstract} environment, but here make sure the abstract +% is followed by a blank page if the 'openright' option is used. +% +\let\py@OldEndAbstract=\endabstract +\renewcommand{\endabstract}{ + \if@openright + \ifodd\value{page} + \typeout{Adding blank page after the abstract.} + \vfil\pagebreak + \fi + \fi + \py@OldEndAbstract +} + +% This wraps the \tableofcontents macro with all the magic to get the spacing +% right and have the right number of pages if the 'openright' option has been +% used. This eliminates a fair amount of crud in the individual document files. +% +\let\py@OldTableofcontents=\tableofcontents +\renewcommand{\tableofcontents}{% + \pagenumbering{roman}% + \setcounter{page}{1}% + \pagebreak% + \pagestyle{plain}% + {% + \parskip = 0mm% + \py@OldTableofcontents% + \if@openright% + \ifodd\value{page}% + \typeout{Adding blank page after the table of contents.}% + \pagebreak\hspace{0pt}% + \fi% + \fi% + \cleardoublepage% + }% + \pagenumbering{arabic}% + \@ifundefined{fancyhf}{}{\pagestyle{normal}}% +} +\pagenumbering{alph} + +% This is needed to get the width of the section # area wide enough in the +% library reference. Doing it here keeps it the same for all the manuals. +% +\renewcommand*\l@section{\@dottedtocline{1}{1.5em}{2.6em}} +\renewcommand*\l@subsection{\@dottedtocline{2}{4.1em}{3.5em}} + +% Fix the bibliography environment to add an entry to the Table of +% Contents. +% For a report document class this environment is a chapter. +\let\py@OldThebibliography=\thebibliography +\renewcommand{\thebibliography}[1]{ + \cleardoublepage + \phantomsection + \py@OldThebibliography{1} + \addcontentsline{toc}{chapter}{\bibname} +} + +% Same for the indices. +% The memoir class already does this, so we don't duplicate it in that case. +% +\@ifclassloaded{memoir}{}{ + \let\py@OldTheindex=\theindex + \renewcommand{\theindex}{ + \cleardoublepage + \phantomsection + \py@OldTheindex + \addcontentsline{toc}{chapter}{\indexname} + } +} diff --git a/doc/latex/tabulary.sty b/doc/latex/tabulary.sty new file mode 100644 index 0000000..11fdf74 --- /dev/null +++ b/doc/latex/tabulary.sty @@ -0,0 +1,452 @@ +%% +%% This is file `tabulary.sty', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% tabulary.dtx (with options: `package') +%% DRAFT VERSION +%% +%% File `tabulary.dtx'. +%% Copyright (C) 1995 1996 2003 2008 David Carlisle +%% This file may be distributed under the terms of the LPPL. +%% See 00readme.txt for details. +%% +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{tabulary} + [2008/12/01 v0.9 tabulary package (DPC)] +\RequirePackage{array} +\catcode`\Z=14 +\DeclareOption{debugshow}{\catcode`\Z=9\relax} +\ProcessOptions +\def\arraybackslash{\let\\=\@arraycr} +\def\@finalstrut#1{% + \unskip\ifhmode\nobreak\fi\vrule\@width\z@\@height\z@\@depth\dp#1} +\newcount\TY@count +\def\tabulary{% + \let\TY@final\tabular + \let\endTY@final\endtabular + \TY@tabular} +\def\TY@tabular#1{% + \edef\TY@{\@currenvir}% + {\ifnum0=`}\fi + \@ovxx\TY@linewidth + \@ovyy\TY@tablewidth + \count@\z@ + \@tempswatrue + \@whilesw\if@tempswa\fi{% + \advance\count@\@ne + \expandafter\ifx\csname TY@F\the\count@\endcsname\relax + \@tempswafalse + \else + \expandafter\let\csname TY@SF\the\count@\expandafter\endcsname + \csname TY@F\the\count@\endcsname + \global\expandafter\let\csname TY@F\the\count@\endcsname\relax + \expandafter\let\csname TY@S\the\count@\expandafter\endcsname + \csname TY@\the\count@\endcsname + \fi}% + \global\TY@count\@ne + \TY@width\xdef{0pt}% + \global\TY@tablewidth\z@ + \global\TY@linewidth#1\relax +Z\message{^^J^^JTable^^J% +Z Target Width: \the\TY@linewidth^^J% +Z \string\tabcolsep: \the\tabcolsep\space +Z \string\arrayrulewidth: \the\arrayrulewidth\space +Z \string\doublerulesep: \the\doublerulesep^^J% +Z \string\tymin: \the\tymin\space +Z \string\tymax: \the\tymax^^J}% + \let\@classz\TY@classz + \let\verb\TX@verb + \toks@{}\TY@get@body} +\let\TY@@mkpream\@mkpream +\def\TY@mkpream{% + \def\@addamp{% + \if@firstamp \@firstampfalse \else + \global\advance\TY@count\@ne + \edef\@preamble{\@preamble &}\fi + \TY@width\xdef{0pt}}% + \def\@acol{% + \TY@subwidth\col@sep + \@addtopreamble{\hskip\col@sep}}% + \let\@arrayrule\TY@arrayrule + \let\@classvi\TY@classvi + \def\@classv{\save@decl + \expandafter\NC@ecs\@nextchar\extracolsep{}\extracolsep\@@@ + \sbox\z@{\d@llarbegin\@nextchar\d@llarend}% + \TY@subwidth{\wd\z@}% + \@addtopreamble{\d@llarbegin\the@toks\the\count@\relax\d@llarend}% + \prepnext@tok}% + \global\let\@mkpream\TY@@mkpream + \TY@@mkpream} +\def\TY@arrayrule{% + \TY@subwidth\arrayrulewidth + \@addtopreamble \vline} +\def\TY@classvi{\ifcase \@lastchclass + \@acol \or + \TY@subwidth\doublerulesep + \@addtopreamble{\hskip \doublerulesep}\or + \@acol \or + \@classvii + \fi} +\def\TY@tab{% + \setbox\z@\hbox\bgroup + \let\[$\let\]$% + \let\equation$\let\endequation$% + \col@sep\tabcolsep + \let\d@llarbegin\begingroup\let\d@llarend\endgroup + \let\@mkpream\TY@mkpream + \def\multicolumn##1##2##3{\multispan##1\relax}% + \CT@start\TY@tabarray} +\def\TY@tabarray{\@ifnextchar[{\TY@array}{\@array[t]}} +\def\TY@array[#1]{\@array[t]} +\def\TY@width#1{% + \expandafter#1\csname TY@\the\TY@count\endcsname} +\def\TY@subwidth#1{% + \TY@width\dimen@ + \advance\dimen@-#1\relax + \TY@width\xdef{\the\dimen@}% + \global\advance\TY@linewidth-#1\relax} +\def\endtabulary{% + \gdef\@halignto{}% + \let\TY@footnote\footnote% + \def\footnote{}% prevent footnotes from doing anything + \expandafter\TY@tab\the\toks@ + \crcr\omit + {\xdef\TY@save@row{}% + \loop + \advance\TY@count\m@ne + \ifnum\TY@count>\z@ + \xdef\TY@save@row{\TY@save@row&\omit}% + \repeat}\TY@save@row + \endarray\global\setbox1=\lastbox\setbox0=\vbox{\unvbox1 + \unskip\global\setbox1=\lastbox}\egroup + \dimen@\TY@linewidth + \divide\dimen@\TY@count + \ifdim\dimen@<\tymin + \TY@warn{tymin too large (\the\tymin), resetting to \the\dimen@}% + \tymin\dimen@ + \fi + \setbox\tw@=\hbox{\unhbox\@ne + \loop +\@tempdima=\lastskip +\ifdim\@tempdima>\z@ +Z \message{ecs=\the\@tempdima^^J}% + \global\advance\TY@linewidth-\@tempdima +\fi + \unskip + \setbox\tw@=\lastbox + \ifhbox\tw@ +Z \message{Col \the\TY@count: Initial=\the\wd\tw@\space}% + \ifdim\wd\tw@>\tymax + \wd\tw@\tymax +Z \message{> max\space}% +Z \else +Z \message{ \@spaces\space}% + \fi + \TY@width\dimen@ +Z \message{\the\dimen@\space}% + \advance\dimen@\wd\tw@ +Z \message{Final=\the\dimen@\space}% + \TY@width\xdef{\the\dimen@}% + \ifdim\dimen@<\tymin +Z \message{< tymin}% + \global\advance\TY@linewidth-\dimen@ + \expandafter\xdef\csname TY@F\the\TY@count\endcsname + {\the\dimen@}% + \else + \expandafter\ifx\csname TY@F\the\TY@count\endcsname\z@ +Z \message{***}% + \global\advance\TY@linewidth-\dimen@ + \expandafter\xdef\csname TY@F\the\TY@count\endcsname + {\the\dimen@}% + \else +Z \message{> tymin}% + \global\advance\TY@tablewidth\dimen@ + \global\expandafter\let\csname TY@F\the\TY@count\endcsname + \maxdimen + \fi\fi + \advance\TY@count\m@ne + \repeat}% + \TY@checkmin + \TY@checkmin + \TY@checkmin + \TY@checkmin + \TY@count\z@ + \let\TY@box\TY@box@v + \let\footnote\TY@footnote % restore footnotes + {\expandafter\TY@final\the\toks@\endTY@final}% + \count@\z@ + \@tempswatrue + \@whilesw\if@tempswa\fi{% + \advance\count@\@ne + \expandafter\ifx\csname TY@SF\the\count@\endcsname\relax + \@tempswafalse + \else + \global\expandafter\let\csname TY@F\the\count@\expandafter\endcsname + \csname TY@SF\the\count@\endcsname + \global\expandafter\let\csname TY@\the\count@\expandafter\endcsname + \csname TY@S\the\count@\endcsname + \fi}% + \TY@linewidth\@ovxx + \TY@tablewidth\@ovyy + \ifnum0=`{\fi}} +\def\TY@checkmin{% + \let\TY@checkmin\relax +\ifdim\TY@tablewidth>\z@ + \Gscale@div\TY@ratio\TY@linewidth\TY@tablewidth + \ifdim\TY@tablewidth <\TY@linewidth + \def\TY@ratio{1}% + \fi +\else + \TY@warn{No suitable columns!}% + \def\TY@ratio{1}% +\fi +\count@\z@ +Z \message{^^JLine Width: \the\TY@linewidth, +Z Natural Width: \the\TY@tablewidth, +Z Ratio: \TY@ratio^^J}% +\@tempdima\z@ +\loop +\ifnum\count@<\TY@count +\advance\count@\@ne + \ifdim\csname TY@F\the\count@\endcsname>\tymin + \dimen@\csname TY@\the\count@\endcsname + \dimen@\TY@ratio\dimen@ + \ifdim\dimen@<\tymin +Z \message{Column \the\count@\space ->}% + \global\expandafter\let\csname TY@F\the\count@\endcsname\tymin + \global\advance\TY@linewidth-\tymin + \global\advance\TY@tablewidth-\csname TY@\the\count@\endcsname + \let\TY@checkmin\TY@@checkmin + \else + \expandafter\xdef\csname TY@F\the\count@\endcsname{\the\dimen@}% + \advance\@tempdima\csname TY@F\the\count@\endcsname + \fi + \fi +Z \dimen@\csname TY@F\the\count@\endcsname\message{\the\dimen@, }% +\repeat +Z \message{^^JTotal:\the\@tempdima^^J}% +} +\let\TY@@checkmin\TY@checkmin +\newdimen\TY@linewidth +\def\tyformat{\everypar{{\nobreak\hskip\z@skip}}} +\newdimen\tymin +\tymin=10pt +\newdimen\tymax +\tymax=2\textwidth +\def\@testpach{\@chclass + \ifnum \@lastchclass=6 \@ne \@chnum \@ne \else + \ifnum \@lastchclass=7 5 \else + \ifnum \@lastchclass=8 \tw@ \else + \ifnum \@lastchclass=9 \thr@@ + \else \z@ + \ifnum \@lastchclass = 10 \else + \edef\@nextchar{\expandafter\string\@nextchar}% + \@chnum + \if \@nextchar c\z@ \else + \if \@nextchar l\@ne \else + \if \@nextchar r\tw@ \else + \if \@nextchar C7 \else + \if \@nextchar L8 \else + \if \@nextchar R9 \else + \if \@nextchar J10 \else + \z@ \@chclass + \if\@nextchar |\@ne \else + \if \@nextchar !6 \else + \if \@nextchar @7 \else + \if \@nextchar <8 \else + \if \@nextchar >9 \else + 10 + \@chnum + \if \@nextchar m\thr@@\else + \if \@nextchar p4 \else + \if \@nextchar b5 \else + \z@ \@chclass \z@ \@preamerr \z@ \fi \fi \fi \fi\fi \fi \fi\fi \fi + \fi \fi \fi \fi \fi \fi \fi \fi \fi \fi \fi} +\def\TY@classz{% + \@classx + \@tempcnta\count@ + \ifx\TY@box\TY@box@v + \global\advance\TY@count\@ne + \fi + \let\centering c% + \let\raggedright\noindent + \let\raggedleft\indent + \let\arraybackslash\relax + \prepnext@tok + \ifnum\@chnum<4 + \global\expandafter\let\csname TY@F\the\TY@count\endcsname\z@ + \fi + \ifnum\@chnum=6 + \global\expandafter\let\csname TY@F\the\TY@count\endcsname\z@ + \fi + \@addtopreamble{% + \ifcase\@chnum + \hfil \d@llarbegin\insert@column\d@llarend \hfil \or + \kern\z@ + \d@llarbegin \insert@column \d@llarend \hfil \or + \hfil\kern\z@ \d@llarbegin \insert@column \d@llarend \or + $\vcenter\@startpbox{\@nextchar}\insert@column \@endpbox $\or + \vtop \@startpbox{\@nextchar}\insert@column \@endpbox \or + \vbox \@startpbox{\@nextchar}\insert@column \@endpbox \or + \d@llarbegin \insert@column \d@llarend \or% dubious "s" case + \TY@box\centering\or + \TY@box\raggedright\or + \TY@box\raggedleft\or + \TY@box\relax + \fi}\prepnext@tok} +\def\TY@box#1{% + \ifx\centering#1% + \hfil \d@llarbegin\insert@column\d@llarend \hfil \else + \ifx\raggedright#1% + \kern\z@%<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< + \d@llarbegin \insert@column \d@llarend \hfil \else + \ifx\raggedleft#1% + \hfil\kern\z@ \d@llarbegin \insert@column \d@llarend \else + \ifx\relax#1% + \d@llarbegin \insert@column \d@llarend + \fi \fi \fi \fi} +\def\TY@box@v#1{% + \vtop \@startpbox{\csname TY@F\the\TY@count\endcsname}% + #1\arraybackslash\tyformat + \insert@column\@endpbox} +\newdimen\TY@tablewidth +\def\Gscale@div#1#2#3{% + \setlength\dimen@{#3}% + \ifdim\dimen@=\z@ + \PackageError{graphics}{Division by 0}\@eha + \dimen@#2% + \fi + \edef\@tempd{\the\dimen@}% + \setlength\dimen@{#2}% + \count@65536\relax + \ifdim\dimen@<\z@ + \dimen@-\dimen@ + \count@-\count@ + \fi + \loop + \ifdim\dimen@<8192\p@ + \dimen@\tw@\dimen@ + \divide\count@\tw@ + \repeat + \dimen@ii=\@tempd\relax + \divide\dimen@ii\count@ + \divide\dimen@\dimen@ii + \edef#1{\strip@pt\dimen@}} +\long\def\TY@get@body#1\end + {\toks@\expandafter{\the\toks@#1}\TY@find@end} +\def\TY@find@end#1{% + \def\@tempa{#1}% + \ifx\@tempa\TY@\def\@tempa{\end{#1}}\expandafter\@tempa + \else\toks@\expandafter + {\the\toks@\end{#1}}\expandafter\TY@get@body\fi} +\def\TY@warn{% + \PackageWarning{tabulary}} +\catcode`\Z=11 +\AtBeginDocument{ +\@ifpackageloaded{colortbl}{% +\expandafter\def\expandafter\@mkpream\expandafter#\expandafter1% + \expandafter{% + \expandafter\let\expandafter\CT@setup\expandafter\relax + \expandafter\let\expandafter\CT@color\expandafter\relax + \expandafter\let\expandafter\CT@do@color\expandafter\relax + \expandafter\let\expandafter\color\expandafter\relax + \expandafter\let\expandafter\CT@column@color\expandafter\relax + \expandafter\let\expandafter\CT@row@color\expandafter\relax + \@mkpream{#1}} +\let\TY@@mkpream\@mkpream +\def\TY@classz{% + \@classx + \@tempcnta\count@ + \ifx\TY@box\TY@box@v + \global\advance\TY@count\@ne + \fi + \let\centering c% + \let\raggedright\noindent + \let\raggedleft\indent + \let\arraybackslash\relax + \prepnext@tok +\expandafter\CT@extract\the\toks\@tempcnta\columncolor!\@nil + \ifnum\@chnum<4 + \global\expandafter\let\csname TY@F\the\TY@count\endcsname\z@ + \fi + \ifnum\@chnum=6 + \global\expandafter\let\csname TY@F\the\TY@count\endcsname\z@ + \fi + \@addtopreamble{% + \setbox\z@\hbox\bgroup\bgroup + \ifcase\@chnum + \hskip\stretch{.5}\kern\z@ + \d@llarbegin\insert@column\d@llarend\hskip\stretch{.5}\or + \kern\z@%<<<<<<<<<<<<<<<<<<<<<<<<<<< + \d@llarbegin \insert@column \d@llarend \hfill \or + \hfill\kern\z@ \d@llarbegin \insert@column \d@llarend \or + $\vcenter\@startpbox{\@nextchar}\insert@column \@endpbox $\or + \vtop \@startpbox{\@nextchar}\insert@column \@endpbox \or + \vbox \@startpbox{\@nextchar}\insert@column \@endpbox \or + \d@llarbegin \insert@column \d@llarend \or% dubious s case + \TY@box\centering\or + \TY@box\raggedright\or + \TY@box\raggedleft\or + \TY@box\relax + \fi + \egroup\egroup +\begingroup + \CT@setup + \CT@column@color + \CT@row@color + \CT@do@color +\endgroup + \@tempdima\ht\z@ + \advance\@tempdima\minrowclearance + \vrule\@height\@tempdima\@width\z@ +\unhbox\z@ +}\prepnext@tok}% + \def\TY@arrayrule{% + \TY@subwidth\arrayrulewidth + \@addtopreamble{{\CT@arc@\vline}}}% + \def\TY@classvi{\ifcase \@lastchclass + \@acol \or + \TY@subwidth\doublerulesep + \ifx\CT@drsc@\relax + \@addtopreamble{\hskip\doublerulesep}% + \else + \@addtopreamble{{\CT@drsc@\vrule\@width\doublerulesep}}% + \fi\or + \@acol \or + \@classvii + \fi}% +}{% +\let\CT@start\relax +} +} +{\uccode`\*=`\ % +\uppercase{\gdef\TX@verb{% + \leavevmode\null\TX@vwarn + {\ifnum0=`}\fi\ttfamily\let\\\ignorespaces + \@ifstar{\let~*\TX@vb}{\TX@vb}}}} +\def\TX@vb#1{\def\@tempa##1#1{\toks@{##1}\edef\@tempa{\the\toks@}% + \expandafter\TX@v\meaning\@tempa\\ \\\ifnum0=`{\fi}}\@tempa!} +\def\TX@v#1!{\afterassignment\TX@vfirst\let\@tempa= } +\begingroup +\catcode`\*=\catcode`\# +\catcode`\#=12 +\gdef\TX@vfirst{% + \if\@tempa#% + \def\@tempb{\TX@v@#}% + \else + \let\@tempb\TX@v@ + \if\@tempa\space~\else\@tempa\fi + \fi + \@tempb} +\gdef\TX@v@*1 *2{% + \TX@v@hash*1##\relax\if*2\\\else~\expandafter\TX@v@\fi*2} +\gdef\TX@v@hash*1##*2{*1\ifx*2\relax\else#\expandafter\TX@v@hash\fi*2} +\endgroup +\def\TX@vwarn{% + \@warning{\noexpand\verb may be unreliable inside tabularx/y}% + \global\let\TX@vwarn\@empty} +\endinput +%% +%% End of file `tabulary.sty'. diff --git a/doc/man/fsck.s3ql.1 b/doc/man/fsck.s3ql.1 new file mode 100644 index 0000000..fb05479 --- /dev/null +++ b/doc/man/fsck.s3ql.1 @@ -0,0 +1,196 @@ +.\" Man page generated from reStructuredText. +. +.TH "FSCK.S3QL" "1" "October 28, 2016" "2.21" "S3QL" +.SH NAME +fsck.s3ql \- Check an S3QL file system for errors +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +fsck.s3ql [options] +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DESCRIPTION +.sp +S3QL is a file system for online data storage. Before using S3QL, make +sure to consult the full documentation (rather than just the man pages +which only briefly document the available userspace commands). +.sp +The \fBfsck.s3ql\fP command checks the file system in the location specified +by \fIstorage url\fP for errors and attempts to repair any problems. The +storage url depends on the backend that is used. The S3QL User\(aqs Guide +should be consulted for a description of the available backends. +.SH OPTIONS +.sp +The \fBfsck.s3ql\fP command accepts the following options. +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.BI \-\-log \ +Destination for log messages. Specify \fBnone\fP for +standard output or \fBsyslog\fP for the system logging +daemon. Anything else will be interpreted as a file +name. Log files will be rotated when they reach 1 MiB, +and at most 5 old log files will be kept. Default: +\fB~/.s3ql/fsck.log\fP +.TP +.BI \-\-cachedir \ +Store cached data in this directory (default: +\fB~/.s3ql)\fP +.TP +.BI \-\-authfile \ +Read authentication credentials from this file +(default: \fB~/.s3ql/authinfo2)\fP +.TP +.BI \-\-debug\-modules \ +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\fB\-\-log\fP option. +.TP +.B \-\-debug +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \fB\-\-log\fP option. +.TP +.B \-\-quiet +be really quiet +.TP +.BI \-\-backend\-options \ +Backend specific options (separate by commas). See +backend documentation for available options. +.TP +.B \-\-version +just print program version and exit +.TP +.B \-\-batch +If user input is required, exit without prompting. +.TP +.B \-\-force +Force checking even if file system is marked clean. +.UNINDENT +.UNINDENT +.UNINDENT +.SH EXIT CODES +.sp +If \fBfsck.s3ql\fP found any file system errors (no matter if they were +corrected or not), the exit code will be 128 plus one of the codes +listed below. If no errors were found, the following exit codes are +used as\-is: +.INDENT 0.0 +.TP +.B 0 +Everything went well. +.TP +.B 1 +An unexpected error occured. This may indicate a bug in the +program. +.TP +.B 2 +Invalid command line argument. +.TP +.B 3 +Invalid backend option. +.TP +.B 10 +Could not open log file for writing. +.TP +.B 11 +No such backend. +.TP +.B 12 +Authentication file has insecure permissions. +.TP +.B 13 +Unable to parse proxy settings. +.TP +.B 14 +Invalid credentials (Authentication failed). +.TP +.B 15 +No permission to access backend (Authorization denied). +.TP +.B 16 +Invalid storage URL, specified location does not exist in backend. +.TP +.B 17 +Wrong file system passphrase. +.TP +.B 18 +No S3QL file system found at given storage URL. +.TP +.B 19 +Unable to connect to backend, can\(aqt resolve hostname. +.TP +.B 32 +Unsupported file system revision (too old). +.TP +.B 33 +Unsupported file system revision (too new). +.TP +.B 40 +Cannot check mounted file system. +.TP +.B 41 +User input required, but running in batch mode. +.TP +.B 42 +File system check aborted by user. +.TP +.B 43 +Local metadata is corrupted. +.TP +.B 44 +Uncorrectable errors found. +.TP +.B 45 +Unable to access cache directory. +.TP +.B 128 +This error code will be \fIadded\fP to one of the codes above if any +file system errors have been found (no matter if they were +corrected or not). +.UNINDENT +.SH SEE ALSO +.sp +The S3QL homepage is at \fI\%https://bitbucket.org/nikratio/s3ql/\fP\&. +.sp +The full S3QL documentation should also be installed somewhere on your +system, common locations are \fB/usr/share/doc/s3ql\fP or +\fB/usr/local/doc/s3ql\fP\&. +.SH COPYRIGHT +© 2008 Nikolaus Rath +.\" Generated by docutils manpage writer. +. diff --git a/doc/man/mkfs.s3ql.1 b/doc/man/mkfs.s3ql.1 new file mode 100644 index 0000000..09c9742 --- /dev/null +++ b/doc/man/mkfs.s3ql.1 @@ -0,0 +1,163 @@ +.\" Man page generated from reStructuredText. +. +.TH "MKFS.S3QL" "1" "October 28, 2016" "2.21" "S3QL" +.SH NAME +mkfs.s3ql \- Create an S3QL file system +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +mkfs.s3ql [options] +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DESCRIPTION +.sp +S3QL is a file system for online data storage. Before using S3QL, make +sure to consult the full documentation (rather than just the man pages +which only briefly document the available userspace commands). +.sp +The \fBmkfs.s3ql\fP command creates a new file system in the location +specified by \fIstorage url\fP\&. The storage url depends on the backend +that is used. The S3QL User\(aqs Guide should be consulted for a +description of the available backends. +.sp +Unless you have specified the \fB\-\-plain\fP option, \fBmkfs.s3ql\fP will ask +you to enter an encryption password. This password will \fInot\fP be read +from an authentication file specified with the \fB\-\-authfile\fP +option to prevent accidental creation of an encrypted file system. +.SH OPTIONS +.sp +The \fBmkfs.s3ql\fP command accepts the following options. +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.BI \-\-cachedir \ +Store cached data in this directory (default: +\fB~/.s3ql)\fP +.TP +.BI \-\-authfile \ +Read authentication credentials from this file +(default: \fB~/.s3ql/authinfo2)\fP +.TP +.BI \-\-debug\-modules \ +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\fB\-\-log\fP option. +.TP +.B \-\-debug +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \fB\-\-log\fP option. +.TP +.B \-\-quiet +be really quiet +.TP +.BI \-\-backend\-options \ +Backend specific options (separate by commas). See +backend documentation for available options. +.TP +.B \-\-version +just print program version and exit +.TP +.BI \-L \ +Filesystem label +.TP +.BI \-\-max\-obj\-size \ +Maximum size of storage objects in KiB. Files bigger +than this will be spread over multiple objects in the +storage backend. Default: 10240 KiB. +.TP +.B \-\-plain +Create unencrypted file system. +.TP +.B \-\-force +Overwrite any existing data. +.UNINDENT +.UNINDENT +.UNINDENT +.SH EXIT CODES +.sp +\fBmkfs.s3ql\fP may terminate with the following exit codes: +.INDENT 0.0 +.TP +.B 0 +Everything went well. +.TP +.B 1 +An unexpected error occured. This may indicate a bug in the +program. +.TP +.B 2 +Invalid command line argument. +.TP +.B 3 +Invalid backend option. +.TP +.B 11 +No such backend. +.TP +.B 12 +Authentication file has insecure permissions. +.TP +.B 13 +Unable to parse proxy settings. +.TP +.B 14 +Invalid credentials (Authentication failed). +.TP +.B 15 +No permission to access backend (Authorization denied). +.TP +.B 16 +Invalid storage URL, specified location does not exist in backend. +.TP +.B 19 +Unable to connect to backend, can\(aqt resolve hostname. +.TP +.B 45 +Unable to access cache directory. +.UNINDENT +.SH SEE ALSO +.sp +The S3QL homepage is at \fI\%https://bitbucket.org/nikratio/s3ql/\fP\&. +.sp +The full S3QL documentation should also be installed somewhere on your +system, common locations are \fB/usr/share/doc/s3ql\fP or +\fB/usr/local/doc/s3ql\fP\&. +.SH COPYRIGHT +© 2008 Nikolaus Rath +.\" Generated by docutils manpage writer. +. diff --git a/doc/man/mount.s3ql.1 b/doc/man/mount.s3ql.1 new file mode 100644 index 0000000..412ab9a --- /dev/null +++ b/doc/man/mount.s3ql.1 @@ -0,0 +1,238 @@ +.\" Man page generated from reStructuredText. +. +.TH "MOUNT.S3QL" "1" "October 28, 2016" "2.21" "S3QL" +.SH NAME +mount.s3ql \- Mount an S3QL file system +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +mount.s3ql [options] +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DESCRIPTION +.sp +S3QL is a file system for online data storage. Before using S3QL, make +sure to consult the full documentation (rather than just the man pages +which only briefly document the available userspace commands). +.sp +The \fBmount.s3ql\fP command mounts the S3QL file system stored in \fIstorage +url\fP in the directory \fImount point\fP\&. The storage url depends on the +backend that is used. The S3QL User\(aqs Guide should be consulted for a +description of the available backends. +.SH OPTIONS +.sp +The \fBmount.s3ql\fP command accepts the following options. +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.BI \-\-log \ +Destination for log messages. Specify \fBnone\fP for +standard output or \fBsyslog\fP for the system logging +daemon. Anything else will be interpreted as a file +name. Log files will be rotated when they reach 1 MiB, +and at most 5 old log files will be kept. Default: +\fB~/.s3ql/mount.log\fP +.TP +.BI \-\-cachedir \ +Store cached data in this directory (default: +\fB~/.s3ql)\fP +.TP +.BI \-\-authfile \ +Read authentication credentials from this file +(default: \fB~/.s3ql/authinfo2)\fP +.TP +.BI \-\-debug\-modules \ +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\fB\-\-log\fP option. +.TP +.B \-\-debug +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \fB\-\-log\fP option. +.TP +.B \-\-quiet +be really quiet +.TP +.BI \-\-backend\-options \ +Backend specific options (separate by commas). See +backend documentation for available options. +.TP +.B \-\-version +just print program version and exit +.TP +.BI \-\-cachesize \ +Cache size in KiB (default: autodetect). +.TP +.BI \-\-max\-cache\-entries \ +Maximum number of entries in cache (default: +autodetect). Each cache entry requires one file +descriptor, so if you increase this number you have to +make sure that your process file descriptor limit (as +set with \fBulimit \-n\fP) is high enough (at least the +number of cache entries + 100). +.TP +.B \-\-allow\-other +Normally, only the user who called \fBmount.s3ql\fP can +access the mount point. This user then also has full +access to it, independent of individual file +permissions. If the \fB\-\-allow\-other\fP option is +specified, other users can access the mount point as +well and individual file permissions are taken into +account for all users. +.TP +.B \-\-allow\-root +Like \fB\-\-allow\-other\fP, but restrict access to the +mounting user and the root user. +.TP +.B \-\-fg +Do not daemonize, stay in foreground +.TP +.B \-\-upstart +Stay in foreground and raise SIGSTOP once mountpoint +is up. +.TP +.BI \-\-compress \ +Compression algorithm and compression level to use +when storing new data. \fIalgorithm\fP may be any of +\fBlzma\fP, \fBbzip2\fP, \fBzlib\fP, or none. \fIlvl\fP may be any +integer from 0 (fastest) to 9 (slowest). Default: +\fBlzma\-6\fP +.TP +.BI \-\-metadata\-upload\-interval \ +Interval in seconds between complete metadata uploads. +Set to 0 to disable. Default: 24h. +.TP +.BI \-\-threads \ +Number of parallel upload threads to use (default: +auto). +.TP +.B \-\-nfs +Enable some optimizations for exporting the file +system over NFS. (default: False) +.UNINDENT +.UNINDENT +.UNINDENT +.SH EXIT CODES +.sp +\fBmount.s3ql\fP may terminate with the following exit codes: +.INDENT 0.0 +.TP +.B 0 +Everything went well. +.TP +.B 1 +An unexpected error occured. This may indicate a bug in the +program. +.TP +.B 2 +Invalid command line argument. +.TP +.B 3 +Invalid backend option. +.TP +.B 10 +Could not open log file for writing. +.TP +.B 11 +No such backend. +.TP +.B 12 +Authentication file has insecure permissions. +.TP +.B 13 +Unable to parse proxy settings. +.TP +.B 14 +Invalid credentials (Authentication failed). +.TP +.B 15 +No permission to access backend (Authorization denied). +.TP +.B 16 +Invalid storage URL, specified location does not exist in backend. +.TP +.B 17 +Wrong file system passphrase. +.TP +.B 18 +No S3QL file system found at given storage URL. +.TP +.B 19 +Unable to connect to backend, can\(aqt resolve hostname. +.TP +.B 30 +File system was not unmounted cleanly. +.TP +.B 31 +File system appears to be mounted elsewhere. +.TP +.B 32 +Unsupported file system revision (too old). +.TP +.B 33 +Unsupported file system revision (too new). +.TP +.B 34 +Insufficient free nodes, need to run \fBfsck.s3ql\fP\&. +.TP +.B 35 +Attempted to mount read\-only, this is not supported. +.TP +.B 36 +Mountpoint does not exist. +.TP +.B 37 +Not enough available file descriptors. +.TP +.B 39 +Unable to bind file system to mountpoint. +.TP +.B 45 +Unable to access cache directory. +.UNINDENT +.SH SEE ALSO +.sp +The S3QL homepage is at \fI\%https://bitbucket.org/nikratio/s3ql/\fP\&. +.sp +The full S3QL documentation should also be installed somewhere on your +system, common locations are \fB/usr/share/doc/s3ql\fP or +\fB/usr/local/doc/s3ql\fP\&. +.SH COPYRIGHT +© 2008 Nikolaus Rath +.\" Generated by docutils manpage writer. +. diff --git a/doc/man/s3ql_oauth_client.1 b/doc/man/s3ql_oauth_client.1 new file mode 100644 index 0000000..7c0d1e1 --- /dev/null +++ b/doc/man/s3ql_oauth_client.1 @@ -0,0 +1,111 @@ +.\" Man page generated from reStructuredText. +. +.TH "S3QL_OAUTH_CLIENT" "1" "October 28, 2016" "2.21" "S3QL" +.SH NAME +s3ql_oauth_client \- Obtain Google Storage OAuth2 tokens +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +s3ql_oauth_client [options] +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DESCRIPTION +.sp +S3QL is a file system for online data storage. Before using S3QL, make +sure to consult the full documentation (rather than just the man pages +which only briefly document the available userspace commands). +.sp +The \fBs3ql_oauth_client\fP command may be used to obtain OAuth2 authentication +tokens for use with Google Storage. It requests "user code" from +Google which has to be pasted into the browser to complete the +authentication process interactively. Once authentication in the +browser has been completed, \fBs3ql_oauth_client\fP displays the OAuth2 refresh +token. +.sp +When combined with the special username \fBoauth2\fP, the refresh token +can be used as a backend passphrase when using the Google Storage S3QL +backend. +.SH OPTIONS +.sp +The \fBs3ql_oauth_client\fP command accepts the following options: +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.BI \-\-debug\-modules \ +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\fB\-\-log\fP option. +.TP +.B \-\-debug +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \fB\-\-log\fP option. +.TP +.B \-\-quiet +be really quiet +.TP +.B \-\-version +just print program version and exit +.UNINDENT +.UNINDENT +.UNINDENT +.SH EXIT CODES +.sp +\fBs3ql_oauth_client\fP may terminate with the following exit codes: +.INDENT 0.0 +.TP +.B 0 +Everything went well. +.TP +.B 1 +An unexpected error occured. This may indicate a bug in the +program. +.TP +.B 2 +Invalid command line argument. +.UNINDENT +.SH SEE ALSO +.sp +The S3QL homepage is at \fI\%https://bitbucket.org/nikratio/s3ql/\fP\&. +.sp +The full S3QL documentation should also be installed somewhere on your +system, common locations are \fB/usr/share/doc/s3ql\fP or +\fB/usr/local/doc/s3ql\fP\&. +.SH COPYRIGHT +© 2008 Nikolaus Rath +.\" Generated by docutils manpage writer. +. diff --git a/doc/man/s3ql_verify.1 b/doc/man/s3ql_verify.1 new file mode 100644 index 0000000..ddb03f5 --- /dev/null +++ b/doc/man/s3ql_verify.1 @@ -0,0 +1,184 @@ +.\" Man page generated from reStructuredText. +. +.TH "S3QL_VERIFY" "1" "October 28, 2016" "2.21" "S3QL" +.SH NAME +s3ql_verify \- Verify data in an S3QL file system +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +s3ql_verify [options] +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DESCRIPTION +.sp +S3QL is a file system for online data storage. Before using S3QL, make +sure to consult the full documentation (rather than just the man pages +which only briefly document the available userspace commands). +.sp +The \fBs3ql_verify\fP command verifies all data in the file system. In +contrast to \fBfsck.s3ql\fP, \fBs3ql_verify\fP does not trust the object +listing returned by the backend, but actually attempts to retrieve +every object. It therefore takes a lot longer. +.sp +The format of \fB\fP depends on the backend that is +used. The S3QL User\(aqs Guide should be consulted for a description of +the available backends. +.SH OPTIONS +.sp +The \fBs3ql_verify\fP command accepts the following options. +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.BI \-\-debug\-modules \ +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\fB\-\-log\fP option. +.TP +.B \-\-debug +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \fB\-\-log\fP option. +.TP +.B \-\-quiet +be really quiet +.TP +.B \-\-version +just print program version and exit +.TP +.BI \-\-cachedir \ +Store cached data in this directory (default: +\fB~/.s3ql)\fP +.TP +.BI \-\-authfile \ +Read authentication credentials from this file +(default: \fB~/.s3ql/authinfo2)\fP +.TP +.BI \-\-backend\-options \ +Backend specific options (separate by commas). See +backend documentation for available options. +.TP +.BI \-\-missing\-file \ +File to store keys of missing objects. +.TP +.BI \-\-corrupted\-file \ +File to store keys of corrupted objects. +.TP +.B \-\-data +Read every object completely, instead of checking just +the metadata. +.TP +.BI \-\-parallel \ PARALLEL +Number of connections to use in parallel. +.TP +.BI \-\-start\-with \ +Skip over first objects and with verifying object ++1. +.UNINDENT +.UNINDENT +.UNINDENT +.SH EXIT CODES +.sp +\fBs3ql_verify\fP may terminate with the following exit codes: +.INDENT 0.0 +.TP +.B 0 +Everything went well. +.TP +.B 1 +An unexpected error occured. This may indicate a bug in the +program. +.TP +.B 2 +Invalid command line argument. +.TP +.B 3 +Invalid backend option. +.TP +.B 10 +Could not open log file for writing. +.TP +.B 11 +No such backend. +.TP +.B 12 +Authentication file has insecure permissions. +.TP +.B 13 +Unable to parse proxy settings. +.TP +.B 14 +Invalid credentials (Authentication failed). +.TP +.B 15 +No permission to access backend (Authorization denied). +.TP +.B 16 +Invalid storage URL, specified location does not exist in backend. +.TP +.B 17 +Wrong file system passphrase. +.TP +.B 18 +No S3QL file system found at given storage URL. +.TP +.B 19 +Unable to connect to backend, can\(aqt resolve hostname. +.TP +.B 32 +Unsupported file system revision (too old). +.TP +.B 33 +Unsupported file system revision (too new). +.TP +.B 45 +Unable to access cache directory. +.TP +.B 46 +The file system data was verified, and some objects were found +to be missing or corrupted. +.UNINDENT +.SH SEE ALSO +.sp +The S3QL homepage is at \fI\%https://bitbucket.org/nikratio/s3ql/\fP\&. +.sp +The full S3QL documentation should also be installed somewhere on your +system, common locations are \fB/usr/share/doc/s3ql\fP or +\fB/usr/local/doc/s3ql\fP\&. +.SH COPYRIGHT +© 2008 Nikolaus Rath +.\" Generated by docutils manpage writer. +. diff --git a/doc/man/s3qladm.1 b/doc/man/s3qladm.1 new file mode 100644 index 0000000..f7e8dfb --- /dev/null +++ b/doc/man/s3qladm.1 @@ -0,0 +1,186 @@ +.\" Man page generated from reStructuredText. +. +.TH "S3QLADM" "1" "October 28, 2016" "2.21" "S3QL" +.SH NAME +s3qladm \- Manage S3QL file systems +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +s3qladm [options] +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +where \fBaction\fP may be either of \fBpassphrase\fP, +\fBupgrade\fP, \fBdelete\fP or \fBdownload\-metadata\fP\&. +.SH DESCRIPTION +.sp +S3QL is a file system for online data storage. Before using S3QL, make +sure to consult the full documentation (rather than just the man pages +which only briefly document the available userspace commands). +.sp +The \fBs3qladm\fP command performs various operations on \fIunmounted\fP S3QL +file systems. The file system \fImust not be mounted\fP when using +\fBs3qladm\fP or things will go wrong badly. +.sp +The storage url depends on the backend that is used. The S3QL User\(aqs +Guide should be consulted for a description of the available backends. +.SH OPTIONS +.sp +The \fBs3qladm\fP command accepts the following options. +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.BI \-\-debug\-modules \ +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\fB\-\-log\fP option. +.TP +.B \-\-debug +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \fB\-\-log\fP option. +.TP +.B \-\-quiet +be really quiet +.TP +.BI \-\-log \ +Destination for log messages. Specify \fBnone\fP for +standard output or \fBsyslog\fP for the system logging +daemon. Anything else will be interpreted as a file +name. Log files will be rotated when they reach 1 MiB, +and at most 5 old log files will be kept. Default: +\fBNone\fP +.TP +.BI \-\-authfile \ +Read authentication credentials from this file +(default: \fB~/.s3ql/authinfo2)\fP +.TP +.BI \-\-backend\-options \ +Backend specific options (separate by commas). See +backend documentation for available options. +.TP +.BI \-\-cachedir \ +Store cached data in this directory (default: +\fB~/.s3ql)\fP +.TP +.B \-\-version +just print program version and exit +.UNINDENT +.UNINDENT +.UNINDENT +.sp +Hint: run \fBs3qladm \-\-help\fP to get help on the additional arguments +that the different actions take. +.SH ACTIONS +.sp +The following actions may be specified: +.INDENT 0.0 +.TP +.B passphrase +Changes the encryption passphrase of the file system. +.TP +.B upgrade +Upgrade the file system to the newest revision. +.TP +.B delete +Delete the file system with all the stored data. +.TP +.B download\-metadata +Interactively download backups of the file system metadata. +.UNINDENT +.SH EXIT CODES +.sp +\fBs3qladm\fP may terminate with the following exit codes: +.INDENT 0.0 +.TP +.B 0 +Everything went well. +.TP +.B 1 +An unexpected error occured. This may indicate a bug in the +program. +.TP +.B 2 +Invalid command line argument. +.TP +.B 3 +Invalid backend option. +.TP +.B 10 +Could not open log file for writing. +.TP +.B 11 +No such backend. +.TP +.B 12 +Authentication file has insecure permissions. +.TP +.B 13 +Unable to parse proxy settings. +.TP +.B 14 +Invalid credentials (Authentication failed). +.TP +.B 15 +No permission to access backend (Authorization denied). +.TP +.B 16 +Invalid storage URL, specified location does not exist in backend. +.TP +.B 17 +Wrong file system passphrase. +.TP +.B 18 +No S3QL file system found at given storage URL. +.TP +.B 19 +Unable to connect to backend, can\(aqt resolve hostname. +.TP +.B 45 +Unable to access cache directory. +.UNINDENT +.SH SEE ALSO +.sp +The S3QL homepage is at \fI\%https://bitbucket.org/nikratio/s3ql/\fP\&. +.sp +The full S3QL documentation should also be installed somewhere on your +system, common locations are \fB/usr/share/doc/s3ql\fP or +\fB/usr/local/doc/s3ql\fP\&. +.SH COPYRIGHT +© 2008 Nikolaus Rath +.\" Generated by docutils manpage writer. +. diff --git a/doc/man/s3qlcp.1 b/doc/man/s3qlcp.1 new file mode 100644 index 0000000..f630396 --- /dev/null +++ b/doc/man/s3qlcp.1 @@ -0,0 +1,155 @@ +.\" Man page generated from reStructuredText. +. +.TH "S3QLCP" "1" "October 28, 2016" "2.21" "S3QL" +.SH NAME +s3qlcp \- Copy-on-write replication on S3QL file systems +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +s3qlcp [options] +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DESCRIPTION +.sp +S3QL is a file system for online data storage. Before using S3QL, make +sure to consult the full documentation (rather than just the man pages +which only briefly document the available userspace commands). +.sp +The \fBs3qlcp\fP command duplicates the directory tree \fBsource\-dir\fP +into \fBdest\-dir\fP without physically copying the file contents. +Both source and destination must lie inside the same S3QL file system. +.sp +The replication will not take any additional space. Only if one of +directories is modified later on, the modified data will take +additional storage space. +.sp +\fBs3qlcp\fP can only be called by the user that mounted the file system +and (if the file system was mounted with \fB\-\-allow\-other\fP or \fB\-\-allow\-root\fP) +the root user. +.sp +Note that: +.INDENT 0.0 +.IP \(bu 2 +After the replication, both source and target directory will still +be completely ordinary directories. You can regard \fB\fP as a +snapshot of \fB\fP or vice versa. However, the most common +usage of \fBs3qlcp\fP is to regularly duplicate the same source +directory, say \fBdocuments\fP, to different target directories. For a +e.g. monthly replication, the target directories would typically be +named something like \fBdocuments_January\fP for the replication in +January, \fBdocuments_February\fP for the replication in February etc. +In this case it is clear that the target directories should be +regarded as snapshots of the source directory. +.IP \(bu 2 +Exactly the same effect could be achieved by an ordinary copy +program like \fBcp \-a\fP\&. However, this procedure would be orders of +magnitude slower, because \fBcp\fP would have to read every file +completely (so that S3QL had to fetch all the data over the network +from the backend) before writing them into the destination folder. +.UNINDENT +.SS Snapshotting vs Hardlinking +.sp +Snapshot support in S3QL is inspired by the hardlinking feature that +is offered by programs like \fI\%rsync\fP or +\fI\%storeBackup\fP\&. +These programs can create a hardlink instead of copying a file if an +identical file already exists in the backup. However, using hardlinks +has two large disadvantages: +.INDENT 0.0 +.IP \(bu 2 +backups and restores always have to be made with a special program +that takes care of the hardlinking. The backup must not be touched +by any other programs (they may make changes that inadvertently +affect other hardlinked files) +.IP \(bu 2 +special care needs to be taken to handle files which are already +hardlinked (the restore program needs to know that the hardlink was +not just introduced by the backup program to safe space) +.UNINDENT +.sp +S3QL snapshots do not have these problems, and they can be used with +any backup program. +.SH OPTIONS +.sp +The \fBs3qlcp\fP command accepts the following options: +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.BI \-\-debug\-modules \ +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\fB\-\-log\fP option. +.TP +.B \-\-debug +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \fB\-\-log\fP option. +.TP +.B \-\-quiet +be really quiet +.TP +.B \-\-version +just print program version and exit +.UNINDENT +.UNINDENT +.UNINDENT +.SH EXIT CODES +.sp +\fBs3qlcp\fP may terminate with the following exit codes: +.INDENT 0.0 +.TP +.B 0 +Everything went well. +.TP +.B 1 +An unexpected error occured. This may indicate a bug in the +program. +.TP +.B 2 +Invalid command line argument. +.UNINDENT +.SH SEE ALSO +.sp +The S3QL homepage is at \fI\%https://bitbucket.org/nikratio/s3ql/\fP\&. +.sp +The full S3QL documentation should also be installed somewhere on your +system, common locations are \fB/usr/share/doc/s3ql\fP or +\fB/usr/local/doc/s3ql\fP\&. +.SH COPYRIGHT +© 2008 Nikolaus Rath +.\" Generated by docutils manpage writer. +. diff --git a/doc/man/s3qlctrl.1 b/doc/man/s3qlctrl.1 new file mode 100644 index 0000000..065de71 --- /dev/null +++ b/doc/man/s3qlctrl.1 @@ -0,0 +1,160 @@ +.\" Man page generated from reStructuredText. +. +.TH "S3QLCTRL" "1" "October 28, 2016" "2.21" "S3QL" +.SH NAME +s3qlctrl \- Control a mounted S3QL file system +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +s3qlctrl [options] ... +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +where \fBaction\fP may be either of \fBflushcache\fP, +\fBupload\-meta\fP, \fBcachesize\fP or +\fBlog\-metadata\fP\&. +.SH DESCRIPTION +.sp +S3QL is a file system for online data storage. Before using S3QL, make +sure to consult the full documentation (rather than just the man pages +which only briefly document the available userspace commands). +.sp +The \fBs3qlctrl\fP command performs various actions on the S3QL file system mounted +in \fBmountpoint\fP\&. +.sp +\fBs3qlctrl\fP can only be called by the user that mounted the file system +and (if the file system was mounted with \fB\-\-allow\-other\fP or +\fB\-\-allow\-root\fP) the root user. +.sp +The following actions may be specified: +.INDENT 0.0 +.TP +.B flushcache +Uploads all changed file data to the backend. +.TP +.B upload\-meta +Upload metadata to the backend. All file system operations will +block while a snapshot of the metadata is prepared for upload. +.TP +.B cachesize +Changes the cache size of the file system. This action requires an +additional argument that specifies the new cache size in KiB, so the +complete command line is: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +s3qlctrl [options] cachesize +.ft P +.fi +.UNINDENT +.UNINDENT +.TP +.B log +Change the amount of information that is logged into +\fB~/.s3ql/mount.log\fP file. The complete syntax is: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +s3qlctrl [options] log [ [ ...]] +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +here \fBlevel\fP is the desired new log level and may be either of +\fIdebug\fP, \fIinfo\fP or \fIwarn\fP\&. One or more \fBmodule\fP may only be +specified with the \fIdebug\fP level and allow to restrict the debug +output to just the listed modules. +.UNINDENT +.SH OPTIONS +.sp +The \fBs3qlctrl\fP command also accepts the following options, no matter +what specific action is being invoked: +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.BI \-\-debug\-modules \ +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\fB\-\-log\fP option. +.TP +.B \-\-debug +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \fB\-\-log\fP option. +.TP +.B \-\-quiet +be really quiet +.TP +.B \-\-version +just print program version and exit +.UNINDENT +.UNINDENT +.UNINDENT +.sp +Hint: run \fBs3qlctrl \-\-help\fP to get help on the additional arguments +that the different actions take. +.SH EXIT CODES +.sp +\fBs3qlctrl\fP may terminate with the following exit codes: +.INDENT 0.0 +.TP +.B 0 +Everything went well. +.TP +.B 1 +An unexpected error occured. This may indicate a bug in the +program. +.TP +.B 2 +Invalid command line argument. +.UNINDENT +.SH SEE ALSO +.sp +The S3QL homepage is at \fI\%https://bitbucket.org/nikratio/s3ql/\fP\&. +.sp +The full S3QL documentation should also be installed somewhere on your +system, common locations are \fB/usr/share/doc/s3ql\fP or +\fB/usr/local/doc/s3ql\fP\&. +.SH COPYRIGHT +© 2008 Nikolaus Rath +.\" Generated by docutils manpage writer. +. diff --git a/doc/man/s3qllock.1 b/doc/man/s3qllock.1 new file mode 100644 index 0000000..0452001 --- /dev/null +++ b/doc/man/s3qllock.1 @@ -0,0 +1,141 @@ +.\" Man page generated from reStructuredText. +. +.TH "S3QLLOCK" "1" "October 28, 2016" "2.21" "S3QL" +.SH NAME +s3qllock \- Make trees on an S3QL file system immutable +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +s3qllock [options] +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DESCRIPTION +.sp +S3QL is a file system for online data storage. Before using S3QL, make +sure to consult the full documentation (rather than just the man pages +which only briefly document the available userspace commands). +.sp +The \fBs3qllock\fP command makes a directory tree in an S3QL file +system immutable. Immutable trees can no longer be changed in any way +whatsoever. You can not add new files or directories and you can not +change or delete existing files and directories. The only way to get +rid of an immutable tree is to use the \fBs3qlrm\fP command. +.sp +\fBs3qllock\fP can only be called by the user that mounted the file system +and (if the file system was mounted with \fB\-\-allow\-other\fP or +\fB\-\-allow\-root\fP) the root user. +.SH RATIONALE +.sp +Immutability is a feature designed for backups. Traditionally, backups +have been made on external tape drives. Once a backup was made, the +tape drive was removed and locked somewhere in a shelf. This has the +great advantage that the contents of the backup are now permanently +fixed. Nothing (short of physical destruction) can change or delete +files in the backup. +.sp +In contrast, when backing up into an online storage system like S3QL, +all backups are available every time the file system is mounted. +Nothing prevents a file in an old backup from being changed again +later on. In the worst case, this may make your entire backup system +worthless. Imagine that your system gets infected by a nasty virus +that simply deletes all files it can find \-\- if the virus is active +while the backup file system is mounted, the virus will destroy all +your old backups as well! +.sp +Even if the possibility of a malicious virus or trojan horse is +excluded, being able to change a backup after it has been made is +generally not a good idea. A common S3QL use case is to keep the file +system mounted at all times and periodically create backups with +\fBrsync \-a\fP\&. This allows every user to recover her files from a +backup without having to call the system administrator. However, this +also allows every user to accidentally change or delete files \fIin\fP one +of the old backups. +.sp +Making a backup immutable protects you against all these problems. +Unless you happen to run into a virus that was specifically programmed +to attack S3QL file systems, backups can be neither deleted nor +changed after they have been made immutable. +.SH OPTIONS +.sp +The \fBs3qllock\fP command accepts the following options: +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.BI \-\-debug\-modules \ +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\fB\-\-log\fP option. +.TP +.B \-\-debug +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \fB\-\-log\fP option. +.TP +.B \-\-quiet +be really quiet +.TP +.B \-\-version +just print program version and exit +.UNINDENT +.UNINDENT +.UNINDENT +.SH EXIT CODES +.sp +\fBs3qllock\fP may terminate with the following exit codes: +.INDENT 0.0 +.TP +.B 0 +Everything went well. +.TP +.B 1 +An unexpected error occured. This may indicate a bug in the +program. +.TP +.B 2 +Invalid command line argument. +.UNINDENT +.SH SEE ALSO +.sp +The S3QL homepage is at \fI\%https://bitbucket.org/nikratio/s3ql/\fP\&. +.sp +The full S3QL documentation should also be installed somewhere on your +system, common locations are \fB/usr/share/doc/s3ql\fP or +\fB/usr/local/doc/s3ql\fP\&. +.SH COPYRIGHT +© 2008 Nikolaus Rath +.\" Generated by docutils manpage writer. +. diff --git a/doc/man/s3qlrm.1 b/doc/man/s3qlrm.1 new file mode 100644 index 0000000..f3433b1 --- /dev/null +++ b/doc/man/s3qlrm.1 @@ -0,0 +1,113 @@ +.\" Man page generated from reStructuredText. +. +.TH "S3QLRM" "1" "October 28, 2016" "2.21" "S3QL" +.SH NAME +s3qlrm \- Fast tree removal on S3QL file systems +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +s3qlrm [options] +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DESCRIPTION +.sp +S3QL is a file system for online data storage. Before using S3QL, make +sure to consult the full documentation (rather than just the man pages +which only briefly document the available userspace commands). +.sp +The \fBs3qlrm\fP command recursively deletes files and directories on an +S3QL file system. Although \fBs3qlrm\fP is faster than using e.g. +\fBrm \-r\(ga\fP, the main reason for its existence is that it allows +you to delete immutable trees (which can be created with +\fBs3qllock\fP) as well. +.sp +Be warned that there is no additional confirmation. The directory will +be removed entirely and immediately. +.sp +\fBs3qlrm\fP can only be called by the user that mounted the file system +and (if the file system was mounted with \fB\-\-allow\-other\fP or +\fB\-\-allow\-root\fP) the root user. +.SH OPTIONS +.sp +The \fBs3qlrm\fP command accepts the following options: +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.BI \-\-debug\-modules \ +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\fB\-\-log\fP option. +.TP +.B \-\-debug +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \fB\-\-log\fP option. +.TP +.B \-\-quiet +be really quiet +.TP +.B \-\-version +just print program version and exit +.UNINDENT +.UNINDENT +.UNINDENT +.SH EXIT CODES +.sp +\fBs3qlrm\fP may terminate with the following exit codes: +.INDENT 0.0 +.TP +.B 0 +Everything went well. +.TP +.B 1 +An unexpected error occured. This may indicate a bug in the +program. +.TP +.B 2 +Invalid command line argument. +.UNINDENT +.SH SEE ALSO +.sp +The S3QL homepage is at \fI\%https://bitbucket.org/nikratio/s3ql/\fP\&. +.sp +The full S3QL documentation should also be installed somewhere on your +system, common locations are \fB/usr/share/doc/s3ql\fP or +\fB/usr/local/doc/s3ql\fP\&. +.SH COPYRIGHT +© 2008 Nikolaus Rath +.\" Generated by docutils manpage writer. +. diff --git a/doc/man/s3qlstat.1 b/doc/man/s3qlstat.1 new file mode 100644 index 0000000..7ccbfd5 --- /dev/null +++ b/doc/man/s3qlstat.1 @@ -0,0 +1,110 @@ +.\" Man page generated from reStructuredText. +. +.TH "S3QLSTAT" "1" "October 28, 2016" "2.21" "S3QL" +.SH NAME +s3qlstat \- Gather S3QL file system statistics +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +s3qlstat [options] +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DESCRIPTION +.sp +S3QL is a file system for online data storage. Before using S3QL, make +sure to consult the full documentation (rather than just the man pages +which only briefly document the available userspace commands). +.sp +The \fBs3qlstat\fP command prints statistics about the S3QL file system mounted +at \fBmountpoint\fP\&. +.sp +\fBs3qlstat\fP can only be called by the user that mounted the file system +and (if the file system was mounted with \fB\-\-allow\-other\fP or +\fB\-\-allow\-root\fP) the root user. +.SH OPTIONS +.sp +The \fBs3qlstat\fP command accepts the following options: +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.BI \-\-debug\-modules \ +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\fB\-\-log\fP option. +.TP +.B \-\-debug +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \fB\-\-log\fP option. +.TP +.B \-\-quiet +be really quiet +.TP +.B \-\-version +just print program version and exit +.TP +.B \-\-raw +Do not pretty\-print numbers +.UNINDENT +.UNINDENT +.UNINDENT +.SH EXIT CODES +.sp +\fBs3qlstat\fP may terminate with the following exit codes: +.INDENT 0.0 +.TP +.B 0 +Everything went well. +.TP +.B 1 +An unexpected error occured. This may indicate a bug in the +program. +.TP +.B 2 +Invalid command line argument. +.UNINDENT +.SH SEE ALSO +.sp +The S3QL homepage is at \fI\%https://bitbucket.org/nikratio/s3ql/\fP\&. +.sp +The full S3QL documentation should also be installed somewhere on your +system, common locations are \fB/usr/share/doc/s3ql\fP or +\fB/usr/local/doc/s3ql\fP\&. +.SH COPYRIGHT +© 2008 Nikolaus Rath +.\" Generated by docutils manpage writer. +. diff --git a/doc/man/umount.s3ql.1 b/doc/man/umount.s3ql.1 new file mode 100644 index 0000000..bf5ba9e --- /dev/null +++ b/doc/man/umount.s3ql.1 @@ -0,0 +1,119 @@ +.\" Man page generated from reStructuredText. +. +.TH "UMOUNT.S3QL" "1" "October 28, 2016" "2.21" "S3QL" +.SH NAME +umount.s3ql \- Unmount an S3QL file system +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +umount.s3ql [options] +.ft P +.fi +.UNINDENT +.UNINDENT +.SH DESCRIPTION +.sp +S3QL is a file system for online data storage. Before using S3QL, make +sure to consult the full documentation (rather than just the man pages +which only briefly document the available userspace commands). +.sp +The \fBumount.s3ql\fP command unmounts the S3QL file system mounted in the +directory \fImount point\fP and blocks until all data has been uploaded to +the storage backend. +.sp +Only the user who mounted the file system with \fBmount.s3ql\fP +is able to unmount it with \fBumount.s3ql\fP\&. If you are root and want to +unmount an S3QL file system mounted by an ordinary user, you have to +use the \fBfusermount \-u\fP or \fBumount\fP command instead. +Note that these commands do not block until all data has been +uploaded, so if you use them instead of \fBumount.s3ql\fP then +you should manually wait for the \fBmount.s3ql\fP process to +terminate before shutting down the system. +.SH OPTIONS +.sp +The \fBumount.s3ql\fP command accepts the following options. +.INDENT 0.0 +.INDENT 3.5 +.INDENT 0.0 +.TP +.BI \-\-debug\-modules \ +Activate debugging output from specified modules (use +commas to separate multiple modules). Debug messages +will be written to the target specified by the +\fB\-\-log\fP option. +.TP +.B \-\-debug +Activate debugging output from all S3QL modules. Debug +messages will be written to the target specified by +the \fB\-\-log\fP option. +.TP +.B \-\-quiet +be really quiet +.TP +.B \-\-version +just print program version and exit +.TP +.B \-\-lazy\fP,\fB \-z +Lazy umount. Detaches the file system immediately, +even if there are still open files. The data will be +uploaded in the background once all open files have +been closed. +.UNINDENT +.UNINDENT +.UNINDENT +.SH EXIT CODES +.sp +\fBumount.s3ql\fP may terminate with the following exit codes: +.INDENT 0.0 +.TP +.B 0 +Everything went well. +.TP +.B 1 +An unexpected error occured. This may indicate a bug in the +program. +.TP +.B 2 +Invalid command line argument. +.UNINDENT +.SH SEE ALSO +.sp +The S3QL homepage is at \fI\%https://bitbucket.org/nikratio/s3ql/\fP\&. +.sp +The full S3QL documentation should also be installed somewhere on your +system, common locations are \fB/usr/share/doc/s3ql\fP or +\fB/usr/local/doc/s3ql\fP\&. +.SH COPYRIGHT +© 2008 Nikolaus Rath +.\" Generated by docutils manpage writer. +. diff --git a/doc/manual.pdf b/doc/manual.pdf new file mode 100644 index 0000000..d803b4a Binary files /dev/null and b/doc/manual.pdf differ diff --git a/rst/_static/sphinxdoc.css b/rst/_static/sphinxdoc.css new file mode 100644 index 0000000..e7410f7 --- /dev/null +++ b/rst/_static/sphinxdoc.css @@ -0,0 +1,340 @@ +/** + * Sphinx stylesheet -- sphinxdoc theme + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * + * Originally created by Armin Ronacher for Werkzeug, adapted by Georg Brandl. + */ + +@import url("basic.css"); + +/* -- page layout ----------------------------------------------------------- */ + +body { + font-family: 'Lucida Grande', 'Lucida Sans Unicode', 'Geneva', + 'DejaVu Sans', 'Verdana', sans-serif; + font-size: 14px; + letter-spacing: -0.01em; + line-height: 150%; + text-align: center; + background-color: #BFD1D4; + color: black; + padding: 0; + border: 1px solid #aaa; + + margin: 0px 80px 0px 80px; + min-width: 740px; +} + +div.document { + background-color: white; + text-align: left; + background-image: url(contents.png); + background-repeat: repeat-x; +} + +div.bodywrapper { + margin: 0 240px 0 0; + border-right: 1px solid #ccc; +} + +div.body { + margin: 0; + padding: 0.5em 20px 20px 20px; +} + +div.related { + font-size: 1em; +} + +div.related ul { + background-image: url(navigation.png); + height: 2em; + border-top: 1px solid #ddd; + border-bottom: 1px solid #ddd; +} + +div.related ul li { + margin: 0; + padding: 0; + height: 2em; + float: left; +} + +div.related ul li.right { + float: right; + margin-right: 5px; +} + +div.related ul li a { + margin: 0; + padding: 0 5px 0 5px; + line-height: 1.75em; + color: #EE9816; +} + +div.related ul li a:hover { + color: #3CA8E7; +} + +div.sphinxsidebarwrapper { + padding: 0; +} + +div.sphinxsidebar { + margin: 0; + padding: 0.5em 15px 15px 0; + width: 210px; + float: right; + font-size: 1em; + text-align: left; +} + +div.sphinxsidebar h3, div.sphinxsidebar h4 { + margin: 1em 0 0.5em 0; + font-size: 1em; + padding: 0.1em 0 0.1em 0.5em; + color: white; + border: 1px solid #86989B; + background-color: #AFC1C4; +} + +div.sphinxsidebar h3 a { + color: white; +} + +div.sphinxsidebar ul { + padding-left: 1.5em; + margin-top: 7px; + padding: 0; + line-height: 130%; +} + +div.sphinxsidebar ul ul { + margin-left: 20px; +} + +div.footer { + background-color: #E3EFF1; + color: #86989B; + padding: 3px 8px 3px 0; + clear: both; + font-size: 0.8em; + text-align: right; +} + +div.footer a { + color: #86989B; + text-decoration: underline; +} + +/* -- body styles ----------------------------------------------------------- */ + +p { + margin: 0.8em 0 0.5em 0; +} + +a { + color: #CA7900; + text-decoration: none; +} + +a:hover { + color: #2491CF; +} + +/* div.body a { + text-decoration: underline; +} */ + +h1 { + margin: 0; + padding: 0.7em 0 0.3em 0; + font-size: 1.5em; + color: #11557C; +} + +h2 { + margin: 1.3em 0 0.2em 0; + font-size: 1.35em; + padding: 0; +} + +h3 { + margin: 1em 0 -0.3em 0; + font-size: 1.1em; +} + +div.body h1 a, div.body h2 a, div.body h3 a, div.body h4 a, div.body h5 a, div.body h6 a { + color: black!important; +} + +h1 a.anchor, h2 a.anchor, h3 a.anchor, h4 a.anchor, h5 a.anchor, h6 a.anchor { + display: none; + margin: 0 0 0 0.3em; + padding: 0 0.2em 0 0.2em; + color: #aaa!important; +} + +h1:hover a.anchor, h2:hover a.anchor, h3:hover a.anchor, h4:hover a.anchor, +h5:hover a.anchor, h6:hover a.anchor { + display: inline; +} + +h1 a.anchor:hover, h2 a.anchor:hover, h3 a.anchor:hover, h4 a.anchor:hover, +h5 a.anchor:hover, h6 a.anchor:hover { + color: #777; + background-color: #eee; +} + +a.headerlink { + color: #c60f0f!important; + font-size: 1em; + margin-left: 6px; + padding: 0 4px 0 4px; + text-decoration: none!important; +} + +a.headerlink:hover { + background-color: #ccc; + color: white!important; +} + +cite, code, tt { + font-family: 'Consolas', 'Deja Vu Sans Mono', + 'Bitstream Vera Sans Mono', monospace; + font-size: 0.95em; + letter-spacing: 0.01em; +} + +tt { + background-color: #f2f2f2; + border-bottom: 1px solid #ddd; + color: #333; +} + +tt.descname, tt.descclassname, tt.xref { + border: 0; +} + +hr { + border: 1px solid #abc; + margin: 2em; +} + +a tt { + border: 0; + color: #CA7900; +} + +a tt:hover { + color: #2491CF; +} + +pre { + font-family: 'Deja Vu Sans Mono', + 'Bitstream Vera Sans Mono', monospace; + font-size: 0.95em; + letter-spacing: 0.015em; + line-height: 120%; + padding: 0.5em; + border: 1px solid #ccc; + background-color: #f8f8f8; +} + +pre a { + color: inherit; + text-decoration: underline; +} + +td.linenos pre { + padding: 0.5em 0; +} + +div.quotebar { + background-color: #f8f8f8; + max-width: 250px; + float: right; + padding: 2px 7px; + border: 1px solid #ccc; +} + +div.topic { + background-color: #f8f8f8; +} + +table { + border-collapse: collapse; + margin: 0 -0.5em 0 -0.5em; +} + +table td, table th { + padding: 0.2em 0.5em 0.2em 0.5em; +} + +div.admonition, div.warning { + font-size: 0.9em; + margin: 1em 0 1em 0; + border: 1px solid #86989B; + background-color: #f7f7f7; + padding: 0; +} + +div.admonition p, div.warning p { + margin: 0.5em 1em 0.5em 1em; + padding: 0; +} + +div.admonition pre, div.warning pre { + margin: 0.4em 1em 0.4em 1em; +} + +div.admonition p.admonition-title, +div.warning p.admonition-title { + margin: 0; + padding: 0.1em 0 0.1em 0.5em; + color: white; + border-bottom: 1px solid #86989B; + font-weight: bold; + background-color: #AFC1C4; +} + +div.warning { + border: 1px solid #940000; +} + +div.warning p.admonition-title { + background-color: #CF0000; + border-bottom-color: #940000; +} + +div.admonition ul, div.admonition ol, +div.warning ul, div.warning ol { + margin: 0.1em 0.5em 0.5em 3em; + padding: 0; +} + +div.versioninfo { + margin: 1em 0 0 0; + border: 1px solid #ccc; + background-color: #DDEAF0; + padding: 8px; + line-height: 1.3em; + font-size: 0.9em; +} + +/* Niko's Styles */ + +div.body li p { + margin-bottom: 0.8em; + margin-top: 0.8em; +} + +table.option-list td, +table.option-list th { + border: 0px; +} + +strong.program { + font-weight: normal; + font-style: italic; +} diff --git a/rst/_templates/layout.html b/rst/_templates/layout.html new file mode 100644 index 0000000..5ae78e1 --- /dev/null +++ b/rst/_templates/layout.html @@ -0,0 +1,72 @@ +{% extends "!layout.html" %} + +{# put the sidebar before the body #} +{% block sidebar1 %}{{ sidebar() }}{% endblock %} +{% block sidebar2 %}{% endblock %} + +{# + - Do not mention Python staff in the search box instructions + - Display complete TOC +#} +{%- macro sidebar() %} + {%- if not embedded %}{% if not theme_nosidebar|tobool %} +
+
+ {%- block sidebarlogo %} + {%- if logo %} + + {%- endif %} + {%- endblock %} + {%- block sidebartoc %} +

{{ _('Table Of Contents') }}

+ {{ toctree() }} + {%- endblock %} + {%- block sidebarrel %} + {# + {%- if prev %} +

{{ _('Previous topic') }}

+

{{ prev.title }}

+ {%- endif %} + {%- if next %} +

{{ _('Next topic') }}

+

{{ next.title }}

+ {%- endif %} + #} + {%- endblock %} + {%- block sidebarsourcelink %} + {%- if show_source and has_source and sourcename %} +

{{ _('This Page') }}

+ + {%- endif %} + {%- endblock %} + {%- if customsidebar %} + {% include customsidebar %} + {%- endif %} + {%- block sidebarsearch %} + {%- if pagename != "search" %} + + + {%- endif %} + {%- endblock %} +
+
+ {%- endif %}{% endif %} +{%- endmacro %} diff --git a/rst/about.rst b/rst/about.rst new file mode 100644 index 0000000..75f169e --- /dev/null +++ b/rst/about.rst @@ -0,0 +1,12 @@ +.. -*- mode: rst -*- + +.. include:: ../README.rst + :end-before: Typical Usage + +Contributing +============ + +The S3QL source code is available both on GitHub_ and BitBucket_. + +.. _BitBucket: https://bitbucket.org/nikratio/s3ql/ +.. _GitHub: https://github.com/s3ql/main diff --git a/rst/adm.rst b/rst/adm.rst new file mode 100644 index 0000000..811eb0d --- /dev/null +++ b/rst/adm.rst @@ -0,0 +1,82 @@ +.. -*- mode: rst -*- + + +Managing File Systems +===================== + +The `s3qladm` command performs various operations on *unmounted* S3QL +file systems. The file system *must not be mounted* when using +`s3qladm` or things will go wrong badly. + +The syntax is :: + + s3qladm [options] + +where :var:`action` may be either of :program:`passphrase`, +:program:`upgrade`, :program:`clear` or :program:`download-metadata`. + +The :program:`s3qladm` accepts the following general options, no +matter what specific action is being invoked: + +.. pipeinclude:: python ../bin/s3qladm --help + :start-after: show this help message and exit + + +Changing the Passphrase +----------------------- + +To change the passphrase of a file system, use the `passphrase` +subcommand:: + + s3qladm passphrase + + +Upgrading the file system +------------------------- + +If you have installed a new version of S3QL, it may sometimes be +necessary to upgrade the file system metadata as well. Note that in +this case the file system can no longer be accessed with older +versions of S3QL after the upgrade. + +During the upgrade you have to make sure that the command is not +interrupted, and that no one else tries to mount, check or upgrade the +file system at the same time. + +To upgrade a file system from the previous to the current revision, +execute :: + + s3qladm upgrade + + +Deleting a file system +---------------------- + +A file system can be deleted with:: + + s3qladm clear + +This physically deletes all the data and file system structures. + + +Restoring Metadata Backups +-------------------------- + +If the most-recent copy of the file system metadata has been damaged +irreparably, it is possible to restore one of the automatically +created backup copies. + +The command :: + + s3qladm download-metadata + +will give you a list of the available metadata backups and allow you +to download them. This will create two new files in the current +directory, ending in ``.db`` and ``.params``. To actually use the +downloaded backup, you need to move these files into the ``~/.s3ql/`` +directory and run ``fsck.s3ql``. + +.. WARNING:: + + You should probably not use this functionality without having asked + for help on the mailing list first (see :ref:`resources`). diff --git a/rst/authinfo.rst b/rst/authinfo.rst new file mode 100644 index 0000000..a8ebc9a --- /dev/null +++ b/rst/authinfo.rst @@ -0,0 +1,68 @@ +.. -*- mode: rst -*- + +.. _authinfo: + +==================================== + Storing Authentication Information +==================================== + +Normally, S3QL reads username and password for the backend as well as +an encryption passphrase for the file system from the terminal. Most +commands also accept an :cmdopt:`--authfile` parameter that can be +used to read this information from a file instead. + +The authentication file consists of sections, led by a ``[section]`` +header and followed by ``name: value`` entries. The section headers +themselves are not used by S3QL but have to be unique within the file. + +In each section, the following entries can be defined: + +:storage-url: + Specifies the storage url to which this section applies. If a + storage url starts with the value of this entry, the section is + considered applicable. + +:backend-login: + Specifies the username to use for authentication with the backend. + +:backend-password: + Specifies the password to use for authentication with the backend. + +:fs-passphrase: + Specifies the passphrase to use to decrypt the file system (if + it is encrypted). + + +When reading the authentication file, S3QL considers every applicable +section in order and uses the last value that it found for each entry. +For example, consider the following authentication file:: + + [s3] + storage-url: s3:// + backend-login: joe + backend-password: notquitesecret + + [fs1] + storage-url: s3://joes-first-bucket + fs-passphrase: neitheristhis + + [fs2] + storage-url: s3://joes-second-bucket + fs-passphrase: swordfish + + [fs3] + storage-url: s3://joes-second-bucket/with-prefix + backend-login: bill + backend-password: bi23ll + fs-passphrase: ll23bi + +With this authentication file, S3QL would try to log in as "joe" +whenever the s3 backend is used, except when accessing a storage url +that begins with "s3://joes-second-bucket/with-prefix". In that case, +the last section becomes active and S3QL would use the "bill" +credentials. Furthermore, file system encryption passphrases will be used +for storage urls that start with "s3://joes-first-bucket" or +"s3://joes-second-bucket". + +The authentication file is parsed by the `Python ConfigParser +module `_. diff --git a/rst/backends.rst b/rst/backends.rst new file mode 100644 index 0000000..1373002 --- /dev/null +++ b/rst/backends.rst @@ -0,0 +1,374 @@ +.. -*- mode: rst -*- + +.. _storage_backends: + +================== + Storage Backends +================== + +S3QL supports different *backends* to store data at different service +providers and using different protocols. A *storage url* specifies a +backend together with some backend-specific information and uniquely +identifies an S3QL file system. The form of the storage url depends on +the backend and is described for every backend below. + +Furthermore, every S3QL commands that accepts a storage url also +accepts a :cmdopt:`--backend-options` parameter than can be used to +pass backend-specific options to the backend module. The available +options are documented with the respective backends below. + +All storage backends respect the :envvar:`!http_proxy` (for plain HTTP +connections) and :envvar:`!https_proxy` (for SSL connections) +environment variables. + +.. note:: + + Storage backends are not necessarily compatible. Don't expect that + you can e.g. copy the data stored by the local backend into Amazon + S3 using some non-S3QL tool and then access it with S3QL's S3 + backend. If you want to copy file systems from one backend to + another, you need to use the :file:`clone_fs.py` script (from the + :file:`contrib` directory in the S3QL tarball). + +Google Storage +============== + +.. program:: gs_backend + +`Google Storage `_ is an online +storage service offered by Google. To use the Google Storage backend, +you need to have (or sign up for) a Google account, and then `activate +Google Storage `_ +for your account. The account is free, you will pay only for the +amount of storage and traffic that you actually use. There are two +ways to access Google storage: + +#. Use S3-like authentication. To do this, first `set a default + project + `_. + Then use the `key management tool + `_ to + retrieve your *Google Storage developer access key* and *Google + Storage developer secret* and use that as backend login and backend + password. + +#. Use OAuth2 authentication. In this case you need to use ``oauth2`` + as the backend login, and a valid OAuth2 refresh token as the + backend password. To obtain a refresh token, you can use the + :ref:`s3ql_oauth_client ` program. It will instruct + you to open a specific URL in your browser, enter a code and + authenticate with your Google account. Once this procedure is + complete, :ref:`s3ql_oauth_client ` will print out + the refresh token. Note that you need to do this procedure only + once, the refresh token will remain valid until you explicitly + revoke it. + +To create a Google Storage bucket, you can use e.g. the `Google +Storage Manager`_. The storage URL for accessing the bucket in S3QL is +then :: + + gs:/// + +Here *bucketname* is the name of the bucket, and *prefix* can be an +arbitrary prefix that will be prepended to all object names used by +S3QL. This allows you to store several S3QL file systems in the same +Google Storage bucket. + +The Google Storage backend accepts the following backend options: + +.. option:: no-ssl + + Disable encrypted (https) connections and use plain HTTP instead. + +.. option:: ssl-ca-path= + + Instead of using the system's default certificate store, validate + the server certificate against the specified CA + certificates. :var:`` may be either a file containing + multiple certificates, or a directory containing one certificate + per file. + +.. option:: tcp-timeout + + Specifies the timeout used for TCP connections. If no data can be + exchanged with the remote server for longer than this period, the + TCP connection is closed and re-established (default: 20 seconds). + +.. _`Google Storage Manager`: https://sandbox.google.com/storage/ + +Amazon S3 +========= + +.. program:: s3_backend + +`Amazon S3 `_ is the online storage service +offered by `Amazon Web Services (AWS) `_. To +use the S3 backend, you first need to sign up for an AWS account. The +account is free, you will pay only for the amount of storage and +traffic that you actually use. After that, you need to create a bucket +that will hold the S3QL file system, e.g. using the `AWS Management +Console `_. For best +performance, it is recommend to create the bucket in the +geographically closest storage region, but not the US Standard region +(see :ref:`durability` for the reason). + +The storage URL for accessing S3 buckets in S3QL has the form :: + + s3:/// + +Here *bucketname* is the name of the bucket, and *prefix* can be an +arbitrary prefix that will be prepended to all object names used by +S3QL. This allows you to store several S3QL file systems in the same +S3 bucket. + +Note that the backend login and password for accessing S3 are not the +user id and password that you use to log into the Amazon Webpage, but +the *AWS access key id* and *AWS secret access key* shown under `My +Account/Access Identifiers +`_. + +The Amazon S3 backend accepts the following backend options: + +.. option:: no-ssl + + Disable encrypted (https) connections and use plain HTTP instead. + +.. option:: ssl-ca-path= + + Instead of using the system's default certificate store, validate + the server certificate against the specified CA + certificates. :var:`` may be either a file containing + multiple certificates, or a directory containing one certificate + per file. + +.. option:: tcp-timeout + + Specifies the timeout used for TCP connections. If no data can be + exchanged with the remote server for longer than this period, the + TCP connection is closed and re-established (default: 20 seconds). + +.. option:: sse + + Enable server side encryption. Both costs & benefits of S3 server + side encryption are probably rather small, and this option does + *not* affect any client side encryption performed by S3QL itself. + +.. option:: ia + + Use infrequent access storage class for new objects. + +.. option:: rrs + + Enable reduced redundancy storage for newly created objects + (overwrites the *ia* option). + + When enabling this option, it is strongly recommended to + periodically run :ref:`s3ql_verify `, because objects + that are lost by the storage backend may cause subsequent data loss + even later in time due to the data de-duplication feature of S3QL (see + :ref:`backend_reliability` for details). + + +.. _openstack_backend: + +OpenStack/Swift +=============== + +.. program:: swift_backend + +OpenStack_ is an open-source cloud server application suite. Swift_ is +the cloud storage module of OpenStack. Swift/OpenStack storage is +offered by many different companies. + +There are two different storage URL for the OpenStack backend that +make use of different authentication APIs. For legacy (v1) +authentication, the storage URL is :: + + swift://[:]/[/] + +for keystore (v2) authentication, the storage URL is :: + + swiftks://[:]/:[/] + +Note that when using keystore authentication, you can (and have to) +specify the storage region of the container as well. + +In both cases, *hostname* name should be the name of the +authentication server. The storage container must already exist (most +OpenStack providers offer either a web frontend or a command line tool +for creating containers). *prefix* can be an arbitrary prefix that +will be prepended to all object names used by S3QL, which can be used +to store multiple S3QL file systems in the same container. + +When using legacy authentication, the backend login and password +correspond to the OpenStack username and API Access Key. When using +keystore authentication, the backend password is your regular +OpenStack password and the backend login combines you OpenStack +username and tenant name in the form `:`. If no tenant +is required, the OpenStack username alone may be used as backend +login. + +The OpenStack backend accepts the following backend options: + +.. option:: no-ssl + + Use plain HTTP to connect to the authentication server. This option + does not directly affect the connection to the storage + server. Whether HTTPS or plain HTTP is used to connect to the + storage server is determined by the authentication server. + +.. option:: ssl-ca-path= + + Instead of using the system's default certificate store, validate + the server certificate against the specified CA + certificates. :var:`` may be either a file containing + multiple certificates, or a directory containing one certificate + per file. + +.. option:: tcp-timeout + + Specifies the timeout used for TCP connections. If no data can be + exchanged with the remote server for longer than this period, the + TCP connection is closed and re-established (default: 20 seconds). + +.. option:: disable-expect100 + + If this option is specified, S3QL does not use the ``Expect: + continue`` header (cf. `RFC2616, section 8.2.3`__) when uploading + data to the server. This can be used to work around broken storage + servers that don't fully support HTTP 1.1, but may decrease + performance as object data will be transmitted to the server more + than once in some circumstances. + +.. __: http://tools.ietf.org/html/rfc2616#section-8.2.3 +.. _OpenStack: http://www.openstack.org/ +.. _Swift: http://openstack.org/projects/storage/ + + +Rackspace CloudFiles +==================== + +Rackspace_ CloudFiles uses OpenStack_ internally, so it is possible to +just use the OpenStack/Swift backend (see above) with +``auth.api.rackspacecloud.com`` as the host name. For convenince, +there is also a special ``rackspace`` backend that uses a storage URL +of the form :: + + rackspace:///[/] + +The storage container must already exist in the selected +region. *prefix* can be an arbitrary prefix that will be prepended to +all object names used by S3QL and can be used to store several S3QL +file systems in the same container. + +You can create a storage container for S3QL using the `Cloud Control +Panel `_ (click on *Files* in the +topmost menu bar). + +The Rackspace backend accepts the same backend options as the +:ref:`OpenStack backend `. + +.. NOTE:: + + As of January 2012, Rackspace does not give any durability or + consistency guarantees (see :ref:`durability` for why this is + important). However, Rackspace support agents seem prone to claim + very high guarantees. Unless explicitly backed by their terms of + service, any such statement should thus be viewed with + suspicion. S3QL developers have also `repeatedly experienced + `_ + similar issues with the credibility and competence of the Rackspace + support. + + +.. _Rackspace: http://www.rackspace.com/ + + +S3 compatible +============= + +.. program:: s3c_backend + +The S3 compatible backend allows S3QL to access any storage service +that uses the same protocol as Amazon S3. The storage URL has the form :: + + s3c://:// + +Here *bucketname* is the name of an (existing) bucket, and *prefix* +can be an arbitrary prefix that will be prepended to all object names +used by S3QL. This allows you to store several S3QL file systems in +the same bucket. + +The S3 compatible backend accepts the following backend options: + +.. option:: no-ssl + + Disable encrypted (https) connections and use plain HTTP instead. + +.. option:: ssl-ca-path= + + Instead of using the system's default certificate store, validate + the server certificate against the specified CA + certificates. :var:`` may be either a file containing + multiple certificates, or a directory containing one certificate + per file. + +.. option:: tcp-timeout + + Specifies the timeout used for TCP connections. If no data can be + exchanged with the remote server for longer than this period, the + TCP connection is closed and re-established (default: 20 seconds). + +.. option:: disable-expect100 + + If this option is specified, S3QL does not use the ``Expect: + continue`` header (cf. `RFC2616, section 8.2.3`__) when uploading + data to the server. This can be used to work around broken storage + servers that don't fully support HTTP 1.1, but may decrease + performance as object data will be transmitted to the server more + than once in some circumstances. + +.. __: http://tools.ietf.org/html/rfc2616#section-8.2.3 + +.. option:: dumb-copy + + If this option is specified, S3QL assumes that a COPY request to + the storage server has succeeded as soon as the server returns a + ``200 OK`` status. The `S3 COPY API`_ specifies that the + storage server may still return an error in the request body (see + the `copy proposal`__ for the rationale), so this + option should only be used if you are certain that your storage + server only returns ``200 OK`` when the copy operation has been + completely and successfully carried out. Using this option may be + neccessary if your storage server does not return a valid response + body for a succesfull copy operation. + +.. _`S3 COPY API`: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html +.. __: https://doc.s3.amazonaws.com/proposals/copy.html + + +Local +===== + +S3QL is also able to store its data on the local file system. This can +be used to backup data on external media, or to access external +services that S3QL can not talk to directly (e.g., it is possible to +store data over SSH by first mounting the remote system using sshfs_ +and then using the local backend to store the data in the sshfs +mountpoint). + +The storage URL for local storage is :: + + local:// + +Note that you have to write three consecutive slashes to specify an +absolute path, e.g. `local:///var/archive`. Also, relative paths will +automatically be converted to absolute paths before the authentication +file (see :ref:`authinfo`) is read, i.e. if you are in the +`/home/john` directory and try to mount `local://s3ql`, the +corresponding section in the authentication file must match the +storage url `local:///home/john/s3ql`. + +The local backend does not accept any backend options. + +.. _sshfs: http://fuse.sourceforge.net/sshfs.html diff --git a/rst/conf.py b/rst/conf.py new file mode 100644 index 0000000..e8dad82 --- /dev/null +++ b/rst/conf.py @@ -0,0 +1,244 @@ +# -*- coding: utf-8 -*- +# +# This file containts the Sphinx configuration to generate the +# HTML and PDF documentation from the plain text (RST) source. +# +# This file does not contain any S3QL documentation itself. +# + +# Add a custom role for command line options that does not try to +# reference anything. +def add_literal_role(rolename): + from docutils.parsers.rst import roles + from docutils import nodes + nodeclass = nodes.literal + generic = roles.GenericRole(rolename, nodeclass) + role = roles.CustomRole(rolename, generic, {'classes': [rolename]}) + roles.register_local_role(rolename, role) +add_literal_role('cmdopt') +add_literal_role('var') # A variable defined in a :samp: role + +# Add our own Pygments Lexer +import pygments.lexers._mapping as pmap +pmap.LEXERS['CommandLineLexer'] = ('cmdline_lexer', 'CommandLine', + ('CommandLine', 'commandline'), (), ()) + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.append(os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ 'sphinx.ext.ifconfig', 'sphinx_pipeinclude' ] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# Warn about unresolved references +nitpicky = True + +# General information about the project. +project = u'S3QL' +copyright = u'© 2008 Nikolaus Rath ' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +#version = +# The full version, including alpha/beta/rc tags. +#release = + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = [ 'include' ] + +# The reST default role (used for this markup: `text`) to use for all documents. +default_role = 'file' + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'colorful' +highlight_language = 'commandline' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'sphinxdoc' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {'stickysidebar': 'true'} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +html_use_modindex = False + +# If false, no index is generated. +html_use_index = False + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +html_show_sourcelink = False + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'S3QLdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'manual.tex', u'S3QL Documentation', + u'Nikolaus Rath', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('man/mkfs', 'mkfs.s3ql', u'Create an S3QL file system', + None, 1), + ('man/mount', 'mount.s3ql', u'Mount an S3QL file system', + None, 1), + ('man/umount', 'umount.s3ql', u'Unmount an S3QL file system', + None, 1), + ('man/fsck', 'fsck.s3ql', u'Check an S3QL file system for errors', + None, 1), + ('man/ctrl', 's3qlctrl', u'Control a mounted S3QL file system', + None, 1), + ('man/adm', 's3qladm', u'Manage S3QL file systems', + None, 1), + ('man/cp', 's3qlcp', u'Copy-on-write replication on S3QL file systems', + None, 1), + ('man/lock', 's3qllock', u'Make trees on an S3QL file system immutable', + None, 1), + ('man/rm', 's3qlrm', u'Fast tree removal on S3QL file systems', + None, 1), + ('man/stat', 's3qlstat', u'Gather S3QL file system statistics', + None, 1), + ('man/expire_backups', 'expire_backups', u'Intelligently expire old backups', + None, 1), + ('man/pcp', 'pcp', u'Recursive, parallel copy of directory trees', + None, 1), + ('man/oauth_client', 's3ql_oauth_client', u'Obtain Google Storage OAuth2 tokens', + None, 1), + ('man/verify', 's3ql_verify', u'Verify data in an S3QL file system', + None, 1), +] diff --git a/rst/contrib.rst b/rst/contrib.rst new file mode 100644 index 0000000..b8a89d5 --- /dev/null +++ b/rst/contrib.rst @@ -0,0 +1,90 @@ +.. -*- mode: rst -*- + +===================== +Contributed Programs +===================== + +S3QL comes with a few contributed programs that are not part of the +core distribution (and are therefore not installed automatically by +default), but which may nevertheless be useful. These programs are in +the `contrib` directory of the source distribution or in +`/usr/share/doc/s3ql/contrib` if you installed S3QL from a package. + + +benchmark.py +============ + +This program measures S3QL write performance, uplink bandwidth and +compression speed to determine the limiting factor. It also gives +recommendation for compression algorithm and number of upload threads +to achieve maximum performance. + + +clone_fs.py +=========== + +This program physically clones an S3QL file system from one backend +into another, without recompressing or reencrypting. It can be used to +migrate S3 buckets to a different storage region or storage class +(standard or reduced redundancy). + +.. _pcp: + +pcp.py +====== + +``pcp.py`` is a wrapper program that starts several rsync processes to +copy directory trees in parallel. This is important because +transferring files in parallel significantly enhances performance when +copying data from an S3QL file system (see :ref:`copy_performance` for +details). + +To recursively copy the directory ``/mnt/home-backup`` into +``/home/joe`` using 8 parallel processes and preserving permissions, +you would execute :: + + pcp.py -a --processes=8 /mnt/home-backup/ /home/joe + + +s3ql_backup.sh +============== + +This is an example script that demonstrates how to set up a simple but +powerful backup solution using S3QL and `rsync +`_. + +The `s3ql_backup.sh` script automates the following steps: + +#. Mount the file system +#. Replicate the previous backup with :ref:`s3qlcp ` +#. Update the new copy with the data from the backup source using rsync +#. Make the new backup immutable with :ref:`s3qllock ` +#. Delete old backups that are no longer needed +#. Unmount the file system + +The backups are stored in directories of the form +`YYYY-MM-DD_HH:mm:SS` and the `expire_backups.py`_ command is used to +delete old backups. + + +expire_backups.py +================= + +:program:`expire_backups.py` is a program to intelligently remove old +backups that are no longer needed. + +.. include:: man/expire_backups.rst + :start-after: begin_main_content + :end-before: end_main_content + +For a full list of available options, run :program:`expire_backups.py +--help`. + +.. _remove_objects: + +remove_objects.py +================= + +:program:`remove_objects.py` is a program to remove a list of objects +from a storage backend. Since it acts on the backend-level, the +backend need not contain an S3QL file system. diff --git a/rst/durability.rst b/rst/durability.rst new file mode 100644 index 0000000..a9792aa --- /dev/null +++ b/rst/durability.rst @@ -0,0 +1,239 @@ +.. -*- mode: rst -*- + +.. _durability: + +======================================= + Important Rules to Avoid Losing Data +======================================= + +Most S3QL backends store data in distributed storage systems. These +systems differ from a traditional, local hard disk in several +important ways. In order to avoid losing data, this section should be +read very carefully. + +Rules in a Nutshell +=================== + +To avoid losing your data, obey the following rules: + +#. Know what durability you can expect from your chosen storage + provider. The durability describes how likely it is that a stored + object becomes damaged over time. Such data corruption can never be + prevented completely, techniques like geographic replication and + RAID storage just reduce the likelihood of it to happen (i.e., + increase the durability). + +#. When choosing a backend and storage provider, keep in mind that + when using S3QL, the effective durability of the file system data + will be reduced because of S3QL's data de-duplication feature. + +#. Determine your storage service's consistency window. The + consistency window that is important for S3QL is the smaller of the + times for which: + + - a newly created object may not yet be included in the list of + stored objects + + - an attempt to read a newly created object may fail with the + storage service reporting that the object does not exist + + If *one* of the above times is zero, we say that as far as S3QL is + concerned the storage service has *immediate* consistency. + + If your storage provider claims that *neither* of the above can + ever happen, while at the same time promising high durability, you + should choose a respectable provider instead. + +#. When mounting the same file system on different computers (or on + the same computer but with different :cmdopt:`--cachedir` directories), + the time that passes between the first and second of invocation of + :program:`mount.s3ql` must be at least as long as your storage + service's consistency window. If your storage service offers + immediate consistency, you do not need to wait at all. + +#. Before running :program:`fsck.s3ql` or :program:`s3qladm`, the file system + must have been left untouched for the length of the consistency + window. If your storage service offers immediate consistency, you + do not need to wait at all. + +The rest of this section explains the above rules and the reasons for +them in more detail. It also contains a list of the consistency +windows for a number of larger storage providers. + +Consistency Window List +======================= + +The following is a list of the consistency windows (as far as S3QL is +concerned) for a number of storage providers. This list doesn't come +with any guarantees and may be outdated. If your storage provider is +not included, or if you need more reliable information, check with +your storage provider. + +======================================= =================== +Storage Provider Consistency Window +======================================= =================== +Amazon S3 in the US standard region No guarantees +Amazon S3 in other regions Immediate +Google Storage Immediate +======================================= =================== + + + +Data Consistency +================ + +In contrast to the typical hard disk, most storage providers do not +guarantee *immediate consistency* of written data. This means that: + +* after an object has been stored, requests to read this object may + still fail or return the prior contents for a little while. + +* after an object has been deleted, attempts to read it may still + return the (old) data for some time, and it may still remain in the + list of stored objects for some time. + +* after a new object has been created, it may still not be included + when retrieving the list of stored objects for some time. + +Of course, none of this is acceptable for a file system, and S3QL +generally handles any of the above situations internally so that it +always provides a fully consistent file system to the user. However, +there are some situations where an S3QL user nevertheless needs to be +aware of the peculiarities of his chosen storage service. + +Suppose that you mount the file system, store some new data, delete +some old data and unmount it. If you then mount the file system again +right away on another computer, there is no guarantee that S3QL will +see any of the changes that the first S3QL process has made. At least +in theory it is therefore possible that on the second mount, S3QL does +not see any of the changes that you have done and presents you an "old +version" of the file system without them. Even worse, if you notice +the problem and unmount the file system, S3QL will upload the old +status (which S3QL necessarily has to consider as current) and thereby +permanently override the newer version (even though this change may +not become immediately visible either). S3QL uses several techniques +to reduce the likelihood of this to happen (see :ref:`impl_details` +for more information on this), but without support from the storage +service, the possibility cannot be eliminated completely. + +The same problem of course also applies when checking the file system. +If the storage service provides S3QL with only partially updated data, +S3QL has no way to find out if this a real consistency problem that +needs to be fixed or if it is only a temporary problem that will +resolve itself automatically (because there are still changes that +have not become visible yet). + +This is where the so called *consistency window* comes in. The +consistency window is the maximum time (after writing or deleting the +object) for which any of the above "outdated responses" may be +received. If the consistency window is zero, i.e. all changes are +immediately effective, the storage service is said to have *immediate +consistency*. If the window is infinite, i.e. there is no upper bound +on the time it may take for changes to become effect, the storage +service is said to be *eventually consistent*. Note that often there +are different consistency windows for the different operations. For +example, Google Storage offers immediate consistency when reading +data, but only eventual consistency for the list of stored objects. + +To prevent the problem of S3QL working with an outdated copy of the +file system data, it is therefore sufficient to simply wait for the +consistency window to pass before mounting the file system again (or +running a file system check). The length of the consistency window +changes from storage service to storage service, and if your service +is not included in the list below, you should check the web page or +ask the technical support of your storage provider. The window that is +important for S3QL is the smaller of the times for which + +- a newly created object may not yet be included in the list of + stored objects + +- an attempt to read a newly created object may fail with the + storage service reporting that the object does not exist + + +Unfortunately, many storage providers are hesitant to guarantee +anything but eventual consistency, i.e. the length of the consistency +window is potentially infinite. In that case you simply have to pick a +length that you consider "safe enough". For example, even though +Amazon is only guaranteeing eventual consistency, the ordinary +consistency window for data stored in S3 is just a few seconds, and +only in exceptional circumstances (i.e., core network outages) it may +rise up to hours (`source +`_). + + +.. _backend_reliability: + +Data Durability +=============== + +The durability of a storage service a measure of the average +probability of a storage object to become corrupted over time. The +lower the chance of data loss, the higher the durability. Storage +services like Amazon S3 claim to achieve a durability of up to +99.999999999% over a year, i.e. if you store 100000000 objects for 100 +years, you can expect that at the end of that time one object will be +corrupted or lost. + +S3QL is designed to reduce redundancy and store data in the smallest +possible form. Therefore, S3QL is generally not able to compensate for +any such losses, and when choosing a storage service you should +carefully review if the offered durability matches your requirements. +When doing this, there are two factors that should be kept in mind. + +Firstly, even though S3QL is not able to compensate for storage +service failures, it is able to detect them: when trying to access +data that has been lost or corrupted by the storage service, an IO +error will be returned and the mount point will become inaccessible to +ensure that the problem is noticed. + +Secondly, the consequences of a data loss by the storage service can +be significantly more severe than you may expect because of S3QL's +data de-duplication feature: a data loss in the storage service at +time *x* may cause data that is written *after* time *x* to be lost as +well. Consider the following scenario: + +#. You store an important file in the S3QL file system. +#. The storage service loses the data blocks of this file. As long as you + do not access the file or run :program:`fsck.s3ql`, S3QL is not + aware that the data has been lost by the storage service. +#. You save an additional copy of the important file in a different + location on the same S3QL file system. +#. S3QL detects that the contents of the new file are identical to the + data blocks that have been stored earlier. Since at this point S3QL + is not aware that these blocks have been lost by the storage service, it + does not save another copy of the file contents in the storage service but + relies on the (presumably) existing blocks instead. +#. Therefore, even though you saved another copy, you still do not + have a backup of the important file (since both copies refer to the + same data blocks that have been lost by the storage service). + +For some storage services, :program:`fsck.s3ql` can mitigate this +effect. When :program:`fsck.s3ql` runs, it asks the storage service +for a list of all stored objects. If objects are missing, it can then +mark the damaged files and prevent the problem from spreading forwards +in time. Figuratively speaking, this establishes a "checkpoint": data +loss that occurred before running :program:`fsck.s3ql` can not affect +any file system operations that are performed after the check. +Unfortunately, many storage services only "discover" that objects are +missing or broken when the object actually needs to be retrieved. In +this case, :program:`fsck.s3ql` will not learn anything by just +querying the list of objects. + +This effect can be mitigated to some degree by using the +:program:`s3ql_verify` command in additon to +:program:`fsck.s3ql`. :program:`s3ql_verify` asks the storage service +to look up every stored object and may therefore take much longer than +running :program:`fsck.s3ql`, but can also offer a much stronger +assurance that no data has been lost by the storage service. To +"recover" from damaged storage objects in the backend, the damaged +objects found by :program:`s3ql_verify` have to be explicitly deleted +(so that a successive :program:`fsck.s3ql` is able detect them as +missing, correct the file system metadata, and move any affected files +to :file:`lost+found`). This procedure is currently not automated, so +it is generally a good idea to choose a storage service where the +expected data durability is high enough so that the possibility of a +lost object (and thus the need to run any full checks) can be +neglected over long periods of time. + + diff --git a/rst/fsck.rst b/rst/fsck.rst new file mode 100644 index 0000000..58e4993 --- /dev/null +++ b/rst/fsck.rst @@ -0,0 +1,86 @@ +.. -*- mode: rst -*- + +Checking for Errors +=================== + +It is recommended to periodically run the :program:`fsck.s3ql` and +:program:`s3ql_verify` commands (in this order) to ensure that the +file system is consistent, and that there has been no data corruption +or data loss in the storage backend. + +:program:`fsck.s3ql` is intended to detect and correct problems with +the internal file system structure, caused by e.g. a file system crash +or a bug in S3QL. It assumes that the storage backend can be fully +trusted, i.e. if the backend reports that a specific storage object +exists, :program:`fsck.s3ql` takes that as proof that the data is +present and intact. + +In contrast to that, the :program:`s3ql_verify` command is intended to +check the consistency of the storage backend. It assumes that the +internal file system data is correct, and verifies that all data can +actually be retrieved from the backend. Running :program:`s3ql_verify` +may therefore take much longer than running :program:`fsck.s3ql`. + + +Checking and repairing internal file system errors +-------------------------------------------------- + +:program:`fsck.s3ql` checks that the internal file system structure is +consistent and attempts to correct any problems it finds. If an S3QL +file system has not been unmounted correcly for any reason, you need +to run :program:`fsck.s3ql` before you can mount the file system +again. + +The :program:`fsck.s3ql` command has the following syntax:: + + fsck.s3ql [options] + +This command accepts the following options: + +.. pipeinclude:: python ../bin/fsck.s3ql --help + :start-after: show this help message and exit + + +.. _s3ql_verify: + +Detecting and handling backend data corruption +---------------------------------------------- + +The :program:`s3ql_verify` command verifies all data in the file +system. In contrast to :program:`fsck.s3ql`, :program:`s3ql_verify` +does not trust the object listing returned by the backend, but +actually attempts to retrieve every object. By default, +:program:`s3ql_verify` will attempt to retrieve just the metadata for +every object (for e.g. the S3-compatible or Google Storage backends +this corresponds to a ``HEAD`` request for each object), which is +generally sufficient to determine if the object still exists. When +specifying the :cmdopt:`--data` option, :program:`s3ql_verify` will +instead read every object entirely. To determine how much data will be +transmitted in total when using :cmdopt:`--data`, look at the *After +compression* row in the :ref:`s3qlstat ` output. + +:program:`s3ql_verify` is not able to correct any data corruption that +it finds. Instead, a list of the corrupted and/or missing objects is +written to a file and the decision about the proper course of action +is left to the user. If you have administrative access to the backend +server, you may want to investigate the cause of the corruption or +check if the missing/corrupted objects can be restored from +backups. If you believe that the missing/corrupted objects are indeed +lost irrevocably, you can use the :ref:`remove_objects` script (from +the :file:`contrib` directory of the S3QL distribution) to explicitly +delete the objects from the storage backend. After that, you should +run :program:`fsck.s3ql`. Since the (now explicitly deleted) objects +should now no longer be included in the object index reported by the +backend, :program:`fsck.s3ql` will identify the objects as missing, +update the internal file system structures accordingly, and move the +affected files into the :file:`lost+found` directory. + +The :program:`s3ql_verify` command has the following syntax:: + + s3ql_verify [options] + +This command accepts the following options: + +.. pipeinclude:: python ../bin/s3ql_verify --help + :start-after: show this help message and exit + diff --git a/rst/impl_details.rst b/rst/impl_details.rst new file mode 100644 index 0000000..8fb0160 --- /dev/null +++ b/rst/impl_details.rst @@ -0,0 +1,138 @@ +.. -*- mode: rst -*- + +.. _impl_details: + +======================== + Implementation Details +======================== + +This section provides some background information on how S3QL works +internally. Reading this section is not necessary to use S3QL. + +Metadata Storage +================ + +Like most unix filesystems, S3QL has a concept of inodes. + +The contents of directory inodes (aka the names and inodes of the +files and sub directories contained in a directory) are stored +directly in an SQLite_ database. This database +is stored in a special storage object that is downloaded when the file +system is mounted and uploaded periodically in the background and when +the file system is unmounted. This has two implications: + +#. The entire file system tree can be read from the + database. Fetching/storing storage objects from/in the storage + backend is only required to access the contents of files (or, more + precisely, inodes). This makes most file system operations very + fast because no data has to be send over the network. + +#. An S3QL filesystem can only be mounted on one computer at a time, + using a single :program:`mount.s3ql` process. Otherwise changes made in + one mountpoint will invariably be overwritten when the second mount + point is unmounted. + +Sockets, FIFOs and character devices do not need any additional +storage, all information about them is contained in the database. + +Data Storage +============ + +The contents of file inodes are split into individual blocks. The +maximum size of a block is specified when the file system is created +and cannot be changed afterwards. Every block is stored as an +individual object in the backend, and the mapping from inodes to +blocks and from blocks to objects is stored in the database. + +While the file system is mounted, blocks are cached locally. + +Blocks can also be compressed and encrypted before they are stored in +the storage backend. + +If some files have blocks with identical contents, the blocks will be +stored in the same backend object (i.e., the data is only stored +once). + +Data De-Duplication +=================== + +Instead of uploading every block, S3QL first computes a checksum (a +SHA256 hash) to check if an identical blocks has already been stored +in an backend object. If that is the case, the new block will be +linked to the existing object instead of being uploaded. + +This procedure is invisible for the user and the contents of the block +can still be changed. If several blocks share a backend object and one +of the blocks is changed, the changed block is automatically stored in +a new object (so that the contents of the other block remain +unchanged). + +Caching +======= + +When an application tries to read or write from a file, S3QL +determines the block that contains the required part of the file and +retrieves it from the backend or creates it if it does not yet exist. +The block is then held in the cache directory. It is committed to S3 +when it has not been accessed for more than a few seconds. Blocks are +removed from the cache only when the maximum cache size is reached. + +When the file system is unmounted, all modified blocks are written to +the backend and the cache is cleaned. + +Eventual Consistency Handling +============================= + +S3QL has to take into account that with some storage providers, +changes in objects do not propagate immediately. For example, when an +Amazon S3 object is uploaded and immediately downloaded again, the +downloaded data might not yet reflect the changes done in the upload +(see also +http://developer.amazonwebservices.com/connect/message.jspa?messageID=38538) + +For the data blocks this is not a problem because a data blocks always +get a new object ID when they are updated. + +For the metadata however, S3QL has to make sure that it always +downloads the most recent copy of the database when mounting the file +system. + +To that end, metadata versions are numbered, and the most recent +version number is stored as part of the object id of a very small +"marker" object. When S3QL has downloaded the metadata it checks the +version number against the marker object and, if the two do not agree, +waits for the most recent metadata to become available. Once the +current metadata is available, the version number is increased and the +marker object updated. + + +Encryption +========== + +When the file system is created, :program:`mkfs.s3ql` generates a 256 bit +master key by reading from :file:`/dev/random`. The master key is +encrypted with the passphrase that is entered by the user, and then +stored with the rest of the file system data. Since the passphrase is +only used to access the master key (which is used to encrypt the +actual file system data), the passphrase can easily be changed. + +Data is encrypted with a new session key for each object and each +upload. The session key is generated by appending a nonce to the +master key and then calculating the SHA256 hash. The nonce is +generated by concatenating the object id and the current UTC time as a +32 bit float. The precision of the time is given by the Python `time() +`_ function and +usually at least 1 millisecond. The SHA256 implementation is included +in the Python standard library. + +Once the session key has been calculated, a SHA256 HMAC is calculated +over the data that is to be uploaded. Afterwards, the data is +compressed (unless :cmdopt:`--compress none` was passed to +:program:`mount.s3ql`) and the HMAC inserted at the beginning. Both HMAC +and compressed data are then encrypted using 256 bit AES in CTR +mode using PyCrypto_. Finally, the nonce is +inserted in front of the encrypted data and HMAC, and the packet is +send to the backend as a new S3 object. + +.. _PyCrypto: http://www.pycrypto.org/ +.. _SQLite: http://www.sqlite.org/ diff --git a/rst/include/about.rst b/rst/include/about.rst new file mode 100644 index 0000000..ade3bae --- /dev/null +++ b/rst/include/about.rst @@ -0,0 +1,7 @@ +.. -*- mode: rst -*- + +.. only:: man + + S3QL is a file system for online data storage. Before using S3QL, make + sure to consult the full documentation (rather than just the man pages + which only briefly document the available userspace commands). diff --git a/rst/include/exitcodes.rst b/rst/include/exitcodes.rst new file mode 100644 index 0000000..3875675 --- /dev/null +++ b/rst/include/exitcodes.rst @@ -0,0 +1,11 @@ +.. -*- mode: rst -*- + +:0: + Everything went well. + +:1: + An unexpected error occured. This may indicate a bug in the + program. + +:2: + Invalid command line argument. diff --git a/rst/include/postman.rst b/rst/include/postman.rst new file mode 100644 index 0000000..20f3b71 --- /dev/null +++ b/rst/include/postman.rst @@ -0,0 +1,10 @@ +.. -*- mode: rst -*- + +See Also +======== + +The S3QL homepage is at https://bitbucket.org/nikratio/s3ql/. + +The full S3QL documentation should also be installed somewhere on your +system, common locations are :file:`/usr/share/doc/s3ql` or +:file:`/usr/local/doc/s3ql`. diff --git a/rst/index.rst b/rst/index.rst new file mode 100644 index 0000000..51981a7 --- /dev/null +++ b/rst/index.rst @@ -0,0 +1,26 @@ +.. -*- mode: rst -*- + +=================== + S3QL User's Guide +=================== + +.. toctree:: + :maxdepth: 2 + + about + installation + backends + durability + mkfs + adm + mount + special + umount + fsck + authinfo + contrib + tips + issues + man/index + resources + impl_details diff --git a/rst/installation.rst b/rst/installation.rst new file mode 100644 index 0000000..e486c05 --- /dev/null +++ b/rst/installation.rst @@ -0,0 +1,172 @@ +.. -*- mode: rst -*- + + +============== + Installation +============== + +S3QL depends on several other programs and libraries that have to be +installed first. The best method to satisfy these dependencies depends +on your distribution. + + +The following instructions are for S3QL |version| and should be +applicable to any system. The `S3QL Wiki +`_ contains `additional +help `_ help +for specific distributions and operating systems. Note, however, that +S3QL wiki is editable by anyone. The information there has thus not +been vetted by the S3QL maintainers, and may be wrong, out-of-date, or +even dangerous. Generally, you should only follow steps from the Wiki +that you fully understand yourself, and fall back on the instructions +below when in doubt. + + +Dependencies +============ + +The following is a list of the programs and libraries required for +running S3QL. Generally, you should first check if your distribution +already provides a suitable packages and only install from source if +that is not the case. + +* Kernel: Linux 2.6.9 or newer or FreeBSD with `FUSE4BSD + `_. Starting with + kernel 2.6.26 you will get significantly better write performance, + so under Linux you should actually use *2.6.26 or newer whenever + possible*. + +* The `psmisc `_ utilities. + +* `SQLite `_ version 3.7.0 or newer. SQLite + has to be installed as a *shared library* with development headers. + +* `Python `_ 3.3.0 or newer. Make sure to also + install the development headers. + +* The following Python modules: + + * `setuptools `_, version 1.0 or newer. + * `pycrypto `_ + * `defusedxml `_ + * `requests `_ (optional, + required for OAuth2 authentication with Google Storage) + * `systemd `_ (optional, + for enabling systemd support). + * `apsw `_, version 3.7.0 or + newer. + * `llfuse `_, any + version between 1.0 (inclusive) and 2.0 (exclusive) + * `dugong `_, any + version between 3.4 (inclusive) and 4.0 (exclusive) + * `pytest `_, version 2.3.3 or newer (optional, to run unit tests) + * `pytest-catchlog `_ + (optional, to run unit tests) + + To check if a specific module :var:`` is installed, execute + :samp:`python3 -c 'import {}; + print({}.__version__)'`. This will result in an + `ImportError` if the module is not installed, and will print the + installed version if the module is installed. + + +.. _inst-s3ql: + +Installing S3QL +=============== + +To build and install S3QL itself, proceed as follows: + +1. Download S3QL from https://bitbucket.org/nikratio/s3ql/downloads +2. Unpack it into a folder of your choice +3. Run `python3 setup.py build_ext --inplace` to build S3QL. +4. Run `python3 -m pytest tests/` to run a self-test. If this fails, ask + for help on the `mailing list + `_ or report a bug in the + `issue tracker `_. + +Now you have three options: + +* You can run the S3QL commands from the `bin/` directory. + +* You can install S3QL system-wide for all users. To do that, you + have to run `sudo python3 setup.py install`. + +* You can install S3QL into `~/.local` by executing `python3 + setup.py install --user`. In this case you should make sure that + `~/.local/bin` is in your `$PATH` environment variable. + + +Development Version +=================== + +If you have checked out the unstable development version from the +Mercurial repository, a bit more effort is required. You'll also need: + +* Version 0.24 or newer of the Cython_ compiler. + +* Version 1.2b1 or newer of the Sphinx_ document processor. + +With these additional dependencies installed, S3QL can be build and +tested with :: + + python3 setup.py build_cython + python3 setup.py build_ext --inplace + python3 -m pytest tests/ + +Note that when building from the Mercurial or Git repository, building +and testing is done with several additional checks. This may cause +compilation and/or tests to fail even though there are no problems +with functionality. For example, any use of functions that are +scheduled for deprecation in future Python version will cause tests to +fail. If you would rather just check for functionality, you can delete +the :file:`MANIFEST.in` file. In that case, the build system will +behave as it does for a regular release. + +The HTML and PDF documentation can be generated with :: + + python3 setup.py build_sphinx + +and S3QL can be installed as usual with :: + + python3 setup.py install [--user] + + +Running tests requiring remote servers +====================================== + +By default, tests requiring a connection to a remote storage backend +are skipped. If you would like to run these tests too (which is always +a good idea), you have to create additional entries in your +`~/.s3ql/authinfo2` file that tell S3QL what server and credentials to +use for these tests. These entries have the following form:: + + [-test] + backend-login: + backend-password: + test-fs: + +Here ** specifies the backend that you want to test +(e.g. *s3*, *s3c*, *gs*, or *swift*), ** and ** are +the backend authentication credentials, and ** specifies +the full storage URL that will be used for testing. **Any existing +S3QL file system in this storage URL will be destroyed during +testing**. + +For example, to run tests that need connection to a Google Storage +server, you would add something like :: + + [gs-test] + backend-login: GOOGIGWLONT238MD7HZ4 + backend-password: rmEbstjscoeunt1249oes1298gauidbs3hl + test-fs: gs://joes-gs-bucket/s3ql_tests/ + +On the next run of `runtest.py` (or `py.test` when using the +development version), the additional tests will be run. If the tests +are still skipped, you can get more information about why tests are +being skipped by passing the :cmdopt:`-rs` argument to +`runtest.py`/`py.test`. + + +.. _Cython: http://www.cython.org/ +.. _Sphinx: http://sphinx.pocoo.org/ diff --git a/rst/issues.rst b/rst/issues.rst new file mode 100644 index 0000000..56bab84 --- /dev/null +++ b/rst/issues.rst @@ -0,0 +1,100 @@ +.. -*- mode: rst -*- + +============ +Known Issues +============ + +* S3QL de-duplicates data blocks based solely only on SHA256 + checksums, without doing a byte-by-byte comparison of the blocks. + Since it is possible for two data blocks to have the same checksum + despite having different contents, this can lead to problems. If two + such blocks are stored in an S3QL file system, the data in one block + will be lost and replaced by the data in the other block. However, + the chances of this occuring for any two blocks are about 1 in 10^77 + (2^256). For a file system that holds a total of 10^34 blocks, the + chances of a collision increase to about 1 in 10^9. Storing more + than 10^34 blocks (or about 10^25 TB with an (extremely small) block + size of 4 kB) is therefore not recommended. Being exceptionally + unlucky may also be a disadvantage. + +* S3QL does not support Access Control Lists (ACLs). This is due to a + bug in the FUSE library and will therefore hopefully be fixed at + some point. See `issue #16 + `_ + for more details. + +* As of Linux kernel 3.5 S3QL file systems do not implement the "write + protect" bit on directories. In other words, even if a directory has + the write protect bit set, the owner of the directory can delete any + files and (empty) subdirectories inside it. This is a bug in the + FUSE kernel module + (cf. https://github.com/libfuse/libfuse/issues/23) and needs to be + fixed in the kernel. Unfortunately it does not look as if this is + going to be fixed anytime soon (as of 2016/2/28). + +* S3QL is rather slow when an application tries to write data in + unreasonably small chunks. If a 1 MiB file is copied in chunks of 1 + KB, this will take more than 10 times as long as when it's copied + with the (recommended) chunk size of 128 KiB. + + This is a limitation of the FUSE library (which does not yet support + write caching) which will hopefully be addressed in some future FUSE + version. + + Most applications, including e.g. GNU `cp` and `rsync`, use + reasonably large buffers and are therefore not affected by this + problem and perform very efficient on S3QL file systems. + + However, if you encounter unexpectedly slow performance with a + specific program, this might be due to the program using very small + write buffers. Although this is not really a bug in the program, + it might be worth to ask the program's authors for help. + +* S3QL always updates file and directory access times as if the ``relatime`` + mount option has been specified: the access time ("atime") is only updated + if it is currently earlier than either the status change time + ("ctime") or modification time ("mtime"). + +* S3QL directories always have an `st_nlink` value of 1. This may confuse + programs that rely on directories having `st_nlink` values of *(2 + + number of sub directories)*. + + Note that this is not a bug in S3QL. Including sub directories in + the `st_nlink` value is a Unix convention, but by no means a + requirement. If an application blindly relies on this convention + being followed, then this is a bug in the application. + + A prominent example are early versions of GNU find, which required + the `--noleaf` option to work correctly on S3QL file systems. This + bug has already been fixed in recent find versions. + +* The `umount` and `fusermount -u` commands will *not* block until all + data has been uploaded to the backend. (this is a FUSE limitation + that will hopefully be removed in the future, see `issue #1 + `_). If + you use either command to unmount an S3QL file system, you have to + take care to explicitly wait for the `mount.s3ql` process to + terminate before you shut down or restart the system. Therefore it + is generally not a good idea to mount an S3QL file system in + `/etc/fstab` (you should use a dedicated init script instead). + +* S3QL relies on the backends not to run out of space. This is a given + for big storage providers like Amazon S3 or Google Storage, but you + may stumble upon this if you use your own server or smaller providers. + + If there is no space left in the backend, attempts to write more + data into the S3QL file system will fail and the file system will be + in an inconsistent state and require a file system check (and you + should make sure to make space available in the backend before + running the check). + + Unfortunately, there is no way to handle insufficient space in the + backend without leaving the file system inconsistent. Since + S3QL first writes data into the cache, it can no longer return an + error when it later turns out that the cache can not be committed to + the backend. + +* When using python-dugong versions 3.3 or earlier, S3QL supports only + CONNECT-style proxying, which may cause issues with some proxy + servers when using plain HTTP. Upgrading to python-dugong 3.4 or + newer removes this limitation. diff --git a/rst/man/adm.rst b/rst/man/adm.rst new file mode 100644 index 0000000..4c3af3a --- /dev/null +++ b/rst/man/adm.rst @@ -0,0 +1,102 @@ +.. -*- mode: rst -*- + +===================== +The |command| command +===================== + + +Synopsis +======== + +:: + + s3qladm [options] + +where :var:`action` may be either of :program:`passphrase`, +:program:`upgrade`, :program:`delete` or :program:`download-metadata`. + +Description +=========== + +.. include:: ../include/about.rst + +The |command| command performs various operations on *unmounted* S3QL +file systems. The file system *must not be mounted* when using +|command| or things will go wrong badly. + +The storage url depends on the backend that is used. The S3QL User's +Guide should be consulted for a description of the available backends. + +Options +======= + +The |command| command accepts the following options. + +.. pipeinclude:: python ../../bin/s3qladm --help + :start-after: show this help message and exit + +Actions +======= + +The following actions may be specified: + +passphrase + Changes the encryption passphrase of the file system. + +upgrade + Upgrade the file system to the newest revision. + +delete + Delete the file system with all the stored data. + +download-metadata + Interactively download backups of the file system metadata. + + +Exit Codes +========== + +|command| may terminate with the following exit codes: + +.. include:: ../include/exitcodes.rst + +:3: + Invalid backend option. + +:10: + Could not open log file for writing. + +:11: + No such backend. + +:12: + Authentication file has insecure permissions. + +:13: + Unable to parse proxy settings. + +:14: + Invalid credentials (Authentication failed). + +:15: + No permission to access backend (Authorization denied). + +:16: + Invalid storage URL, specified location does not exist in backend. + +:17: + Wrong file system passphrase. + +:18: + No S3QL file system found at given storage URL. + +:19: + Unable to connect to backend, can't resolve hostname. + +:45: + Unable to access cache directory. + + +.. include:: ../include/postman.rst + +.. |command| replace:: :program:`s3qladm` diff --git a/rst/man/cp.rst b/rst/man/cp.rst new file mode 100644 index 0000000..4c49519 --- /dev/null +++ b/rst/man/cp.rst @@ -0,0 +1,100 @@ +.. -*- mode: rst -*- + +===================== +The |command| command +===================== + + +Synopsis +======== + +:: + + s3qlcp [options] + +Description +=========== + +.. include:: ../include/about.rst + +The |command| command duplicates the directory tree :var:`source-dir` +into :var:`dest-dir` without physically copying the file contents. +Both source and destination must lie inside the same S3QL file system. + +.. begin_main_content + +The replication will not take any additional space. Only if one of +directories is modified later on, the modified data will take +additional storage space. + +`s3qlcp` can only be called by the user that mounted the file system +and (if the file system was mounted with `--allow-other` or `--allow-root`) +the root user. + +Note that: + +* After the replication, both source and target directory will still + be completely ordinary directories. You can regard `` as a + snapshot of `` or vice versa. However, the most common + usage of `s3qlcp` is to regularly duplicate the same source + directory, say `documents`, to different target directories. For a + e.g. monthly replication, the target directories would typically be + named something like `documents_January` for the replication in + January, `documents_February` for the replication in February etc. + In this case it is clear that the target directories should be + regarded as snapshots of the source directory. + +* Exactly the same effect could be achieved by an ordinary copy + program like `cp -a`. However, this procedure would be orders of + magnitude slower, because `cp` would have to read every file + completely (so that S3QL had to fetch all the data over the network + from the backend) before writing them into the destination folder. + + +Snapshotting vs Hardlinking +--------------------------- + +Snapshot support in S3QL is inspired by the hardlinking feature that +is offered by programs like `rsync `_ or +`storeBackup `_. +These programs can create a hardlink instead of copying a file if an +identical file already exists in the backup. However, using hardlinks +has two large disadvantages: + +* backups and restores always have to be made with a special program + that takes care of the hardlinking. The backup must not be touched + by any other programs (they may make changes that inadvertently + affect other hardlinked files) + +* special care needs to be taken to handle files which are already + hardlinked (the restore program needs to know that the hardlink was + not just introduced by the backup program to safe space) + +S3QL snapshots do not have these problems, and they can be used with +any backup program. + + +.. end_main_content + + +Options +======= + +The |command| command accepts the following options: + +.. pipeinclude:: python ../../bin/s3qlcp --help + :start-after: show this help message and exit + + +Exit Codes +========== + +|command| may terminate with the following exit codes: + +.. include:: ../include/exitcodes.rst + + +.. include:: ../include/postman.rst + + +.. |command| replace:: :program:`s3qlcp` diff --git a/rst/man/ctrl.rst b/rst/man/ctrl.rst new file mode 100644 index 0000000..9bf59dc --- /dev/null +++ b/rst/man/ctrl.rst @@ -0,0 +1,80 @@ +.. -*- mode: rst -*- + +===================== +The |command| command +===================== + + +Synopsis +======== + +:: + + s3qlctrl [options] ... + +where :var:`action` may be either of :program:`flushcache`, +:program:`upload-meta`, :program:`cachesize` or +:program:`log-metadata`. + +Description +=========== + +.. include:: ../include/about.rst + +The |command| command performs various actions on the S3QL file system mounted +in :var:`mountpoint`. + +|command| can only be called by the user that mounted the file system +and (if the file system was mounted with :cmdopt:`--allow-other` or +:cmdopt:`--allow-root`) the root user. + +The following actions may be specified: + +flushcache + Uploads all changed file data to the backend. + +upload-meta + Upload metadata to the backend. All file system operations will + block while a snapshot of the metadata is prepared for upload. + +cachesize + Changes the cache size of the file system. This action requires an + additional argument that specifies the new cache size in KiB, so the + complete command line is:: + + s3qlctrl [options] cachesize + +log + Change the amount of information that is logged into + :file:`~/.s3ql/mount.log` file. The complete syntax is:: + + s3qlctrl [options] log [ [ ...]] + + here :var:`level` is the desired new log level and may be either of + *debug*, *info* or *warn*. One or more :var:`module` may only be + specified with the *debug* level and allow to restrict the debug + output to just the listed modules. + + +Options +======= + +The |command| command also accepts the following options, no matter +what specific action is being invoked: + +.. pipeinclude:: python ../../bin/s3qlctrl --help + :start-after: show this help message and exit + + +Exit Codes +========== + +|command| may terminate with the following exit codes: + +.. include:: ../include/exitcodes.rst + + +.. include:: ../include/postman.rst + + +.. |command| replace:: :program:`s3qlctrl` diff --git a/rst/man/expire_backups.rst b/rst/man/expire_backups.rst new file mode 100644 index 0000000..a330dd2 --- /dev/null +++ b/rst/man/expire_backups.rst @@ -0,0 +1,107 @@ +.. -*- mode: rst -*- + +===================== +The |command| command +===================== + + +Synopsis +======== + +:: + + expire_backups [options] [ ...] + + +Description +=========== + +The |command| command intelligently remove old backups that are no +longer needed. + +.. begin_main_content + +To define what backups you want to keep for how long, you define a +number of *age ranges*. :program:`expire_backups` ensures that you +will have at least one backup in each age range at all times. It will +keep exactly as many backups as are required for that and delete any +backups that become redundant. + +Age ranges are specified by giving a list of range boundaries in terms +of backup cycles. Every time you create a new backup, the existing +backups age by one cycle. + +Example: when :program:`expire_backups` is called with the age range +definition ``1 3 7 14 31``, it will guarantee that you always have the +following backups available: + +#. A backup that is 0 to 1 cycles old (i.e, the most recent backup) +#. A backup that is 1 to 3 cycles old +#. A backup that is 3 to 7 cycles old +#. A backup that is 7 to 14 cycles old +#. A backup that is 14 to 31 cycles old + +.. NOTE:: + + If you do backups in fixed intervals, then one cycle will be + equivalent to the backup interval. The advantage of specifying the + age ranges in terms of backup cycles rather than days or weeks is + that it allows you to gracefully handle irregular backup intervals. + Imagine that for some reason you do not turn on your computer for + one month. Now all your backups are at least a month old, and if you + had specified the above backup strategy in terms of absolute ages, + they would all be deleted! Specifying age ranges in terms of backup + cycles avoids these sort of problems. + +:program:`expire_backups` usage is simple. It requires backups to be +stored in directories of the form ``year-month-day_hour:minute:seconds`` +(``YYYY-MM-DD_HH:mm:ss``) and works on all backups in the current +directory. So for the above backup strategy, the correct invocation +would be:: + + expire_backups.py 1 3 7 14 31 + +When storing your backups on an S3QL file system, you probably want to +specify the ``--use-s3qlrm`` option as well. This tells +:program:`expire_backups` to use the :ref:`s3qlrm ` command to +delete directories. + +:program:`expire_backups` uses a "state file" to keep track which +backups are how many cycles old (since this cannot be inferred from +the dates contained in the directory names). The standard name for +this state file is :file:`.expire_backups.dat`. If this file gets +damaged or deleted, :program:`expire_backups` no longer knows the ages +of the backups and refuses to work. In this case you can use the +:cmdopt:`--reconstruct-state` option to try to reconstruct the state +from the backup dates. However, the accuracy of this reconstruction +depends strongly on how rigorous you have been with making backups (it +is only completely correct if the time between subsequent backups has +always been exactly the same), so it's generally a good idea not to +tamper with the state file. + +.. end_main_content + + +Options +======= + +The |command| command accepts the following options: + +.. pipeinclude:: python ../../contrib/expire_backups.py --help + :start-after: show this help message and exit + + +Exit Codes +========== + +|command| may terminate with the following exit codes: + +.. include:: ../include/exitcodes.rst + + +See Also +======== + +|command| is shipped as part of S3QL, https://bitbucket.org/nikratio/s3ql/. + +.. |command| replace:: :program:`expire_backups` diff --git a/rst/man/fsck.rst b/rst/man/fsck.rst new file mode 100644 index 0000000..39e4800 --- /dev/null +++ b/rst/man/fsck.rst @@ -0,0 +1,108 @@ +.. -*- mode: rst -*- + +===================== +The |command| command +===================== + + +Synopsis +======== + +:: + + fsck.s3ql [options] + +Description +=========== + +.. include:: ../include/about.rst + +The |command| command checks the file system in the location specified +by *storage url* for errors and attempts to repair any problems. The +storage url depends on the backend that is used. The S3QL User's Guide +should be consulted for a description of the available backends. + +Options +======= + +The |command| command accepts the following options. + +.. pipeinclude:: python ../../bin/fsck.s3ql --help --log none + :start-after: show this help message and exit + + +Exit Codes +========== + +If |command| found any file system errors (no matter if they were +corrected or not), the exit code will be 128 plus one of the codes +listed below. If no errors were found, the following exit codes are +used as-is: + +.. include:: ../include/exitcodes.rst + +:3: + Invalid backend option. + +:10: + Could not open log file for writing. + +:11: + No such backend. + +:12: + Authentication file has insecure permissions. + +:13: + Unable to parse proxy settings. + +:14: + Invalid credentials (Authentication failed). + +:15: + No permission to access backend (Authorization denied). + +:16: + Invalid storage URL, specified location does not exist in backend. + +:17: + Wrong file system passphrase. + +:18: + No S3QL file system found at given storage URL. + +:19: + Unable to connect to backend, can't resolve hostname. + +:32: + Unsupported file system revision (too old). + +:33: + Unsupported file system revision (too new). + +:40: + Cannot check mounted file system. + +:41: + User input required, but running in batch mode. + +:42: + File system check aborted by user. + +:43: + Local metadata is corrupted. + +:44: + Uncorrectable errors found. + +:45: + Unable to access cache directory. + +:128: + This error code will be *added* to one of the codes above if any + file system errors have been found (no matter if they were + corrected or not). + +.. include:: ../include/postman.rst + +.. |command| replace:: :program:`fsck.s3ql` diff --git a/rst/man/index.rst b/rst/man/index.rst new file mode 100644 index 0000000..9653b6a --- /dev/null +++ b/rst/man/index.rst @@ -0,0 +1,25 @@ + +Manpages +======== + +The man pages are installed with S3QL on your system and can be viewed +with the :command:`man` command. For reference, they are also included +here in the User's Guide. + +.. toctree:: + :maxdepth: 1 + + mkfs + adm + mount + stat + ctrl + cp + rm + lock + umount + fsck + oauth_client + verify + pcp + expire_backups diff --git a/rst/man/lock.rst b/rst/man/lock.rst new file mode 100644 index 0000000..225fd06 --- /dev/null +++ b/rst/man/lock.rst @@ -0,0 +1,89 @@ +.. -*- mode: rst -*- + +===================== +The |command| command +===================== + + +Synopsis +======== + +:: + + s3qllock [options] + +Description +=========== + +.. include:: ../include/about.rst + +The :program:`s3qllock` command makes a directory tree in an S3QL file +system immutable. Immutable trees can no longer be changed in any way +whatsoever. You can not add new files or directories and you can not +change or delete existing files and directories. The only way to get +rid of an immutable tree is to use the :program:`s3qlrm` command. + +|command| can only be called by the user that mounted the file system +and (if the file system was mounted with :cmdopt:`--allow-other` or +:cmdopt:`--allow-root`) the root user. + +Rationale +========= + +.. begin_main_content + +Immutability is a feature designed for backups. Traditionally, backups +have been made on external tape drives. Once a backup was made, the +tape drive was removed and locked somewhere in a shelf. This has the +great advantage that the contents of the backup are now permanently +fixed. Nothing (short of physical destruction) can change or delete +files in the backup. + +In contrast, when backing up into an online storage system like S3QL, +all backups are available every time the file system is mounted. +Nothing prevents a file in an old backup from being changed again +later on. In the worst case, this may make your entire backup system +worthless. Imagine that your system gets infected by a nasty virus +that simply deletes all files it can find -- if the virus is active +while the backup file system is mounted, the virus will destroy all +your old backups as well! + +Even if the possibility of a malicious virus or trojan horse is +excluded, being able to change a backup after it has been made is +generally not a good idea. A common S3QL use case is to keep the file +system mounted at all times and periodically create backups with +:program:`rsync -a`. This allows every user to recover her files from a +backup without having to call the system administrator. However, this +also allows every user to accidentally change or delete files *in* one +of the old backups. + +Making a backup immutable protects you against all these problems. +Unless you happen to run into a virus that was specifically programmed +to attack S3QL file systems, backups can be neither deleted nor +changed after they have been made immutable. + + +.. end_main_content + + +Options +======= + +The |command| command accepts the following options: + +.. pipeinclude:: python ../../bin/s3qllock --help + :start-after: show this help message and exit + + +Exit Codes +========== + +|command| may terminate with the following exit codes: + +.. include:: ../include/exitcodes.rst + + +.. include:: ../include/postman.rst + + +.. |command| replace:: :program:`s3qllock` diff --git a/rst/man/mkfs.rst b/rst/man/mkfs.rst new file mode 100644 index 0000000..4621094 --- /dev/null +++ b/rst/man/mkfs.rst @@ -0,0 +1,77 @@ +.. -*- mode: rst -*- + +===================== +The |command| command +===================== + + +Synopsis +======== + +:: + + mkfs.s3ql [options] + +Description +=========== + +.. include:: ../include/about.rst + +The |command| command creates a new file system in the location +specified by *storage url*. The storage url depends on the backend +that is used. The S3QL User's Guide should be consulted for a +description of the available backends. + +Unless you have specified the `--plain` option, `mkfs.s3ql` will ask +you to enter an encryption password. This password will *not* be read +from an authentication file specified with the :cmdopt:`--authfile` +option to prevent accidental creation of an encrypted file system. + + +Options +======= + +The |command| command accepts the following options. + +.. pipeinclude:: python ../../bin/mkfs.s3ql --help + :start-after: show this help message and exit + + +Exit Codes +========== + +|command| may terminate with the following exit codes: + +.. include:: ../include/exitcodes.rst + +:3: + Invalid backend option. + +:11: + No such backend. + +:12: + Authentication file has insecure permissions. + +:13: + Unable to parse proxy settings. + +:14: + Invalid credentials (Authentication failed). + +:15: + No permission to access backend (Authorization denied). + +:16: + Invalid storage URL, specified location does not exist in backend. + +:19: + Unable to connect to backend, can't resolve hostname. + +:45: + Unable to access cache directory. + + +.. include:: ../include/postman.rst + +.. |command| replace:: :program:`mkfs.s3ql` diff --git a/rst/man/mount.rst b/rst/man/mount.rst new file mode 100644 index 0000000..67a7cb0 --- /dev/null +++ b/rst/man/mount.rst @@ -0,0 +1,109 @@ +.. -*- mode: rst -*- + +===================== +The |command| command +===================== + + +Synopsis +======== + +:: + + mount.s3ql [options] + + +Description +=========== + +.. include:: ../include/about.rst + +The |command| command mounts the S3QL file system stored in *storage +url* in the directory *mount point*. The storage url depends on the +backend that is used. The S3QL User's Guide should be consulted for a +description of the available backends. + + +Options +======= + +The |command| command accepts the following options. + +.. pipeinclude:: python ../../bin/mount.s3ql --help --log none + :start-after: show this help message and exit + + +Exit Codes +========== + +|command| may terminate with the following exit codes: + +.. include:: ../include/exitcodes.rst + +:3: + Invalid backend option. + +:10: + Could not open log file for writing. + +:11: + No such backend. + +:12: + Authentication file has insecure permissions. + +:13: + Unable to parse proxy settings. + +:14: + Invalid credentials (Authentication failed). + +:15: + No permission to access backend (Authorization denied). + +:16: + Invalid storage URL, specified location does not exist in backend. + +:17: + Wrong file system passphrase. + +:18: + No S3QL file system found at given storage URL. + +:19: + Unable to connect to backend, can't resolve hostname. + +:30: + File system was not unmounted cleanly. + +:31: + File system appears to be mounted elsewhere. + +:32: + Unsupported file system revision (too old). + +:33: + Unsupported file system revision (too new). + +:34: + Insufficient free nodes, need to run :program:`fsck.s3ql`. + +:35: + Attempted to mount read-only, this is not supported. + +:36: + Mountpoint does not exist. + +:37: + Not enough available file descriptors. + +:39: + Unable to bind file system to mountpoint. + +:45: + Unable to access cache directory. + + +.. include:: ../include/postman.rst + +.. |command| replace:: :program:`mount.s3ql` diff --git a/rst/man/oauth_client.rst b/rst/man/oauth_client.rst new file mode 100644 index 0000000..30caa40 --- /dev/null +++ b/rst/man/oauth_client.rst @@ -0,0 +1,54 @@ +.. -*- mode: rst -*- + +.. _oauth_client: + +===================== +The |command| command +===================== + + +Synopsis +======== + +:: + + s3ql_oauth_client [options] + + +Description +=========== + +.. include:: ../include/about.rst + +The |command| command may be used to obtain OAuth2 authentication +tokens for use with Google Storage. It requests "user code" from +Google which has to be pasted into the browser to complete the +authentication process interactively. Once authentication in the +browser has been completed, |command| displays the OAuth2 refresh +token. + +When combined with the special username ``oauth2``, the refresh token +can be used as a backend passphrase when using the Google Storage S3QL +backend. + + +Options +======= + +The |command| command accepts the following options: + +.. pipeinclude:: python ../../bin/s3ql_oauth_client --help + :start-after: show this help message and exit + + +Exit Codes +========== + +|command| may terminate with the following exit codes: + +.. include:: ../include/exitcodes.rst + + +.. include:: ../include/postman.rst + +.. |command| replace:: :program:`s3ql_oauth_client` diff --git a/rst/man/pcp.rst b/rst/man/pcp.rst new file mode 100644 index 0000000..173e0a3 --- /dev/null +++ b/rst/man/pcp.rst @@ -0,0 +1,51 @@ +.. -*- mode: rst -*- + +===================== +The |command| command +===================== + + +Synopsis +======== + +:: + + pcp [options] [ ...] + + +Description +=========== + +The |command| command is a is a wrapper that starts several +:program:`sync` processes to copy directory trees in parallel. This is +allows much better copying performance on file system that have +relatively high latency when retrieving individual files like S3QL. + +**Note**: Using this program only improves performance when copying +*from* an S3QL file system. When copying *to* an S3QL file system, +using |command| is more likely to *decrease* performance. + + +Options +======= + +The |command| command accepts the following options: + +.. pipeinclude:: python ../../contrib/pcp.py --help + :start-after: show this help message and exit + + +Exit Codes +========== + +|command| may terminate with the following exit codes: + +.. include:: ../include/exitcodes.rst + + +See Also +======== + +|command| is shipped as part of S3QL, https://bitbucket.org/nikratio/s3ql/. + +.. |command| replace:: :program:`pcp` diff --git a/rst/man/rm.rst b/rst/man/rm.rst new file mode 100644 index 0000000..1ec0873 --- /dev/null +++ b/rst/man/rm.rst @@ -0,0 +1,52 @@ +.. -*- mode: rst -*- + +===================== +The |command| command +===================== + +Synopsis +======== + +:: + + s3qlrm [options] + +Description +=========== + +.. include:: ../include/about.rst + +The |command| command recursively deletes files and directories on an +S3QL file system. Although |command| is faster than using e.g. +:command:`rm -r``, the main reason for its existence is that it allows +you to delete immutable trees (which can be created with +:program:`s3qllock`) as well. + +Be warned that there is no additional confirmation. The directory will +be removed entirely and immediately. + +|command| can only be called by the user that mounted the file system +and (if the file system was mounted with :cmdopt:`--allow-other` or +:cmdopt:`--allow-root`) the root user. + + +Options +======= + +The |command| command accepts the following options: + +.. pipeinclude:: python ../../bin/s3qlrm --help + :start-after: show this help message and exit + + +Exit Codes +========== + +|command| may terminate with the following exit codes: + +.. include:: ../include/exitcodes.rst + + +.. include:: ../include/postman.rst + +.. |command| replace:: :program:`s3qlrm` diff --git a/rst/man/stat.rst b/rst/man/stat.rst new file mode 100644 index 0000000..a796a90 --- /dev/null +++ b/rst/man/stat.rst @@ -0,0 +1,46 @@ +.. -*- mode: rst -*- + +===================== +The |command| command +===================== + +Synopsis +======== + +:: + + s3qlstat [options] + +Description +=========== + +.. include:: ../include/about.rst + +The |command| command prints statistics about the S3QL file system mounted +at :var:`mountpoint`. + +|command| can only be called by the user that mounted the file system +and (if the file system was mounted with :cmdopt:`--allow-other` or +:cmdopt:`--allow-root`) the root user. + + +Options +======= + +The |command| command accepts the following options: + +.. pipeinclude:: python ../../bin/s3qlstat --help + :start-after: show this help message and exit + + +Exit Codes +========== + +|command| may terminate with the following exit codes: + +.. include:: ../include/exitcodes.rst + + +.. include:: ../include/postman.rst + +.. |command| replace:: :program:`s3qlstat` diff --git a/rst/man/umount.rst b/rst/man/umount.rst new file mode 100644 index 0000000..3c5a2c4 --- /dev/null +++ b/rst/man/umount.rst @@ -0,0 +1,53 @@ +.. -*- mode: rst -*- + +===================== +The |command| command +===================== + +Synopsis +======== + +:: + + umount.s3ql [options] + + +Description +=========== + +.. include:: ../include/about.rst + +The |command| command unmounts the S3QL file system mounted in the +directory *mount point* and blocks until all data has been uploaded to +the storage backend. + +Only the user who mounted the file system with :program:`mount.s3ql` +is able to unmount it with |command|. If you are root and want to +unmount an S3QL file system mounted by an ordinary user, you have to +use the :program:`fusermount -u` or :command:`umount` command instead. +Note that these commands do not block until all data has been +uploaded, so if you use them instead of :program:`umount.s3ql` then +you should manually wait for the :program:`mount.s3ql` process to +terminate before shutting down the system. + + +Options +======= + +The |command| command accepts the following options. + +.. pipeinclude:: python ../../bin/umount.s3ql --help + :start-after: show this help message and exit + + +Exit Codes +========== + +|command| may terminate with the following exit codes: + +.. include:: ../include/exitcodes.rst + + +.. include:: ../include/postman.rst + +.. |command| replace:: :program:`umount.s3ql` diff --git a/rst/man/verify.rst b/rst/man/verify.rst new file mode 100644 index 0000000..ea68c86 --- /dev/null +++ b/rst/man/verify.rst @@ -0,0 +1,93 @@ +.. -*- mode: rst -*- + +===================== +The |command| command +===================== + + +Synopsis +======== + +:: + + s3ql_verify [options] + +Description +=========== + +.. include:: ../include/about.rst + +The |command| command verifies all data in the file system. In +contrast to :program:`fsck.s3ql`, |command| does not trust the object +listing returned by the backend, but actually attempts to retrieve +every object. It therefore takes a lot longer. + +The format of :var:`` depends on the backend that is +used. The S3QL User's Guide should be consulted for a description of +the available backends. + +Options +======= + +The |command| command accepts the following options. + +.. pipeinclude:: python ../../bin/s3ql_verify --help --log none + :start-after: show this help message and exit + + +Exit Codes +========== + +|command| may terminate with the following exit codes: + +.. include:: ../include/exitcodes.rst + +:3: + Invalid backend option. + +:10: + Could not open log file for writing. + +:11: + No such backend. + +:12: + Authentication file has insecure permissions. + +:13: + Unable to parse proxy settings. + +:14: + Invalid credentials (Authentication failed). + +:15: + No permission to access backend (Authorization denied). + +:16: + Invalid storage URL, specified location does not exist in backend. + +:17: + Wrong file system passphrase. + +:18: + No S3QL file system found at given storage URL. + +:19: + Unable to connect to backend, can't resolve hostname. + +:32: + Unsupported file system revision (too old). + +:33: + Unsupported file system revision (too new). + +:45: + Unable to access cache directory. + +:46: + The file system data was verified, and some objects were found + to be missing or corrupted. + +.. include:: ../include/postman.rst + +.. |command| replace:: :program:`s3ql_verify` diff --git a/rst/mkfs.rst b/rst/mkfs.rst new file mode 100644 index 0000000..2f078f2 --- /dev/null +++ b/rst/mkfs.rst @@ -0,0 +1,39 @@ +.. -*- mode: rst -*- + +==================== +File System Creation +==================== + +A S3QL file system is created with the :program:`mkfs.s3ql` command. It has the +following syntax:: + + mkfs.s3ql [options] + +This command accepts the following options: + +.. pipeinclude:: python ../bin/mkfs.s3ql --help + :start-after: show this help message and exit + +Unless you have specified the :cmdopt:`--plain` option, +:program:`mkfs.s3ql` will ask you to enter an encryption +password. This password will *not* be read from an authentication file +specified with the :cmdopt:`--authfile` option to prevent accidental +creation of an encrypted file system. + +Note that: + +* All data that is stored under the given storage url is assumed to + managed exclusively by S3QL. Trying to manually save additional + objects (or remove or manipulate existing objects) will lead to file + system corruption, and :program:`fsck.s3ql` may delete objects that + do not belong to the file system. + +* With most storage backends, slashes in the storage url prefix do not + have special meaning. For example, the storage urls + ``s3://mybucket/myprefix/`` and ``s3://mybucket/myprefix`` are + distinct. In the first case, the prefix is ``myprefix/``, while in + the second it is ``myprefix``. + +* S3QL file systems can not be "stacked", i.e. you cannot have one + file system stored at ``s3://bucketname/outerprefix`` and a second + one at ``s3://bucketname/outerprefix/innerprefix``. diff --git a/rst/mount.rst b/rst/mount.rst new file mode 100644 index 0000000..ed275f3 --- /dev/null +++ b/rst/mount.rst @@ -0,0 +1,181 @@ +.. -*- mode: rst -*- + +========== + Mounting +========== + + +A S3QL file system is mounted with the :program:`mount.s3ql` +command. It has the following syntax:: + + mount.s3ql [options] + +.. NOTE:: + + S3QL is not a network file system like `NFS + `_ + or `CIFS `_. It can only be + mounted on one computer at a time. + +This command accepts the following options: + +.. pipeinclude:: python ../bin/mount.s3ql --help + :start-after: show this help message and exit + + +Permission Checking +=================== + +If the file system is mounted with neither the :cmdopt:`allow-root` +nor :cmdopt:`allow-other` option, the mounting user has full +permissions on the S3QL file system (he is effectively root). If one +(or both) of the options is used, standard unix permission checks +apply, i.e. only the real root user has full access and all other +users (including the mounting user) are subject to permission checks. + + +Compression Algorithms +====================== + +S3QL supports three compression algorithms, LZMA, Bzip2 and zlib (with +LZMA being the default). The compression algorithm can be specified +freely whenever the file system is mounted, since it affects only the +compression of new data blocks. + +Roughly speaking, LZMA is slower but achieves better compression +ratios than Bzip2, while Bzip2 in turn is slower but achieves better +compression ratios than zlib. + +For maximum file system performance, the best algorithm therefore +depends on your network connection speed: the compression algorithm +should be fast enough to saturate your network connection. + +To find the optimal algorithm and number of parallel compression +threads for your system, S3QL ships with a program called +`benchmark.py` in the `contrib` directory. You should run this program +on a file that has a size that is roughly equal to the block size of +your file system and has similar contents. It will then determine the +compression speeds for the different algorithms and the upload speeds +for the specified backend and recommend the best algorithm that is +fast enough to saturate your network connection. + +Obviously you should make sure that there is little other system load +when you run `benchmark.py` (i.e., don't compile software or encode +videos at the same time). + + +Notes about Caching +=================== + +S3QL maintains a local cache of the file system data to speed up +access. The cache is block based, so it is possible that only parts of +a file are in the cache. + +Maximum Number of Cache Entries +------------------------------- + +The maximum size of the cache can be configured with the +:cmdopt:`--cachesize` option. In addition to that, the maximum number +of objects in the cache is limited by the +:cmdopt:`--max-cache-entries` option, so it is possible that the cache +does not grow up to the maximum cache size because the maximum number +of cache elements has been reached. The reason for this limit is that +each cache entry requires one open file descriptor, and Linux +distributions usually limit the total number of file descriptors per +process to about a thousand. + +If you specify a value for :cmdopt:`--max-cache-entries`, you should +therefore make sure to also configure your system to increase the +maximum number of open file handles. This can be done temporarily with +the :program:`ulimit -n` command. The method to permanently change this limit +system-wide depends on your distribution. + + + +Cache Flushing and Expiration +----------------------------- + +S3QL flushes changed blocks in the cache to the backend whenever a block +has not been accessed for at least 10 seconds. Note that when a block is +flushed, it still remains in the cache. + +Cache expiration (i.e., removal of blocks from the cache) is only done +when the maximum cache size is reached. S3QL always expires the least +recently used blocks first. + + +Failure Modes +============= + +Once an S3QL file system has been mounted, there is a multitude of +problems that can occur when communicating with the remote +server. Generally, :program:`mount.s3ql` always tries to keep the file +system as accessible as possible under the circumstances. That means +that if network connectivity is lost, data can still be written as +long as there is space in the local cache. Attempts to read data not +already present in the cache, however, will block until connection is +re-established. If any sort of data corruption is detected, the file +system will switch to read-only mode. Attempting to read files that +are affected by the corruption will return an input/output error +(*errno* set to ``EIO``). + +In case of other unexpected or fatal problems, :program:`mount.s3ql` +terminates, but does not unmount the file system. Any attempt to +access the mountpoint will result in a "Transport endpoint not +connected" error (*errno* set to ``ESHUTDOWN``). This ensures that a +mountpoint whose :program:`mount.s3ql` process has terminated can not +be confused with a mountpoint containing an empty file system (which +would be fatal if e.g. the mountpoint is automatically mirrored). When +this has happened, the mountpoint can be cleared by using the +:program:`fusermount` command (provided by FUSE) with the ``-u`` +parameter. + +:program:`mount.s3ql` will automatically try to re-establish the +connection to the server if network connectivity is lost, and retry +sending a request when the connection is established but the remote +server signals a temporary problem. These attempts will be made at +increasing intervals for a period up to 24 hours, with retry intervals +starting at 20 ms and increasing up to 5 minutes. After 24 hours, +:program:`mount.s3ql` will give up and terminate, leaving the +mountpoint inaccessible as described above. + +Generally, :program:`mount.s3ql` will also emit log messages for any +unusual conditions that it encounters. The destination for these +messages can be set with the :cmdopt:`--log` parameter. It is highly +recommended to periodically check these logs, for example with a tool +like logcheck_. Many potential issues that :program:`mount.s3ql` may +encounter do not justify restricting access to the file system, but +should nevertheless be investigated if they occur. Checking the log +messages is the only way to find out about them. + +.. _logcheck: http://sourceforge.net/projects/logcheck/ + + +Automatic Mounting +================== + +If you want to mount and umount an S3QL file system automatically at +system startup and shutdown, you should do so with a dedicated S3QL +init job (instead of using :file:`/etc/fstab`. When using systemd, +:program:`mount.s3ql` can be run as a service of type ``notify``. + +.. NOTE:: + + In principle, it is also possible to automatically mount an S3QL + file system with an appropriate entry in `/etc/fstab`. However, + this is not recommended for several reasons: + + * file systems mounted in :file:`/etc/fstab` will be unmounted with the + :program:`umount` command, so your system will not wait until all data has + been uploaded but shutdown (or restart) immediately (this is a + FUSE limitation, see `issue #1 + `_). + + * There is no way to tell the system that mounting S3QL requires a + Python interpreter to be available, so it may attempt to run + :program:`mount.s3ql` before it has mounted the volume containing + the Python interpreter. + + * There is no standard way to tell the system that internet + connection has to be up before the S3QL file system can be + mounted. diff --git a/rst/resources.rst b/rst/resources.rst new file mode 100644 index 0000000..57a4618 --- /dev/null +++ b/rst/resources.rst @@ -0,0 +1,22 @@ +.. -*- mode: rst -*- + +.. _resources: + +================================ +Further Resources / Getting Help +================================ + +If you have questions or problems with S3QL that you weren't able to +resolve with this manual, you might want to consider the following other resources: + +* The `S3QL Wiki `_ + +* The `S3QL FAQ `_ + +* The `S3QL Mailing List `_. You + can subscribe by sending a mail to + `s3ql+subscribe@googlegroups.com `_. + + +Please report any bugs you may encounter in the `Issue Tracker +`_. diff --git a/rst/special.rst b/rst/special.rst new file mode 100644 index 0000000..05c22e1 --- /dev/null +++ b/rst/special.rst @@ -0,0 +1,122 @@ +.. -*- mode: rst -*- + + +======================== + Advanced S3QL Features +======================== + +.. _s3qlcp: + +Snapshotting and Copy-on-Write +============================== + +The command `s3qlcp` can be used to duplicate a directory tree without +physically copying the file contents. This is made possible by the +data de-duplication feature of S3QL. + +The syntax of `s3qlcp` is:: + + s3qlcp [options] + +This will replicate the contents of the directory `` in the +directory ``. `` has to be an existing directory and +`` must not exist. Moreover, both directories have to be +within the same S3QL file system. + +.. include:: man/cp.rst + :start-after: begin_main_content + :end-before: end_main_content + +.. _s3qlstat: + +Getting Statistics +================== + +You can get more information about a mounted S3QL file system with the +`s3qlstat` command. It has the following syntax:: + + s3qlstat [options] + +This will print out something like this :: + + Directory entries: 1488068 + Inodes: 1482991 + Data blocks: 87948 + Total data size: 400 GiB + After de-duplication: 51 GiB (12.98% of total) + After compression: 43 GiB (10.85% of total, 83.60% of de-duplicated) + Database size: 172 MiB (uncompressed) + (some values do not take into account not-yet-uploaded dirty blocks in cache) + +Probably the most interesting numbers are the total size of your data, +the total size after duplication, and the final size after +de-duplication and compression. + +`s3qlstat` can only be called by the user that mounted the file system +and (if the file system was mounted with `--allow-other` or `--allow-root`) +the root user. + +For a full list of available options, run `s3qlstat --help`. + +.. _s3qllock: + +Immutable Trees +=============== + +The command :program:`s3qllock` can be used to make a directory tree +immutable. Immutable trees can no longer be changed in any way +whatsoever. You can not add new files or directories and you can not +change or delete existing files and directories. The only way to get +rid of an immutable tree is to use the :program:`s3qlrm` command (see +below). + +For example, to make the directory tree beneath the directory +``2010-04-21`` immutable, execute :: + + s3qllock 2010-04-21 + +.. include:: man/lock.rst + :start-after: begin_main_content + :end-before: end_main_content + + +.. _s3qlrm: + +Fast Recursive Removal +====================== + +The ``s3qlrm`` command can be used to recursively delete files and +directories on an S3QL file system. Although ``s3qlrm`` is faster than +using e.g. ``rm -r``, the main reason for its existence is that it +allows you to delete immutable trees as well. The syntax is rather +simple:: + + s3qlrm + +Be warned that there is no additional confirmation. The directory will +be removed entirely and immediately. + +.. _s3qlctrl: + +Runtime Configuration +===================== + + +The `s3qlctrl` can be used to control a mounted S3QL file system. Its +syntax is :: + + s3qlctrl [options] ... + +`` must be the location of a mounted S3QL file system. +For a list of valid options, run `s3qlctrl --help`. `` +may be either of: + + :flushcache: + Flush file system cache. The command blocks until the cache has + been flushed. + :log: + Change log level. + :cachesize: + Change file system cache size. + :upload-meta: + Trigger a metadata upload. diff --git a/rst/tips.rst b/rst/tips.rst new file mode 100644 index 0000000..d95829b --- /dev/null +++ b/rst/tips.rst @@ -0,0 +1,99 @@ +.. -*- mode: rst -*- + +============= +Tips & Tricks +============= + +.. _ssh_tipp: + +SSH Backend +=========== + +By combining S3QL's local backend with `sshfs +`_, it is possible to store an +S3QL file system on arbitrary SSH servers: first mount the remote +target directory into the local filesystem, :: + + sshfs user@my.server.com:/mnt/s3ql /mnt/sshfs + +and then give the mountpoint to S3QL as a local destination:: + + mount.s3ql local:///mnt/sshfs/myfsdata /mnt/s3ql + + +Permanently mounted backup file system +====================================== + +If you use S3QL as a backup file system, it can be useful to mount the +file system permanently (rather than just mounting it for a backup and +unmounting it afterwards). Especially if your file system becomes +large, this saves you long mount- and unmount times if you only want +to restore a single file. + +If you decide to do so, you should make sure to + +* Use :ref:`s3qllock ` to ensure that backups are immutable + after they have been made. + +* Call :ref:`s3qlctrl upload-meta ` right after a every + backup to make sure that the newest metadata is stored safely (if + you do backups often enough, this may also allow you to set the + :cmdopt:`--metadata-upload-interval` option of :program:`mount.s3ql` + to zero). + +.. _copy_performance: + +Improving copy performance +========================== + +.. NOTE:: + + The following applies only when copying data **from** an S3QL file + system, **not** when copying data **to** an S3QL file system. + +If you want to copy a lot of smaller files *from* an S3QL file system +(e.g. for a system restore) you will probably notice that the +performance is rather bad. + +The reason for this is intrinsic to the way S3QL works. Whenever you +read a file, S3QL first has to retrieve this file over the network +from the backend. This takes a minimum amount of time (the network +latency), no matter how big or small the file is. So when you copy +lots of small files, 99% of the time is actually spend waiting for +network data. + +Theoretically, this problem is easy to solve: you just have to copy +several files at the same time. In practice, however, almost all unix +utilities (``cp``, ``rsync``, ``tar`` and friends) insist on copying +data one file at a time. This makes a lot of sense when copying data +on the local hard disk, but in case of S3QL this is really +unfortunate. + +The best workaround that has been found so far is to copy files by +starting several rsync processes at once and use exclusion rules to +make sure that they work on different sets of files. + +For example, the following script will start 3 rsync instances. The +first instance handles all filenames starting with a-f, the second the +filenames from g-l and the third covers the rest. The ``+ */`` rule +ensures that every instance looks into all directories. :: + + #!/bin/bash + + RSYNC_ARGS="-aHv /mnt/s3ql/ /home/restore/" + + rsync -f "+ */" -f "-! [a-f]*" $RSYNC_ARGS & + rsync -f "+ */" -f "-! [g-l]*" $RSYNC_ARGS & + rsync -f "+ */" -f "- [a-l]*" $RSYNC_ARGS & + + wait + +The optimum number of parallel processes depends on your network +connection and the size of the files that you want to transfer. +However, starting about 10 processes seems to be a good compromise +that increases performance dramatically in almost all situations. + +S3QL comes with a script named ``pcp.py`` in the ``contrib`` directory +that can be used to transfer files in parallel without having to write +an explicit script first. See the description of :ref:`pcp` for +details. diff --git a/rst/umount.rst b/rst/umount.rst new file mode 100644 index 0000000..6f078c5 --- /dev/null +++ b/rst/umount.rst @@ -0,0 +1,30 @@ +.. -*- mode: rst -*- + +========== +Unmounting +========== + +To unmount an S3QL file system, use the command:: + + umount.s3ql [options] + +This will block until all data has been written to the backend. + +Only the user who mounted the file system with :program:`mount.s3ql` +is able to unmount it again. If you are root and want to unmount an +S3QL file system mounted by an ordinary user, you have to use the +:command:`fusermount -u` or :command:`umount` command instead. Note +that these commands do not block until all data has been uploaded, so +if you use them instead of `umount.s3ql` then you should manually wait +for the `mount.s3ql` process to terminate before shutting down the +system. + +The :program:`umount.s3ql` command accepts the following options: + +.. pipeinclude:: python ../bin/umount.s3ql --help + :start-after: show this help message and exit + +If, for some reason, the `umount.sql` command does not work, the file +system can also be unmounted with `fusermount -u -z`. Note that this +command will return immediately and the file system may continue to +upload data in the background for a while longer. diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b466358 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,8 @@ +[easy_install] +allow_hosts = None + +[egg_info] +tag_svn_revision = 0 +tag_build = +tag_date = 0 + diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..ae1c00a --- /dev/null +++ b/setup.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +''' +setup.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +# Python version check +import sys +if sys.version_info < (3,3): + raise SystemExit('Python version is %d.%d.%d, but S3QL requires Python 3.3 or newer' + % sys.version_info[:3]) + +try: + import setuptools +except ImportError: + raise SystemExit('Setuptools package not found. Please install from ' + 'https://pypi.python.org/pypi/setuptools') +from setuptools import Extension + +from distutils.version import LooseVersion +import os +import subprocess +import re +from glob import glob +import faulthandler +faulthandler.enable() + +basedir = os.path.abspath(os.path.dirname(sys.argv[0])) +DEVELOPER_MODE = os.path.exists(os.path.join(basedir, 'MANIFEST.in')) +if DEVELOPER_MODE: + print('MANIFEST.in exists, running in developer mode') + +# Add S3QL sources +sys.path.insert(0, os.path.join(basedir, 'src')) +sys.path.insert(0, os.path.join(basedir, 'util')) +import s3ql + +class build_docs(setuptools.Command): + description = 'Build Sphinx documentation' + user_options = [ + ('fresh-env', 'E', 'discard saved environment'), + ('all-files', 'a', 'build all files'), + ] + boolean_options = ['fresh-env', 'all-files'] + + def initialize_options(self): + self.fresh_env = False + self.all_files = False + + def finalize_options(self): + pass + + def run(self): + try: + from sphinx.application import Sphinx + from docutils.utils import SystemMessage + except ImportError: + raise SystemExit('This command requires Sphinx to be installed.') from None + + fix_docutils() + + dest_dir = os.path.join(basedir, 'doc') + src_dir = os.path.join(basedir, 'rst') + + confoverrides = {} + confoverrides['version'] = s3ql.VERSION + confoverrides['release'] = s3ql.RELEASE + + for builder in ('html', 'latex', 'man'): + print('Running %s builder...' % builder) + self.mkpath(os.path.join(dest_dir, builder)) + app = Sphinx(srcdir=src_dir, confdir=src_dir, + outdir=os.path.join(dest_dir, builder), + doctreedir=os.path.join(dest_dir, 'doctrees'), + buildername=builder, confoverrides=confoverrides, + freshenv=self.fresh_env) + self.fresh_env = False + self.all_files = False + + try: + if self.all_files: + app.builder.build_all() + else: + app.builder.build_update() + except SystemMessage as err: + print('reST markup error:', + err.args[0].encode('ascii', 'backslashreplace'), + file=sys.stderr) + + # These shouldn't be installed by default + for name in ('expire_backups.1', 'pcp.1'): + os.rename(os.path.join(dest_dir, 'man', name), + os.path.join(basedir, 'contrib', name)) + + print('Running pdflatex...') + for _ in range(3): + with open('/dev/null', 'wb') as null: + subprocess.check_call(['pdflatex', '-interaction', 'batchmode', 'manual.tex'], + cwd=os.path.join(dest_dir, 'latex'), stdout=null) + os.rename(os.path.join(dest_dir, 'latex', 'manual.pdf'), + os.path.join(dest_dir, 'manual.pdf')) + + +def main(): + + with open(os.path.join(basedir, 'README.rst'), 'r') as fh: + long_desc = fh.read() + + compile_args = ['-Wall', '-Wextra', '-Wconversion', '-Wsign-compare'] + + # Value-changing conversions should always be explicit. + compile_args.append('-Werror=conversion') + + # Note that (i > -1) is false if i is unsigned (-1 will be converted to + # a large positive value). We certainly don't want to do this by + # accident. + compile_args.append('-Werror=sign-compare') + + # Enable all fatal warnings only when compiling from Mercurial tip. + # (otherwise we break forward compatibility because compilation with newer + # compiler may fail if additional warnings are added) + if DEVELOPER_MODE: + compile_args.append('-Werror') + compile_args.append('-Wfatal-errors') + compile_args.append('-Wno-unused-function') + + required_pkgs = ['apsw >= 3.7.0', + 'pycrypto', + 'requests', + 'defusedxml', + 'dugong >= 3.4', + 'llfuse >= 1.0, < 2.0' ] + + setuptools.setup( + name='s3ql', + zip_safe=True, + version=s3ql.VERSION, + description='a full-featured file system for online data storage', + long_description=long_desc, + author='Nikolaus Rath', + author_email='Nikolaus@rath.org', + url='https://bitbucket.org/nikratio/s3ql/', + download_url='https://bitbucket.org/nikratio/s3ql/downloads', + license='GPLv3', + classifiers=['Development Status :: 5 - Production/Stable', + 'Environment :: No Input/Output (Daemon)', + 'Environment :: Console', + 'License :: OSI Approved :: GNU Library or Lesser General Public License (GPLv3)', + 'Topic :: Internet', + 'Operating System :: POSIX', + 'Topic :: System :: Archiving'], + platforms=[ 'POSIX', 'UNIX', 'Linux' ], + keywords=['FUSE', 'backup', 'archival', 'compression', 'encryption', + 'deduplication', 'aws', 's3' ], + package_dir={'': 'src'}, + packages=setuptools.find_packages('src'), + provides=['s3ql'], + ext_modules=[Extension('s3ql.deltadump', ['src/s3ql/deltadump.c'], + extra_compile_args=compile_args, + extra_link_args=[ '-lsqlite3'])], + data_files=[ ('share/man/man1', + [ os.path.join('doc/man/', x) for x + in glob(os.path.join(basedir, 'doc', 'man', '*.1')) ]) ], + entry_points={ 'console_scripts': + [ + 'mkfs.s3ql = s3ql.mkfs:main', + 'fsck.s3ql = s3ql.fsck:main', + 'mount.s3ql = s3ql.mount:main', + 'umount.s3ql = s3ql.umount:main', + 's3qlcp = s3ql.cp:main', + 's3qlstat = s3ql.statfs:main', + 's3qladm = s3ql.adm:main', + 's3qlctrl = s3ql.ctrl:main', + 's3qllock = s3ql.lock:main', + 's3qlrm = s3ql.remove:main', + 's3ql_oauth_client = s3ql.oauth_client:main', + 's3ql_verify = s3ql.verify:main', + ] + }, + install_requires=required_pkgs, + cmdclass={'upload_docs': upload_docs, + 'build_cython': build_cython, + 'build_sphinx': build_docs }, + command_options={ 'sdist': { 'formats': ('setup.py', 'bztar') } }, + ) + +class build_cython(setuptools.Command): + user_options = [] + boolean_options = [] + description = "Compile .pyx to .c" + + def initialize_options(self): + pass + + def finalize_options(self): + # Attribute defined outside init + #pylint: disable=W0201 + self.extensions = self.distribution.ext_modules + + def run(self): + cython = None + for c in ('cython3', 'cython'): + try: + version = subprocess.check_output([c, '--version'], + universal_newlines=True, stderr=subprocess.STDOUT) + cython = c + except FileNotFoundError: + pass + if cython is None: + raise SystemExit('Cython needs to be installed for this command') from None + + hit = re.match('^Cython version (.+)$', version) + if not hit or LooseVersion(hit.group(1)) < "0.17": + raise SystemExit('Need Cython 0.17 or newer, found ' + version) + + cmd = [cython, '-Wextra', '-f', '-3', + '-X', 'embedsignature=True' ] + if DEVELOPER_MODE: + cmd.append('-Werror') + + # Work around http://trac.cython.org/cython_trac/ticket/714 + cmd += ['-X', 'warn.maybe_uninitialized=False' ] + + for extension in self.extensions: + for file_ in extension.sources: + (file_, ext) = os.path.splitext(file_) + path = os.path.join(basedir, file_) + if ext != '.c': + continue + if os.path.exists(path + '.pyx'): + print('compiling %s to %s' % (file_ + '.pyx', file_ + ext)) + if subprocess.call(cmd + [path + '.pyx']) != 0: + raise SystemExit('Cython compilation failed') + +class upload_docs(setuptools.Command): + user_options = [] + boolean_options = [] + description = "Upload documentation" + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + subprocess.check_call(['rsync', '-aHv', '--del', os.path.join(basedir, 'doc', 'html') + '/', + 'ebox.rath.org:/srv/www.rath.org/s3ql-docs/']) + subprocess.check_call(['rsync', '-aHv', '--del', os.path.join(basedir, 'doc', 'manual.pdf'), + 'ebox.rath.org:/srv/www.rath.org/s3ql-docs/']) + +def fix_docutils(): + '''Work around https://bitbucket.org/birkenfeld/sphinx/issue/1154/''' + + import docutils.parsers + from docutils.parsers import rst + old_getclass = docutils.parsers.get_parser_class + + # Check if bug is there + try: + old_getclass('rst') + except AttributeError: + pass + else: + return + + def get_parser_class(parser_name): + """Return the Parser class from the `parser_name` module.""" + if parser_name in ('rst', 'restructuredtext'): + return rst.Parser + else: + return old_getclass(parser_name) + docutils.parsers.get_parser_class = get_parser_class + + assert docutils.parsers.get_parser_class('rst') is rst.Parser + + +if __name__ == '__main__': + main() diff --git a/src/s3ql.egg-info/PKG-INFO b/src/s3ql.egg-info/PKG-INFO new file mode 100644 index 0000000..2c99854 --- /dev/null +++ b/src/s3ql.egg-info/PKG-INFO @@ -0,0 +1,175 @@ +Metadata-Version: 1.1 +Name: s3ql +Version: 2.21 +Summary: a full-featured file system for online data storage +Home-page: https://bitbucket.org/nikratio/s3ql/ +Author: Nikolaus Rath +Author-email: Nikolaus@rath.org +License: GPLv3 +Download-URL: https://bitbucket.org/nikratio/s3ql/downloads +Description: .. + NOTE: We cannot use sophisticated ReST syntax here because this + file is rendered by Bitbucket. + + ============ + About S3QL + ============ + + S3QL is a file system that stores all its data online using storage + services like `Google Storage`_, `Amazon S3`_, or OpenStack_. S3QL + effectively provides a hard disk of dynamic, infinite capacity that + can be accessed from any computer with internet access. + + S3QL is a standard conforming, full featured UNIX file system that is + conceptually indistinguishable from any local file system. + Furthermore, S3QL has additional features like compression, + encryption, data de-duplication, immutable trees and snapshotting + which make it especially suitable for online backup and archival. + + S3QL is designed to favor simplicity and elegance over performance and + feature-creep. Care has been taken to make the source code as + readable and serviceable as possible. Solid error detection and error + handling have been included from the very first line, and S3QL comes + with extensive automated test cases for all its components. + + .. _`Google Storage`: http://code.google.com/apis/storage/ + .. _`Amazon S3`: http://aws.amazon.com/s3 + .. _OpenStack: http://openstack.org/projects/storage/ + + + Features + ======== + + * **Transparency.** Conceptually, S3QL is indistinguishable from a + local file system. For example, it supports hardlinks, symlinks, + standard unix permissions, extended attributes and file + sizes up to 2 TB. + + * **Dynamic Size.** The size of an S3QL file system grows and shrinks + dynamically as required. + + * **Compression.** Before storage, all data may compressed with the + LZMA, bzip2 or deflate (gzip) algorithm. + + * **Encryption.** After compression (but before upload), all data can be + AES encrypted with a 256 bit key. An additional SHA256 HMAC checksum + is used to protect the data against manipulation. + + * **Data De-duplication.** If several files have identical contents, + the redundant data will be stored only once. This works across all + files stored in the file system, and also if only some parts of the + files are identical while other parts differ. + + * **Immutable Trees.** Directory trees can be made immutable, so that + their contents can no longer be changed in any way whatsoever. This + can be used to ensure that backups can not be modified after they + have been made. + + * **Copy-on-Write/Snapshotting.** S3QL can replicate entire directory + trees without using any additional storage space. Only if one of the + copies is modified, the part of the data that has been modified will + take up additional storage space. This can be used to create + intelligent snapshots that preserve the state of a directory at + different points in time using a minimum amount of space. + + * **High Performance independent of network latency.** All operations + that do not write or read file contents (like creating directories + or moving, renaming, and changing permissions of files and + directories) are very fast because they are carried out without any + network transactions. + + S3QL achieves this by saving the entire file and directory structure + in a database. This database is locally cached and the remote + copy updated asynchronously. + + * **Support for low bandwidth connections.** S3QL splits file contents + into smaller blocks and caches blocks locally. This minimizes both + the number of network transactions required for reading and writing + data, and the amount of data that has to be transferred when only + parts of a file are read or written. + + + Development Status + ================== + + S3QL is considered stable and suitable for production use. Starting + with version 2.17.1, S3QL uses semantic versioning. This means that + backwards-incompatible versions (e.g., versions that require an + upgrade of the file system revision) will be reflected in an increase + of the major version number. + + + Supported Platforms + =================== + + S3QL is developed and tested under Linux. Users have also reported + running S3QL successfully on OS-X, FreeBSD and NetBSD. We try to + maintain compatibility with these systems, but (due to lack of + pre-release testers) we cannot guarantee that every release will run + on all non-Linux systems. Please report any bugs you find, and we will + try to fix them. + + + Typical Usage + ============= + + Before a file system can be mounted, the backend which will hold the + data has to be initialized. This is done with the *mkfs.s3ql* + command. Here we are using the Amazon S3 backend, and + *nikratio-s3ql-bucket* is the S3 bucket in which the file system will + be stored. :: + + mkfs.s3ql s3://nikratio-s3ql-bucket + + To mount the S3QL file system stored in the S3 bucket + *nikratio_s3ql_bucket* in the directory ``/mnt/s3ql``, enter:: + + mount.s3ql s3://nikratio-s3ql-bucket /mnt/s3ql + + Now you can instruct your favorite backup program to run a backup into + the directory ``/mnt/s3ql`` and the data will be stored an Amazon + S3. When you are done, the file system has to be unmounted with :: + + umount.s3ql /mnt/s3ql + + + Need Help? + ========== + + The following resources are available: + + * The `S3QL User's Guide`_. + * The `S3QL Wiki`_, which also contains the `S3QL FAQ`_. + * The `S3QL Mailing List`_. You can subscribe by sending a mail to + `s3ql+subscribe@googlegroups.com + `_. + + Please report any bugs you may encounter in the `Bitbucket Issue Tracker`_. + + Contributing + ============ + + The S3QL source code is available both on GitHub_ and BitBucket_. + + + .. _`S3QL User's Guide`: http://www.rath.org/s3ql-docs/index.html + .. _`S3QL Wiki`: https://bitbucket.org/nikratio/s3ql/wiki/ + .. _`Installation Instructions`: https://bitbucket.org/nikratio/s3ql/wiki/Installation + .. _`S3QL FAQ`: https://bitbucket.org/nikratio/s3ql/wiki/FAQ + .. _`S3QL Mailing List`: http://groups.google.com/group/s3ql + .. _`Bitbucket Issue Tracker`: https://bitbucket.org/nikratio/s3ql/issues + .. _BitBucket: https://bitbucket.org/nikratio/s3ql/ + .. _GitHub: https://github.com/s3ql/main + +Keywords: FUSE,backup,archival,compression,encryption,deduplication,aws,s3 +Platform: POSIX +Platform: UNIX +Platform: Linux +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: No Input/Output (Daemon) +Classifier: Environment :: Console +Classifier: License :: OSI Approved :: GNU Library or Lesser General Public License (GPLv3) +Classifier: Topic :: Internet +Classifier: Operating System :: POSIX +Classifier: Topic :: System :: Archiving +Provides: s3ql diff --git a/src/s3ql.egg-info/SOURCES.txt b/src/s3ql.egg-info/SOURCES.txt new file mode 100644 index 0000000..ae91c5b --- /dev/null +++ b/src/s3ql.egg-info/SOURCES.txt @@ -0,0 +1,251 @@ +Changes.txt +LICENSE +README.rst +setup.cfg +setup.py +bin/fsck.s3ql +bin/mkfs.s3ql +bin/mount.s3ql +bin/s3ql_oauth_client +bin/s3ql_verify +bin/s3qladm +bin/s3qlcp +bin/s3qlctrl +bin/s3qllock +bin/s3qlrm +bin/s3qlstat +bin/umount.s3ql +contrib/benchmark.py +contrib/clone_fs.py +contrib/expire_backups.1 +contrib/expire_backups.py +contrib/fsck_db.py +contrib/pcp.1 +contrib/pcp.py +contrib/remove_objects.py +contrib/s3ql_backup.sh +contrib/scramble_db.py +doc/manual.pdf +doc/html/.buildinfo +doc/html/about.html +doc/html/adm.html +doc/html/authinfo.html +doc/html/backends.html +doc/html/contrib.html +doc/html/durability.html +doc/html/fsck.html +doc/html/impl_details.html +doc/html/index.html +doc/html/installation.html +doc/html/issues.html +doc/html/mkfs.html +doc/html/mount.html +doc/html/objects.inv +doc/html/resources.html +doc/html/search.html +doc/html/searchindex.js +doc/html/special.html +doc/html/tips.html +doc/html/umount.html +doc/html/_sources/about.txt +doc/html/_sources/adm.txt +doc/html/_sources/authinfo.txt +doc/html/_sources/backends.txt +doc/html/_sources/contrib.txt +doc/html/_sources/durability.txt +doc/html/_sources/fsck.txt +doc/html/_sources/impl_details.txt +doc/html/_sources/index.txt +doc/html/_sources/installation.txt +doc/html/_sources/issues.txt +doc/html/_sources/mkfs.txt +doc/html/_sources/mount.txt +doc/html/_sources/resources.txt +doc/html/_sources/special.txt +doc/html/_sources/tips.txt +doc/html/_sources/umount.txt +doc/html/_sources/man/adm.txt +doc/html/_sources/man/cp.txt +doc/html/_sources/man/ctrl.txt +doc/html/_sources/man/expire_backups.txt +doc/html/_sources/man/fsck.txt +doc/html/_sources/man/index.txt +doc/html/_sources/man/lock.txt +doc/html/_sources/man/mkfs.txt +doc/html/_sources/man/mount.txt +doc/html/_sources/man/oauth_client.txt +doc/html/_sources/man/pcp.txt +doc/html/_sources/man/rm.txt +doc/html/_sources/man/stat.txt +doc/html/_sources/man/umount.txt +doc/html/_sources/man/verify.txt +doc/html/_static/ajax-loader.gif +doc/html/_static/basic.css +doc/html/_static/comment-bright.png +doc/html/_static/comment-close.png +doc/html/_static/comment.png +doc/html/_static/contents.png +doc/html/_static/doctools.js +doc/html/_static/down-pressed.png +doc/html/_static/down.png +doc/html/_static/file.png +doc/html/_static/jquery.js +doc/html/_static/minus.png +doc/html/_static/navigation.png +doc/html/_static/plus.png +doc/html/_static/pygments.css +doc/html/_static/searchtools.js +doc/html/_static/sphinxdoc.css +doc/html/_static/underscore.js +doc/html/_static/up-pressed.png +doc/html/_static/up.png +doc/html/_static/websupport.js +doc/html/man/adm.html +doc/html/man/cp.html +doc/html/man/ctrl.html +doc/html/man/expire_backups.html +doc/html/man/fsck.html +doc/html/man/index.html +doc/html/man/lock.html +doc/html/man/mkfs.html +doc/html/man/mount.html +doc/html/man/oauth_client.html +doc/html/man/pcp.html +doc/html/man/rm.html +doc/html/man/stat.html +doc/html/man/umount.html +doc/html/man/verify.html +doc/latex/Makefile +doc/latex/fncychap.sty +doc/latex/manual.aux +doc/latex/manual.idx +doc/latex/manual.out +doc/latex/manual.tex +doc/latex/manual.toc +doc/latex/python.ist +doc/latex/sphinx.sty +doc/latex/sphinxhowto.cls +doc/latex/sphinxmanual.cls +doc/latex/tabulary.sty +doc/man/fsck.s3ql.1 +doc/man/mkfs.s3ql.1 +doc/man/mount.s3ql.1 +doc/man/s3ql_oauth_client.1 +doc/man/s3ql_verify.1 +doc/man/s3qladm.1 +doc/man/s3qlcp.1 +doc/man/s3qlctrl.1 +doc/man/s3qllock.1 +doc/man/s3qlrm.1 +doc/man/s3qlstat.1 +doc/man/umount.s3ql.1 +rst/about.rst +rst/adm.rst +rst/authinfo.rst +rst/backends.rst +rst/conf.py +rst/contrib.rst +rst/durability.rst +rst/fsck.rst +rst/impl_details.rst +rst/index.rst +rst/installation.rst +rst/issues.rst +rst/mkfs.rst +rst/mount.rst +rst/resources.rst +rst/special.rst +rst/tips.rst +rst/umount.rst +rst/_static/sphinxdoc.css +rst/_templates/layout.html +rst/include/about.rst +rst/include/exitcodes.rst +rst/include/postman.rst +rst/man/adm.rst +rst/man/cp.rst +rst/man/ctrl.rst +rst/man/expire_backups.rst +rst/man/fsck.rst +rst/man/index.rst +rst/man/lock.rst +rst/man/mkfs.rst +rst/man/mount.rst +rst/man/oauth_client.rst +rst/man/pcp.rst +rst/man/rm.rst +rst/man/stat.rst +rst/man/umount.rst +rst/man/verify.rst +src/s3ql/__init__.py +src/s3ql/adm.py +src/s3ql/block_cache.py +src/s3ql/calc_mro.py +src/s3ql/common.py +src/s3ql/cp.py +src/s3ql/ctrl.py +src/s3ql/daemonize.py +src/s3ql/database.py +src/s3ql/deltadump.c +src/s3ql/deltadump.pyx +src/s3ql/endian_indep.h +src/s3ql/exit_stack.py +src/s3ql/fs.py +src/s3ql/fsck.py +src/s3ql/inherit_docstrings.py +src/s3ql/inode_cache.py +src/s3ql/lock.py +src/s3ql/logging.py +src/s3ql/metadata.py +src/s3ql/mkfs.py +src/s3ql/mount.py +src/s3ql/multi_lock.py +src/s3ql/oauth_client.py +src/s3ql/parse_args.py +src/s3ql/remove.py +src/s3ql/statfs.py +src/s3ql/umount.py +src/s3ql/verify.py +src/s3ql.egg-info/PKG-INFO +src/s3ql.egg-info/SOURCES.txt +src/s3ql.egg-info/dependency_links.txt +src/s3ql.egg-info/entry_points.txt +src/s3ql.egg-info/requires.txt +src/s3ql.egg-info/top_level.txt +src/s3ql.egg-info/zip-safe +src/s3ql/backends/__init__.py +src/s3ql/backends/common.py +src/s3ql/backends/comprenc.py +src/s3ql/backends/gs.py +src/s3ql/backends/local.py +src/s3ql/backends/pool.py +src/s3ql/backends/rackspace.py +src/s3ql/backends/s3.py +src/s3ql/backends/s3c.py +src/s3ql/backends/swift.py +src/s3ql/backends/swiftks.py +tests/common.py +tests/conftest.py +tests/mock_server.py +tests/pytest.ini +tests/pytest_checklogs.py +tests/t1_backends.py +tests/t1_dump.py +tests/t1_retry.py +tests/t1_serialization.py +tests/t2_block_cache.py +tests/t3_fs_api.py +tests/t3_fsck.py +tests/t3_inode_cache.py +tests/t3_verify.py +tests/t4_adm.py +tests/t4_fuse.py +tests/t5_cp.py +tests/t5_ctrl.py +tests/t5_failsafe.py +tests/t5_fsck.py +tests/t5_full.py +tests/t5_lock_rm.py +tests/t6_upgrade.py +util/cmdline_lexer.py +util/sphinx_pipeinclude.py \ No newline at end of file diff --git a/src/s3ql.egg-info/dependency_links.txt b/src/s3ql.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/s3ql.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/src/s3ql.egg-info/entry_points.txt b/src/s3ql.egg-info/entry_points.txt new file mode 100644 index 0000000..a4daa09 --- /dev/null +++ b/src/s3ql.egg-info/entry_points.txt @@ -0,0 +1,14 @@ +[console_scripts] +fsck.s3ql = s3ql.fsck:main +mkfs.s3ql = s3ql.mkfs:main +mount.s3ql = s3ql.mount:main +s3ql_oauth_client = s3ql.oauth_client:main +s3ql_verify = s3ql.verify:main +s3qladm = s3ql.adm:main +s3qlcp = s3ql.cp:main +s3qlctrl = s3ql.ctrl:main +s3qllock = s3ql.lock:main +s3qlrm = s3ql.remove:main +s3qlstat = s3ql.statfs:main +umount.s3ql = s3ql.umount:main + diff --git a/src/s3ql.egg-info/requires.txt b/src/s3ql.egg-info/requires.txt new file mode 100644 index 0000000..d3516d7 --- /dev/null +++ b/src/s3ql.egg-info/requires.txt @@ -0,0 +1,6 @@ +apsw >= 3.7.0 +pycrypto +requests +defusedxml +dugong >= 3.4 +llfuse >= 1.0, < 2.0 diff --git a/src/s3ql.egg-info/top_level.txt b/src/s3ql.egg-info/top_level.txt new file mode 100644 index 0000000..fdb8283 --- /dev/null +++ b/src/s3ql.egg-info/top_level.txt @@ -0,0 +1,2 @@ +dugong +s3ql diff --git a/src/s3ql.egg-info/zip-safe b/src/s3ql.egg-info/zip-safe new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/s3ql.egg-info/zip-safe @@ -0,0 +1 @@ + diff --git a/src/s3ql/__init__.py b/src/s3ql/__init__.py new file mode 100644 index 0000000..826c30f --- /dev/null +++ b/src/s3ql/__init__.py @@ -0,0 +1,51 @@ +''' +__init__.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +# We must not import s3ql.logging.logging as s3ql.logging, +# otherwise future imports of s3ql.logging will incorrectly +# use s3ql.logging.logging. +from . import logging # Ensure use of custom logger class +from llfuse import ROOT_INODE + +# False positives, pylint doesn't grok that these are module names +#pylint: disable=E0603 +__all__ = [ 'adm', 'backends', 'block_cache', 'common', 'calc_mro', + 'cp', 'ctrl', 'daemonize', 'database', 'deltadump', 'fs', + 'fsck', 'inherit_docstrings', 'inode_cache', 'lock', + 'logging', 'metadata', 'mkfs', 'mount', 'parse_args', + 'remove', 'statfs', 'umount', 'VERSION', 'CURRENT_FS_REV', + 'REV_VER_MAP', 'RELEASE', 'BUFSIZE', + 'CTRL_NAME', 'CTRL_INODE' ] + +VERSION = '2.21' +RELEASE = '%s' % VERSION + +# TODO: On next revision bump, consider removing support for TIME +# values from deltadump.pyx. It is only present to allow upgrades +# from revisions <= 22. +CURRENT_FS_REV = 23 + +# Buffer size when writing objects +BUFSIZE = 64 * 1024 + +# Name and inode of the special s3ql control file +CTRL_NAME = '.__s3ql__ctrl__' +CTRL_INODE = 2 + +# Maps file system revisions to the last S3QL version that +# supported this revision. +REV_VER_MAP = { + 21: '2.13', + 20: '2.9', + 16: '1.15', + 15: '1.10', + 14: '1.8.1', + 13: '1.6', + 12: '1.3', + 11: '1.0.1', + } diff --git a/src/s3ql/adm.py b/src/s3ql/adm.py new file mode 100644 index 0000000..d0e6757 --- /dev/null +++ b/src/s3ql/adm.py @@ -0,0 +1,355 @@ +''' +adm.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging, QuietError, setup_logging +from . import CURRENT_FS_REV, REV_VER_MAP +from .backends.comprenc import ComprencBackend +from .database import Connection +from .deltadump import TIME, INTEGER +from .common import (get_backend_cachedir, get_seq_no, is_mounted, get_backend, + load_params, save_params) +from .metadata import dump_and_upload_metadata, download_metadata +from . import metadata +from .parse_args import ArgumentParser +from datetime import datetime as Datetime +from getpass import getpass +from contextlib import contextmanager +import os +import shutil +import functools +import sys +import textwrap +import time + +log = logging.getLogger(__name__) + +def parse_args(args): + '''Parse command line''' + + parser = ArgumentParser( + description="Manage S3QL File Systems.", + epilog=textwrap.dedent('''\ + Hint: run `%(prog)s --help` to get help on the additional + arguments that the different actions take.''')) + + pparser = ArgumentParser(add_help=False, epilog=textwrap.dedent('''\ + Hint: run `%(prog)s --help` to get help on other available actions and + optional arguments that can be used with all actions.''')) + pparser.add_storage_url() + + subparsers = parser.add_subparsers(metavar='', dest='action', + help='may be either of') + subparsers.add_parser("passphrase", help="change file system passphrase", + parents=[pparser]) + subparsers.add_parser("clear", help="delete file system and all data", + parents=[pparser]) + subparsers.add_parser("download-metadata", + help="Interactively download metadata backups. " + "Use only if you know what you are doing.", + parents=[pparser]) + sparser = subparsers.add_parser("upgrade", help="upgrade file system to newest revision", + parents=[pparser]) + sparser.add_argument("--threads", type=int, default=20, + help='Number of threads to use') + + parser.add_debug() + parser.add_quiet() + parser.add_log() + parser.add_authfile() + parser.add_backend_options() + parser.add_cachedir() + parser.add_version() + + options = parser.parse_args(args) + + return options + +def main(args=None): + '''Change or show S3QL file system parameters''' + + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + # Check if fs is mounted on this computer + # This is not foolproof but should prevent common mistakes + if is_mounted(options.storage_url): + raise QuietError('Can not work on mounted file system.') + + if options.action == 'clear': + with get_backend(options, raw=True) as backend: + return clear(backend, options) + + with get_backend(options) as backend: + if options.action == 'upgrade': + return upgrade(backend, get_backend_cachedir(options.storage_url, + options.cachedir)) + elif options.action == 'passphrase': + return change_passphrase(backend) + + elif options.action == 'download-metadata': + return download_metadata_cmd(backend, options.storage_url) + +def download_metadata_cmd(backend, storage_url): + '''Download old metadata backups''' + + backups = sorted(backend.list('s3ql_metadata')) + + if not backups: + raise QuietError('No metadata backups found.') + + log.info('The following backups are available:') + log.info('%3s %-23s %-15s', 'No', 'Name', 'Date') + for (i, name) in enumerate(backups): + try: + params = backend.lookup(name) + except: + log.error('Error retrieving information about %s, skipping', name) + continue + + if 'last-modified' in params: + date = Datetime.fromtimestamp(params['last-modified']).strftime('%Y-%m-%d %H:%M:%S') + else: + # (metadata might from an older fs revision) + date = '(unknown)' + + log.info('%3d %-23s %-15s', i, name, date) + + name = None + while name is None: + buf = input('Enter no to download: ') + try: + name = backups[int(buf.strip())] + except: + log.warning('Invalid input') + + cachepath = get_backend_cachedir(storage_url, '.') + for i in ('.db', '.params'): + if os.path.exists(cachepath + i): + raise QuietError('%s already exists, aborting.' % cachepath + i) + + param = backend.lookup(name) + download_metadata(backend, cachepath + ".db", name) + + # Raise sequence number so that fsck.s3ql actually uses the + # downloaded backup + seq_nos = [ int(x[len('s3ql_seq_no_'):]) for x in backend.list('s3ql_seq_no_') ] + param['seq_no'] = max(seq_nos) + 1 + save_params(cachepath, param) + +def change_passphrase(backend): + '''Change file system passphrase''' + + if not isinstance(backend, ComprencBackend) and backend.passphrase: + raise QuietError('File system is not encrypted.') + + data_pw = backend.passphrase + + if sys.stdin.isatty(): + wrap_pw = getpass("Enter new encryption password: ") + if not wrap_pw == getpass("Confirm new encryption password: "): + raise QuietError("Passwords don't match") + else: + wrap_pw = sys.stdin.readline().rstrip() + wrap_pw = wrap_pw.encode('utf-8') + + backend.passphrase = wrap_pw + backend['s3ql_passphrase'] = data_pw + backend['s3ql_passphrase_bak1'] = data_pw + backend['s3ql_passphrase_bak2'] = data_pw + backend['s3ql_passphrase_bak3'] = data_pw + backend.passphrase = data_pw + +def clear(backend, options): + print('I am about to delete all data in %s.' % backend, + 'This includes any S3QL file systems as well as any other stored objects.', + 'Please enter "yes" to continue.', '> ', sep='\n', end='') + sys.stdout.flush() + + if sys.stdin.readline().strip().lower() != 'yes': + raise QuietError() + + log.info('Deleting...') + cachepath = get_backend_cachedir(options.storage_url, options.cachedir) + for suffix in ('.db', '.params'): + name = cachepath + suffix + if os.path.exists(name): + os.unlink(name) + + name = cachepath + '-cache' + if os.path.exists(name): + shutil.rmtree(name) + + backend.clear() + + log.info('File system deleted.') + log.info('Note: it may take a while for the removals to propagate through the backend.') + +def get_old_rev_msg(rev, prog): + return textwrap.dedent('''\ + The last S3QL version that supported this file system revision + was %(version)s. To run this version's %(prog)s, proceed along + the following steps: + + $ wget http://s3ql.googlecode.com/files/s3ql-%(version)s.tar.bz2 \ + || wget https://bitbucket.org/nikratio/s3ql/downloads/s3ql-%(version)s.tar.bz2 + $ tar xjf s3ql-%(version)s.tar.bz2 + $ (cd s3ql-%(version)s; ./setup.py build_ext --inplace) + $ s3ql-%(version)s/bin/%(prog)s + ''' % { 'version': REV_VER_MAP[rev], + 'prog': prog }) + +def upgrade(backend, cachepath): + '''Upgrade file system to newest revision''' + + log.info('Getting file system parameters..') + + # Check for cached metadata + db = None + seq_no = get_seq_no(backend) + if os.path.exists(cachepath + '.params'): + param = load_params(cachepath) + if param['seq_no'] < seq_no: + log.info('Ignoring locally cached metadata (outdated).') + param = backend.lookup('s3ql_metadata') + elif param['seq_no'] > seq_no: + print('File system not unmounted cleanly, need to run fsck before upgrade.') + print(get_old_rev_msg(param['revision'], 'fsck.s3ql')) + raise QuietError() + else: + log.info('Using cached metadata.') + db = Connection(cachepath + '.db') + else: + param = backend.lookup('s3ql_metadata') + + # Check for unclean shutdown + if param['seq_no'] < seq_no: + print(textwrap.fill(textwrap.dedent('''\ + Backend reports that fs is still mounted. If this is not the case, the file system may + have not been unmounted cleanly or the data from the most-recent mount may have not yet + propagated through the backend. In the later case, waiting for a while should fix the + problem, in the former case you should try to run fsck on the computer where the file + system has been mounted most recently. + '''))) + + print(get_old_rev_msg(param['revision'], 'fsck.s3ql')) + raise QuietError() + + # Check that the fs itself is clean + if param['needs_fsck']: + print('File system is damaged, need to run fsck before upgrade.') + print(get_old_rev_msg(param['revision'], 'fsck.s3ql')) + raise QuietError() + + # Check revision + if param['revision'] < CURRENT_FS_REV-1: + print(textwrap.dedent(''' + File system revision too old to upgrade! + + You need to use an older S3QL version to upgrade to a more recent + revision before you can use this version to upgrade to the newest + revision. + ''')) + print(get_old_rev_msg(param['revision'] + 1, 's3qladm')) + raise QuietError() + + elif param['revision'] >= CURRENT_FS_REV: + print('File system already at most-recent revision') + return + + print(textwrap.dedent(''' + I am about to update the file system to the newest revision. + You will not be able to access the file system with any older version + of S3QL after this operation. + + You should make very sure that this command is not interrupted and + that no one else tries to mount, fsck or upgrade the file system at + the same time. + + ''')) + + print('Please enter "yes" to continue.', '> ', sep='\n', end='') + sys.stdout.flush() + + if sys.stdin.readline().strip().lower() != 'yes': + raise QuietError() + + if not db: + with monkeypatch_metadata_retrieval(): + db = download_metadata(backend, cachepath + '.db') + + log.info('Upgrading from revision %d to %d...', param['revision'], CURRENT_FS_REV) + + param['revision'] = CURRENT_FS_REV + param['last-modified'] = time.time() + param['seq_no'] += 1 + + # Upgrade + for name in ('atime', 'mtime', 'ctime'): + db.execute('ALTER TABLE inodes ADD COLUMN {time}_ns ' + 'INT NOT NULL DEFAULT 0'.format(time=name)) + db.execute('UPDATE inodes SET {time}_ns = {time} * 1e9'.format(time=name)) + + dump_and_upload_metadata(backend, db, param) + backend['s3ql_seq_no_%d' % param['seq_no']] = b'Empty' + + # Declare local metadata as outdated so that it won't be used. It still + # contains the old [acm]time columns which are NON NULL, and would prevent + # us from inserting new rows. + param['seq_no'] = 0 + save_params(cachepath, param) + + print('File system upgrade complete.') + + +@contextmanager +def monkeypatch_metadata_retrieval(): + create_tables_bak = metadata.create_tables + @functools.wraps(metadata.create_tables) + def create_tables(conn): + create_tables_bak(conn) + conn.execute('DROP TABLE inodes') + conn.execute(""" + CREATE TABLE inodes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + uid INT NOT NULL, + gid INT NOT NULL, + mode INT NOT NULL, + mtime REAL NOT NULL, + atime REAL NOT NULL, + ctime REAL NOT NULL, + refcount INT NOT NULL, + size INT NOT NULL DEFAULT 0, + rdev INT NOT NULL DEFAULT 0, + locked BOOLEAN NOT NULL DEFAULT 0 + )""") + metadata.create_tables = create_tables + + DUMP_SPEC_bak = metadata.DUMP_SPEC[2] + metadata.DUMP_SPEC[2] = ('inodes', 'id', (('id', INTEGER, 1), + ('uid', INTEGER), + ('gid', INTEGER), + ('mode', INTEGER), + ('mtime', TIME), + ('atime', TIME), + ('ctime', TIME), + ('size', INTEGER), + ('rdev', INTEGER), + ('locked', INTEGER), + ('refcount', INTEGER))) + + try: + yield + finally: + metadata.DUMP_SPEC[2] = DUMP_SPEC_bak + metadata.create_tables = create_tables_bak + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/src/s3ql/backends/__init__.py b/src/s3ql/backends/__init__.py new file mode 100644 index 0000000..e80f9b2 --- /dev/null +++ b/src/s3ql/backends/__init__.py @@ -0,0 +1,21 @@ +''' +backends/__init__.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from . import local, s3, gs, s3c, swift, rackspace, swiftks + +#: Mapping from storage URL prefixes to backend classes +prefix_map = { 's3': s3.Backend, + 'local': local.Backend, + 'gs': gs.Backend, + 's3c': s3c.Backend, + 'swift': swift.Backend, + 'swiftks': swiftks.Backend, + 'rackspace': rackspace.Backend } + +__all__ = [ 'common', 'pool', 'comprenc' ] + list(prefix_map.keys()) + diff --git a/src/s3ql/backends/common.py b/src/s3ql/backends/common.py new file mode 100644 index 0000000..28a728a --- /dev/null +++ b/src/s3ql/backends/common.py @@ -0,0 +1,701 @@ +''' +common.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from ..logging import logging, QuietError, LOG_ONCE # Ensure use of custom logger class +from abc import abstractmethod, ABCMeta +from functools import wraps +import time +import textwrap +import hashlib +import struct +import hmac +import random +import inspect +import ssl +import os +import re +import threading + +log = logging.getLogger(__name__) + +class RateTracker: + ''' + Maintain an average occurence rate for events over a configurable time + window. The rate is computed with one second resolution. + ''' + + def __init__(self, window_length): + if not isinstance(window_length, int): + raise ValueError('only integer window lengths are supported') + + self.buckets = [0] * window_length + self.window_length = window_length + self.last_update = int(time.monotonic()) + self.lock = threading.Lock() + + def register(self, _not_really=False): + '''Register occurence of an event. + + The keyword argument is for class-internal use only. + ''' + + buckets = self.buckets + bucket_count = len(self.buckets) + now = int(time.monotonic()) + + elapsed = min(now - self.last_update, bucket_count) + for i in range(elapsed): + buckets[(now - i) % bucket_count] = 0 + + if _not_really: + return + + with self.lock: + buckets[now % bucket_count] += 1 + self.last_update = now + + def get_rate(self): + '''Return average rate of event occurance''' + + self.register(_not_really=True) + return sum(self.buckets) / len(self.buckets) + + def get_count(self): + '''Return total number of events in window''' + + self.register(_not_really=True) + return sum(self.buckets) + + +# We maintain a (global) running average of temporary errors, so +# that we can log a warning if this number becomes large. We +# use a relatively large window to prevent bogus spikes if +# multiple threads all have to retry after a long period of +# inactivity. +RETRY_TIMEOUT = 60 * 60 * 24 +def retry(method, _tracker=RateTracker(60)): + '''Wrap *method* for retrying on some exceptions + + If *method* raises an exception for which the instance's + `is_temp_failure(exc)` method is true, the *method* is called again at + increasing intervals. If this persists for more than `RETRY_TIMEOUT` + seconds, the most-recently caught exception is re-raised. If the + method defines a keyword parameter *is_retry*, then this parameter + will be set to True whenever the function is retried. + ''' + + if inspect.isgeneratorfunction(method): + raise TypeError('Wrapping a generator function is pointless') + + sig = inspect.signature(method) + has_is_retry = 'is_retry' in sig.parameters + + @wraps(method) + def wrapped(*a, **kw): + self = a[0] + interval = 1 / 50 + waited = 0 + retries = 0 + while True: + if has_is_retry: + kw['is_retry'] = (retries > 0) + try: + return method(*a, **kw) + except Exception as exc: + # Access to protected member ok + #pylint: disable=W0212 + if not self.is_temp_failure(exc): + raise + + _tracker.register() + rate = _tracker.get_rate() + if rate > 5: + log.warning('Had to retry %d times over the last %d seconds, ' + 'server or network problem?', + rate * _tracker.window_length, _tracker.window_length) + else: + log.debug('Average retry rate: %.2f Hz', rate) + + if waited > RETRY_TIMEOUT: + log.error('%s.%s(*): Timeout exceeded, re-raising %r exception', + self.__class__.__name__, method.__name__, exc) + raise + + retries += 1 + if retries <= 2: + log_fn = log.debug + elif retries <= 4: + log_fn = log.info + else: + log_fn = log.warning + + log_fn('Encountered %s (%s), retrying %s.%s (attempt %d)...', + type(exc).__name__, exc, self.__class__.__name__, method.__name__, + retries) + + if hasattr(exc, 'retry_after') and exc.retry_after: + log.debug('retry_after is %.2f seconds', exc.retry_after) + interval = exc.retry_after + + # Add some random variation to prevent flooding the + # server with too many concurrent requests. + time.sleep(interval * random.uniform(1, 1.5)) + waited += interval + interval = min(5*60, 2*interval) + + extend_docstring(wrapped, + 'This method has been wrapped and will automatically re-execute in ' + 'increasing intervals for up to `s3ql.backends.common.RETRY_TIMEOUT` ' + 'seconds if it raises an exception for which the instance\'s ' + '`is_temp_failure` method returns True.') + + return wrapped + +def extend_docstring(fun, s): + '''Append *s* to *fun*'s docstring with proper wrapping and indentation''' + + if fun.__doc__ is None: + fun.__doc__ = '' + + # Figure out proper indentation + indent = 60 + for line in fun.__doc__.splitlines()[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + + indent_s = '\n' + ' ' * indent + fun.__doc__ += ''.join(indent_s + line + for line in textwrap.wrap(s, width=80 - indent)) + fun.__doc__ += '\n' + +class RetryIterator: + ''' + A RetryIterator instance iterates over the elements produced by any + generator function passed to its constructor, i.e. it wraps the iterator + obtained by calling the generator function. When retrieving elements from the + wrapped iterator, exceptions may occur. Most such exceptions are + propagated. However, exceptions for which the *is_temp_failure_fn* function + returns True are caught. If that happens, the wrapped iterator is replaced + by a new one obtained by calling the generator function again with the + *start_after* parameter set to the last element that was retrieved before + the exception occured. + + If attempts to retrieve the next element fail repeatedly, the iterator is + replaced only after sleeping for increasing intervals. If no new element can + be obtained after `RETRY_TIMEOUT` seconds, the last exception is no longer + caught but propagated to the caller. This behavior is implemented by + wrapping the __next__ method with the `retry` decorator. + ''' + + def __init__(self, generator, is_temp_failure_fn, args=(), kwargs=None): + if not inspect.isgeneratorfunction(generator): + raise TypeError('*generator* must be generator function') + + self.generator = generator + self.iterator = None + self.is_temp_failure = is_temp_failure_fn + if kwargs is None: + kwargs = {} + self.kwargs = kwargs + self.args = args + + def __iter__(self): + return self + + @retry + def __next__(self): + if self.iterator is None: + self.iterator = self.generator(*self.args, **self.kwargs) + + try: + el = next(self.iterator) + except Exception as exc: + if self.is_temp_failure(exc): + self.iterator = None + raise + + self.kwargs['start_after'] = el + return el + +def retry_generator(method): + '''Wrap *method* in a `RetryIterator` + + *method* must return a generator, and accept a keyword argument + *start_with*. The RetryIterator's `is_temp_failure` attribute + will be set to the `is_temp_failure` method of the instance + to which *method* is bound. + ''' + + @wraps(method) + def wrapped(*a, **kw): + return RetryIterator(method, a[0].is_temp_failure, args=a, kwargs=kw) + + extend_docstring(wrapped, + 'This generator method has been wrapped and will return a ' + '`RetryIterator` instance.') + + return wrapped + +class AbstractBackend(object, metaclass=ABCMeta): + '''Functionality shared between all backends. + + Instances behave similarly to dicts. They can be iterated over and + indexed into, but raise a separate set of exceptions. + + The backend guarantees get after create consistency, i.e. a newly created + object will be immediately retrievable. Additional consistency guarantees + may or may not be available and can be queried for with instance methods. + ''' + + needs_login = True + + def __init__(self): + super().__init__() + + def __getitem__(self, key): + return self.fetch(key)[0] + + def __setitem__(self, key, value): + self.store(key, value) + + def __delitem__(self, key): + self.delete(key) + + def __iter__(self): + return self.list() + + def __contains__(self, key): + return self.contains(key) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, tb): + self.close() + return False + + def iteritems(self): + for key in self.list(): + yield (key, self[key]) + + @property + @abstractmethod + def has_native_rename(self): + '''True if the backend has a native, atomic rename operation''' + pass + + def reset(self): + '''Reset backend + + This resets the backend and ensures that it is ready to process + requests. In most cases, this method does nothing. However, if e.g. a + file handle returned by a previous call to `open_read` was not properly + closed (e.g. because an exception happened during reading), the `reset` + method will make sure that any underlying connection is properly closed. + + Obviously, this method must not be called while any file handles + returned by the backend are still in use. + ''' + + pass + + @retry + def perform_read(self, fn, key): + '''Read object data using *fn*, retry on temporary failure + + Open object for reading, call `fn(fh)` and close object. If a temporary + error (as defined by `is_temp_failure`) occurs during opening, closing + or execution of *fn*, the operation is retried. + ''' + + fh = self.open_read(key) + try: + res = fn(fh) + except Exception as exc: + # If this is a temporary failure, we now that the call will be + # retried, so we don't need to warn that the object was not read + # completely. + fh.close(checksum_warning=not self.is_temp_failure(exc)) + raise + else: + fh.close() + return res + + @retry + def perform_write(self, fn, key, metadata=None, is_compressed=False): + '''Read object data using *fn*, retry on temporary failure + + Open object for writing, call `fn(fh)` and close object. If a temporary + error (as defined by `is_temp_failure`) occurs during opening, closing + or execution of *fn*, the operation is retried. + ''' + + with self.open_write(key, metadata, is_compressed) as fh: + return fn(fh) + + def fetch(self, key): + """Return data stored under `key`. + + Returns a tuple with the data and metadata. If only the data itself is + required, ``backend[key]`` is a more concise notation for + ``backend.fetch(key)[0]``. + """ + + def do_read(fh): + data = fh.read() + return (data, fh.metadata) + + return self.perform_read(do_read, key) + + def store(self, key, val, metadata=None): + """Store data under `key`. + + `metadata` can be mapping with additional attributes to store with the + object. Keys have to be of type `str`, values have to be of elementary + type (`str`, `bytes`, `int`, `float` or `bool`). + + If no metadata is required, one can simply assign to the subscripted + backend instead of using this function: ``backend[key] = val`` is + equivalent to ``backend.store(key, val)``. + """ + + self.perform_write(lambda fh: fh.write(val), key, metadata) + + @abstractmethod + def is_temp_failure(self, exc): + '''Return true if exc indicates a temporary error + + Return true if the given exception indicates a temporary problem. Most + instance methods automatically retry the request in this case, so the + caller does not need to worry about temporary failures. + + However, in same cases (e.g. when reading or writing an object), the + request cannot automatically be retried. In these case this method can + be used to check for temporary problems and so that the request can be + manually restarted if applicable. + ''' + + pass + + @abstractmethod + def lookup(self, key): + """Return metadata for given key. + + If the key does not exist, `NoSuchObject` is raised. + """ + + pass + + @abstractmethod + def get_size(self, key): + '''Return size of object stored under *key*''' + pass + + @abstractmethod + def open_read(self, key): + """Open object for reading + + Return a file-like object. Data can be read using the `read` + method. Metadata is returned in the file-like object's *metadata* + attribute and can be modified by the caller at will. The object must be + closed explicitly. + """ + + pass + + @abstractmethod + def open_write(self, key, metadata=None, is_compressed=False): + """Open object for writing + + `metadata` can be mapping with additional attributes to store with the + object. Keys have to be of type `str`, values have to be of elementary + type (`str`, `bytes`, `int`, `float` or `bool`). + + Returns a file- like object. The object must be closed closed + explicitly. After closing, the *get_obj_size* may be used to retrieve + the size of the stored object (which may differ from the size of the + written data). + + The *is_compressed* parameter indicates that the caller is going to + write compressed data, and may be used to avoid recompression by the + backend. + """ + + pass + + @abstractmethod + def clear(self): + """Delete all objects in backend""" + pass + + def contains(self, key): + '''Check if `key` is in backend''' + + try: + self.lookup(key) + except NoSuchObject: + return False + else: + return True + + @abstractmethod + def delete(self, key, force=False): + """Delete object stored under `key` + + ``backend.delete(key)`` can also be written as ``del backend[key]``. If + `force` is true, do not return an error if the key does not exist. Note, + however, that even if *force* is False, it is not guaranteed that an + attempt to delete a non-existing object will raise an error. + """ + pass + + def delete_multi(self, keys, force=False): + """Delete objects stored under `keys` + + Deleted objects are removed from the *keys* list, so that the caller can + determine which objects have not yet been processed if an exception is + occurs. + + If *force* is True, attempts to delete non-existing objects will + succeed. Note, however, that even if *force* is False, it is not + guaranteed that an attempt to delete a non-existing object will raise an + error. + """ + + if not isinstance(keys, list): + raise TypeError('*keys* parameter must be a list') + + for (i, key) in enumerate(keys): + try: + self.delete(key, force=force) + except: + del keys[:i] + raise + + del keys[:] + + @abstractmethod + def list(self, prefix=''): + '''List keys in backend + + Returns an iterator over all keys in the backend. + ''' + pass + + @abstractmethod + def copy(self, src, dest, metadata=None): + """Copy data stored under key `src` to key `dest` + + If `dest` already exists, it will be overwritten. If *metadata* is + `None` metadata will be copied from the source as well, otherwise + *metadata* becomes the metadata for the new object. + + Copying will be done on the remote side without retrieving object data. + """ + + pass + + @abstractmethod + def update_meta(self, key, metadata): + """Replace metadata of *key* with *metadata* + + `metadata` must be a mapping with keys of type `str`, and values of an + elementary type (`str`, `bytes`, `int`, `float` or `bool`). + """ + + pass + + def rename(self, src, dest, metadata=None): + """Rename key `src` to `dest` + + If `dest` already exists, it will be overwritten. If *metadata* is + `None` metadata will be preserved, otherwise *metadata* becomes the + metadata for the renamed object. + + Rename done remotely without retrieving object data. + """ + + self.copy(src, dest, metadata) + self.delete(src) + + def close(self): + '''Close any opened resources + + This method closes any resources allocated by the backend (e.g. network + sockets). This method should be called explicitly before a backend + object is garbage collected. The backend object may be re-used after + `close` has been called, in this case the necessary resources are + transparently allocated again. + ''' + + pass + +class NoSuchObject(Exception): + '''Raised if the requested object does not exist in the backend''' + + def __init__(self, key): + super().__init__() + self.key = key + + def __str__(self): + return 'Backend does not have anything stored under key %r' % self.key + +class DanglingStorageURLError(Exception): + '''Raised if the backend can't store data at the given location''' + + def __init__(self, loc, msg=None): + super().__init__() + self.loc = loc + self.msg = msg + + def __str__(self): + if self.msg is None: + return '%r does not exist' % self.loc + else: + return self.msg + +class AuthorizationError(Exception): + '''Raised if the credentials don't give access to the requested backend''' + + def __init__(self, msg): + super().__init__() + self.msg = msg + + def __str__(self): + return 'Access denied. Server said: %s' % self.msg + +class AuthenticationError(Exception): + '''Raised if the credentials are invalid''' + + def __init__(self, msg): + super().__init__() + self.msg = msg + + def __str__(self): + return 'Access denied. Server said: %s' % self.msg + +class CorruptedObjectError(Exception): + """ + Raised if a storage object is corrupted. + + Note that this is different from BadDigest error, which is raised + if a transmission error has been detected. + """ + + def __init__(self, str_): + super().__init__() + self.str = str_ + + def __str__(self): + return self.str + +def get_ssl_context(path): + '''Construct SSLContext object''' + + # Best practice according to http://docs.python.org/3/library/ssl.html#protocol-versions + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.options |= ssl.OP_NO_SSLv2 + context.verify_mode = ssl.CERT_REQUIRED + + if path is None: + log.debug('Reading default CA certificates.') + context.set_default_verify_paths() + elif os.path.isfile(path): + log.debug('Reading CA certificates from file %s', path) + context.load_verify_locations(cafile=path) + else: + log.debug('Reading CA certificates from directory %s', path) + context.load_verify_locations(capath=path) + + return context + +def get_proxy(ssl): + '''Read system proxy settings + + Returns either `None`, or a tuple ``(host, port)``. + + This function may raise `QuietError`. + ''' + + if ssl: + proxy_env = 'https_proxy' + else: + proxy_env = 'http_proxy' + + if proxy_env in os.environ: + proxy = os.environ[proxy_env] + hit = re.match(r'^(https?://)?([a-zA-Z0-9.-]+)(:[0-9]+)?/?$', proxy) + if not hit: + raise QuietError('Unable to parse proxy setting %s=%r' % + (proxy_env, proxy), exitcode=13) + + if hit.group(1) == 'https://': + log.warning('HTTPS connection to proxy is probably pointless and not supported, ' + 'will use standard HTTP', extra=LOG_ONCE) + + if hit.group(3): + proxy_port = int(hit.group(3)[1:]) + else: + proxy_port = 80 + + proxy_host = hit.group(2) + log.info('Using CONNECT proxy %s:%d', proxy_host, proxy_port, + extra=LOG_ONCE) + proxy = (proxy_host, proxy_port) + else: + proxy = None + + return proxy + +def checksum_basic_mapping(metadata, key=None): + '''Compute checksum for mapping of elementary types + + Keys of *d* must be strings. Values of *d* must be of elementary + type (i.e., `str`, `bytes`, `int`, `float`, `complex`, `bool` or + None). If there is a key named ``signature``, then it is excluded + from the checksum computation. + + If *key* is None, compute MD5. Otherwise compute HMAC using *key*. + ''' + + # In order to compute a safe checksum, we need to convert each object to a + # unique representation (i.e, we can't use repr(), as it's output may change + # in the future). Furthermore, we have to make sure that the representation + # is theoretically reversible, or there is a potential for collision + # attacks. + + if key is None: + chk = hashlib.md5() + else: + chk = hmac.new(key, digestmod=hashlib.sha256) + + for mkey in sorted(metadata.keys()): + assert isinstance(mkey, str) + if mkey == 'signature': + continue + chk.update(mkey.encode('utf-8')) + val = metadata[mkey] + if isinstance(val, str): + val = b'\0s' + val.encode('utf-8') + b'\0' + elif val is None: + val = b'\0n' + elif isinstance(val, int): + val = b'\0i' + ('%d' % val).encode() + b'\0' + elif isinstance(val, bool): + val = b'\0t' if val else b'\0f' + elif isinstance(val, float): + val = b'\0d' + struct.pack(' + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from ..logging import logging # Ensure use of custom logger class +from .. import BUFSIZE +from .common import AbstractBackend, CorruptedObjectError, checksum_basic_mapping +from ..common import ThawError, freeze_basic_mapping, thaw_basic_mapping +from ..inherit_docstrings import (copy_ancestor_docstring, prepend_ancestor_docstring, + ABCDocstMeta) +from Crypto.Cipher import AES +from Crypto.Util import Counter +import bz2 +import hashlib +import hmac +import lzma +import io +import struct +import time +import zlib + +log = logging.getLogger(__name__) + +HMAC_SIZE = 32 + +def sha256(s): + return hashlib.sha256(s).digest() + +def aes_cipher(key): + '''Return AES cipher in CTR mode for *key*''' + + return AES.new(key, AES.MODE_CTR, + counter=Counter.new(128, initial_value=0)) + +class ComprencBackend(AbstractBackend, metaclass=ABCDocstMeta): + ''' + This class adds encryption, compression and integrity protection to a plain + backend. + ''' + + def __init__(self, passphrase, compression, backend): + super().__init__() + + assert passphrase is None or isinstance(passphrase, (bytes, bytearray, memoryview)) + + self.passphrase = passphrase + self.compression = compression + self.backend = backend + + if (compression[0] not in ('bzip2', 'lzma', 'zlib', None) + or compression[1] not in range(10)): + raise ValueError('Unsupported compression: %s' % compression) + + @property + @copy_ancestor_docstring + def has_native_rename(self): + return self.backend.has_native_rename + + @copy_ancestor_docstring + def reset(self): + self.backend.reset() + + @copy_ancestor_docstring + def lookup(self, key): + meta_raw = self.backend.lookup(key) + return self._verify_meta(key, meta_raw)[1] + + @prepend_ancestor_docstring + def get_size(self, key): + ''' + This method returns the compressed size, i.e. the storage space + that's actually occupied by the object. + ''' + + return self.backend.get_size(key) + + @copy_ancestor_docstring + def is_temp_failure(self, exc): + return self.backend.is_temp_failure(exc) + + def _verify_meta(self, key, metadata): + '''Unwrap and authenticate metadata + + If the backend has a password set but the object is not encrypted, + `ObjectNotEncrypted` is raised. Returns the object nonce and its + metadata. If the object is not encrypted, the nonce is `None`. + ''' + + if not isinstance(metadata, dict): + raise CorruptedObjectError('metadata should be dict, not %s' % type(metadata)) + + format_version = metadata.get('format_version', 0) + if format_version != 2: + raise CorruptedObjectError('format_version %s unsupported' % format_version) + + for mkey in ('encryption', 'compression', 'data'): + if mkey not in metadata: + raise CorruptedObjectError('meta key %s is missing' % mkey) + + encr_alg = metadata['encryption'] + encrypted = (encr_alg != 'None') + + if encrypted and self.passphrase is None: + raise CorruptedObjectError('Encrypted object and no passphrase supplied') + + elif not encrypted and self.passphrase is not None: + raise ObjectNotEncrypted() + + meta_buf = metadata['data'] + if not encrypted: + try: + return (None, thaw_basic_mapping(meta_buf)) + except ThawError: + raise CorruptedObjectError('Invalid metadata') + + # Encrypted + for mkey in ('nonce', 'signature', 'object_id'): + if mkey not in metadata: + raise CorruptedObjectError('meta key %s is missing' % mkey) + + nonce = metadata['nonce'] + stored_key = metadata['object_id'] + meta_key = sha256(self.passphrase + nonce + b'meta') + meta_sig = checksum_basic_mapping(metadata, meta_key) + if not hmac.compare_digest(metadata['signature'], meta_sig): + raise CorruptedObjectError('HMAC mismatch') + + if stored_key != key: + raise CorruptedObjectError('Object content does not match its key (%s vs %s)' + % (stored_key, key)) + + buf = aes_cipher(meta_key).decrypt(meta_buf) + try: + return (nonce, thaw_basic_mapping(buf)) + except ThawError: + raise CorruptedObjectError('Invalid metadata') + + @prepend_ancestor_docstring + def open_read(self, key): + """ + If the backend has a password set but the object is not encrypted, + `ObjectNotEncrypted` is raised. + """ + + fh = self.backend.open_read(key) + try: + meta_raw = fh.metadata + (nonce, meta) = self._verify_meta(key, meta_raw) + if nonce: + data_key = sha256(self.passphrase + nonce) + + # The `payload_offset` key only exists if the storage object was + # created with on old S3QL version. In order to avoid having to + # download and re-upload the entire object during the upgrade, the + # upgrade procedure adds this header to tell us how many bytes at + # the beginning of the object we have to skip to get to the payload. + if 'payload_offset' in meta_raw: + to_skip = meta_raw['payload_offset'] + while to_skip: + to_skip -= len(fh.read(to_skip)) + + encr_alg = meta_raw['encryption'] + if encr_alg == 'AES_v2': + fh = DecryptFilter(fh, data_key) + elif encr_alg != 'None': + raise RuntimeError('Unsupported encryption: %s' % encr_alg) + + compr_alg = meta_raw['compression'] + if compr_alg == 'BZIP2': + fh = DecompressFilter(fh, bz2.BZ2Decompressor()) + elif compr_alg == 'LZMA': + fh = DecompressFilter(fh, lzma.LZMADecompressor()) + elif compr_alg == 'ZLIB': + fh = DecompressFilter(fh,zlib.decompressobj()) + elif compr_alg != 'None': + raise RuntimeError('Unsupported compression: %s' % compr_alg) + + fh.metadata = meta + except: + # Don't emit checksum warning, caller hasn't even + # started reading anything. + fh.close(checksum_warning=False) + raise + + return fh + + @copy_ancestor_docstring + def open_write(self, key, metadata=None, is_compressed=False): + + if metadata is None: + metadata = dict() + elif not isinstance(metadata, dict): + raise TypeError('*metadata*: expected dict or None, got %s' % type(metadata)) + + meta_buf = freeze_basic_mapping(metadata) + meta_raw = dict(format_version=2) + + if is_compressed or self.compression[0] is None: + compr = None + meta_raw['compression'] = 'None' + elif self.compression[0] == 'zlib': + compr = zlib.compressobj(self.compression[1]) + meta_raw['compression'] = 'ZLIB' + elif self.compression[0] == 'bzip2': + compr = bz2.BZ2Compressor(self.compression[1]) + meta_raw['compression'] = 'BZIP2' + elif self.compression[0] == 'lzma': + compr = lzma.LZMACompressor(preset=self.compression[1]) + meta_raw['compression'] = 'LZMA' + + if self.passphrase is not None: + nonce = struct.pack(' HMAC_SIZE or self.fh.read(1): + # Read rest of stream, so that we raise MD5 error instead + # if problem is on lower layer + self.discard_input() + raise CorruptedObjectError('Extraneous data at end of object') + + if not hmac.compare_digest(inbuf, self.hmac.digest()): + raise CorruptedObjectError('HMAC mismatch') + + self.hmac_checked = True + break + + return outbuf + + def close(self, *a, **kw): + self.fh.close(*a, **kw) + + def __enter__(self): + return self + + def __exit__(self, *a): + self.close() + return False + +def decompress(decomp, buf): + '''Decompress *buf* using *decomp* + + This method encapsulates exception handling for different + decompressors. + ''' + + try: + return decomp.decompress(buf) + except IOError as exc: + if exc.args[0].lower().startswith('invalid data stream'): + raise CorruptedObjectError('Invalid compressed stream') + raise + except lzma.LZMAError as exc: + if (exc.args[0].lower().startswith('corrupt input data') + or exc.args[0].startswith('Input format not supported')): + raise CorruptedObjectError('Invalid compressed stream') + raise + except zlib.error as exc: + if exc.args[0].lower().startswith('error -3 while decompressing'): + raise CorruptedObjectError('Invalid compressed stream') + raise + +class ObjectNotEncrypted(Exception): + ''' + Raised by the backend if an object was requested from an encrypted + backend, but the object was stored without encryption. + + We do not want to simply return the uncrypted object, because the + caller may rely on the objects integrity being cryptographically + verified. + ''' + + pass diff --git a/src/s3ql/backends/gs.py b/src/s3ql/backends/gs.py new file mode 100644 index 0000000..ceb8b89 --- /dev/null +++ b/src/s3ql/backends/gs.py @@ -0,0 +1,240 @@ +''' +backends/gs.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from ..logging import logging, QuietError # Ensure use of custom logger class +from . import s3c +from .s3c import C_DAY_NAMES, C_MONTH_NAMES, HTTPError, S3Error +from .common import AuthenticationError, retry, NoSuchObject +from .. import oauth_client +from ..inherit_docstrings import copy_ancestor_docstring +from dugong import CaseInsensitiveDict, HTTPConnection +from urllib.parse import urlencode +import re +import json +import threading +import time + +# Pylint goes berserk with false positives +#pylint: disable=E1002,E1101,W0201 + +log = logging.getLogger(__name__) + +class Backend(s3c.Backend): + """A backend to store data in Google Storage + + This class uses standard HTTP connections to connect to GS. + + The backend guarantees immediate get consistency and eventual list + consistency. + """ + + xml_ns_prefix = '{http://doc.s3.amazonaws.com/2006-03-01}' + known_options = (s3c.Backend.known_options + - {'dumb-copy', 'disable-expect100' }) + + # We don't want to request an access token for each instance, + # because there is a limit on the total number of valid tokens. + # This class variable holds the mapping from refresh tokens to + # access tokens. + access_token = dict() + _refresh_lock = threading.Lock() + + def __init__(self, storage_url, gs_key, gs_secret, options): + super().__init__(storage_url, gs_key, gs_secret, options) + + self.use_oauth2 = (gs_key == 'oauth2') + + self.options['disable-expect100'] = True + if self.use_oauth2: + self.hdr_prefix = 'x-goog-' + + @staticmethod + def _parse_storage_url(storage_url, ssl_context): + # Special case for unit testing against local mock server + hit = re.match(r'^gs://!unittest!' + r'([^/:]+)' # Hostname + r':([0-9]+)' # Port + r'/([^/]+)' # Bucketname + r'(?:/(.*))?$', # Prefix + storage_url) + if hit: + hostname = hit.group(1) + port = int(hit.group(2)) + bucket_name = hit.group(3) + prefix = hit.group(4) or '' + return (hostname, port, bucket_name, prefix) + + hit = re.match(r'^gs://([^/]+)(?:/(.*))?$', storage_url) + if not hit: + raise QuietError('Invalid storage URL', exitcode=2) + + bucket_name = hit.group(1) + + # Dots in the bucket cause problems with SSL certificate validation, + # because server certificate is for *.commondatastorage.googleapis.com + # (which does not match e.g. a.b.commondatastorage.googleapis.com) + if '.' in bucket_name and ssl_context: + hostname = 'commondatastorage.googleapis.com' + else: + hostname = '%s.commondatastorage.googleapis.com' % bucket_name + + prefix = hit.group(2) or '' + port = 443 if ssl_context else 80 + return (hostname, port, bucket_name, prefix) + + def __str__(self): + return 'Google Storage bucket %s, prefix %s' % (self.bucket_name, self.prefix) + + def _authorize_request(self, method, path, headers, subres): + '''Add authorization information to *headers*''' + + if not self.use_oauth2: + return super()._authorize_request(method, path, headers, subres) + + headers['Authorization'] = 'Bearer ' + self.access_token[self.password] + + now = time.gmtime() + headers['Date'] = ('%s, %02d %s %04d %02d:%02d:%02d GMT' + % (C_DAY_NAMES[now.tm_wday], + now.tm_mday, + C_MONTH_NAMES[now.tm_mon - 1], + now.tm_year, now.tm_hour, + now.tm_min, now.tm_sec)) + + # This method performs a different kind of HTTP request than the methods + # decorated with `retry` that it is called by, so in theory it should do its + # own retry handling (perhaps with a new `retry_on` decorator that allows to + # specify a custom `is_temp_failure` function instead of calling the + # instance method). However, in practice there is currently no difference in + # the set of exceptions that are considered temporary when retrieving an + # access token, and the set of exceptions checked for in the + # `_is_temp_failure` method. Therefore, for now we avoid the additional + # complexity of custom retry handling and rely on the @retry decorator of + # the caller to handle temporary errors. This should be kept in mind + # when modifying either method. + def _get_access_token(self): + log.info('Requesting new access token') + + headers = CaseInsensitiveDict() + headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8' + + body = urlencode({'client_id': oauth_client.CLIENT_ID, + 'client_secret': oauth_client.CLIENT_SECRET, + 'refresh_token': self.password, + 'grant_type': 'refresh_token' }) + + conn = HTTPConnection('accounts.google.com', 443, proxy=self.proxy, + ssl_context=self.ssl_context) + try: + + conn.send_request('POST', '/o/oauth2/token', headers=headers, + body=body.encode('utf-8')) + resp = conn.read_response() + + if resp.status > 299 or resp.status < 200: + raise HTTPError(resp.status, resp.reason, resp.headers) + + content_type = resp.headers.get('Content-Type', None) + if content_type: + hit = re.match(r'application/json(?:; charset="(.+)")?$', + resp.headers['Content-Type'], re.IGNORECASE) + else: + hit = None + + if not hit: + log.error('Unexpected server reply when refreshing access token:\n%s', + self._dump_response(resp)) + raise RuntimeError('Unable to parse server response') + + charset = hit.group(1) or 'utf-8' + body = conn.readall().decode(charset) + resp_json = json.loads(body) + + if not isinstance(resp_json, dict): + log.error('Invalid json server response. Expected dict, got:\n%s', body) + raise RuntimeError('Unable to parse server response') + + if 'error' in resp_json: + raise AuthenticationError(resp_json['error']) + + if 'access_token' not in resp_json: + log.error('Unable to find access token in server response:\n%s', body) + raise RuntimeError('Unable to parse server response') + + self.access_token[self.password] = resp_json['access_token'] + + finally: + conn.disconnect() + + def _do_request(self, method, path, subres=None, query_string=None, + headers=None, body=None): + + # When not using OAuth2, fall-through. + if not self.use_oauth2: + return super()._do_request(method, path, subres=subres, headers=headers, + query_string=query_string, body=body) + + # If we have an access token, try to use it. + token = self.access_token.get(self.password, None) + if token is not None: + try: + return super()._do_request(method, path, subres=subres, headers=headers, + query_string=query_string, body=body) + except HTTPError as exc: + if exc.status != 401: + raise + except S3Error as exc: + if exc.code != 'AuthenticationRequired': + raise + + # If we reach this point, then the access token must have + # expired, so we try to get a new one. We use a lock to prevent + # multiple threads from refreshing the token simultaneously. + with self._refresh_lock: + # Don't refresh if another thread has already done so while + # we waited for the lock. + if token is None or self.access_token.get(self.password, None) == token: + self._get_access_token() + + # Reset body, so we can resend the request with the new access token + if body and not isinstance(body, (bytes, bytearray, memoryview)): + body.seek(0) + + # Try request again. If this still fails, propagate the error + # (because we have just refreshed the access token). + # FIXME: We can't rely on this if e.g. the system hibernated + # after refreshing the token, but before reaching this line. + return super()._do_request(method, path, subres=subres, headers=headers, + query_string=query_string, body=body) + + # Overwrite, because Google Storage does not return errors after + # 200 OK. + @retry + @copy_ancestor_docstring + def copy(self, src, dest, metadata=None): + log.debug('started with %s, %s', src, dest) + + if not (metadata is None or isinstance(metadata, dict)): + raise TypeError('*metadata*: expected dict or None, got %s' % type(metadata)) + + headers = CaseInsensitiveDict() + headers[self.hdr_prefix + 'copy-source'] = \ + '/%s/%s%s' % (self.bucket_name, self.prefix, src) + + if metadata is None: + headers[self.hdr_prefix + 'metadata-directive'] = 'COPY' + else: + headers[self.hdr_prefix + 'metadata-directive'] = 'REPLACE' + self._add_meta_headers(headers, metadata) + + try: + self._do_request('PUT', '/%s%s' % (self.prefix, dest), headers=headers) + self.conn.discard() + except s3c.NoSuchKeyError: + raise NoSuchObject(src) + diff --git a/src/s3ql/backends/local.py b/src/s3ql/backends/local.py new file mode 100644 index 0000000..a8ebf66 --- /dev/null +++ b/src/s3ql/backends/local.py @@ -0,0 +1,338 @@ +''' +local.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from ..logging import logging # Ensure use of custom logger class +from .. import BUFSIZE +from ..inherit_docstrings import (copy_ancestor_docstring, ABCDocstMeta) +from .common import (AbstractBackend, DanglingStorageURLError, NoSuchObject, + CorruptedObjectError) +from ..common import ThawError, freeze_basic_mapping, thaw_basic_mapping +import _thread +import struct +import io +import os +import shutil + +log = logging.getLogger(__name__) + +class Backend(AbstractBackend, metaclass=ABCDocstMeta): + ''' + A backend that stores data on the local hard disk + ''' + + needs_login = False + known_options = set() + + def __init__(self, storage_url, backend_login=None, backend_pw=None, + options=None): + '''Initialize local backend + + Login and password are ignored. + ''' + # Unused argument + #pylint: disable=W0613 + + super().__init__() + self.prefix = storage_url[len('local://'):].rstrip('/') + + if not os.path.exists(self.prefix): + raise DanglingStorageURLError(self.prefix) + + @property + @copy_ancestor_docstring + def has_native_rename(self): + return False + + def __str__(self): + return 'local directory %s' % self.prefix + + @copy_ancestor_docstring + def is_temp_failure(self, exc): #IGNORE:W0613 + return False + + @copy_ancestor_docstring + def lookup(self, key): + path = self._key_to_path(key) + try: + with open(path, 'rb') as src: + return _read_meta(src) + except FileNotFoundError: + raise NoSuchObject(key) + + @copy_ancestor_docstring + def get_size(self, key): + return os.path.getsize(self._key_to_path(key)) + + @copy_ancestor_docstring + def open_read(self, key): + path = self._key_to_path(key) + try: + fh = ObjectR(path) + except FileNotFoundError: + raise NoSuchObject(key) + + try: + fh.metadata = _read_meta(fh) + except ThawError: + fh.close() + raise CorruptedObjectError('Invalid metadata') + return fh + + @copy_ancestor_docstring + def open_write(self, key, metadata=None, is_compressed=False): + if metadata is None: + metadata = dict() + elif not isinstance(metadata, dict): + raise TypeError('*metadata*: expected dict or None, got %s' % type(metadata)) + + path = self._key_to_path(key) + buf = freeze_basic_mapping(metadata) + if len(buf).bit_length() > 16: + raise ValueError('Metadata too large') + + # By renaming, we make sure that there are no + # conflicts between parallel reads, the last one wins + tmpname = '%s#%d-%d.tmp' % (path, os.getpid(), _thread.get_ident()) + + dest = ObjectW(tmpname) + os.rename(tmpname, path) + + dest.write(b's3ql_1\n') + dest.write(struct.pack(' 16: + raise ValueError('Metadata too large') + + path_src = self._key_to_path(src) + path_dest = self._key_to_path(dest) + + try: + src = open(path_src, 'rb') + except FileNotFoundError: + raise NoSuchObject(src) + + dest = None + try: + # By renaming, we make sure that there are no conflicts between + # parallel writes, the last one wins + tmpname = '%s#%d-%d.tmp' % (path_dest, os.getpid(), _thread.get_ident()) + dest = ObjectW(tmpname) + + if metadata is not None: + try: + _read_meta(src) + except ThawError: + raise CorruptedObjectError('Invalid metadata') + dest.write(b's3ql_1\n') + dest.write(struct.pack(' + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from ..logging import logging # Ensure use of custom logger class +import threading +from contextlib import contextmanager + +log = logging.getLogger(__name__) + +class BackendPool: + '''A pool of backends + + This class is threadsafe. All methods (except for internal methods + starting with underscore) may be called concurrently by different + threads. + ''' + + def __init__(self, factory): + '''Init pool + + *factory* should be a callable that provides new + connections. + ''' + + self.factory = factory + self.pool = [] + self.lock = threading.Lock() + + def pop_conn(self): + '''Pop connection from pool''' + + with self.lock: + if self.pool: + return self.pool.pop() + else: + return self.factory() + + def push_conn(self, conn): + '''Push connection back into pool''' + + conn.reset() + with self.lock: + self.pool.append(conn) + + def flush(self): + '''Close all backends in pool + + This method calls the `close` method on all backends + currently in the pool. + ''' + with self.lock: + while self.pool: + self.pool.pop().close() + + @contextmanager + def __call__(self, close=False): + '''Provide connection from pool (context manager) + + If *close* is True, the backend's close method is automatically called + (which frees any allocated resources, but may slow down reuse of the + backend object). + ''' + + conn = self.pop_conn() + try: + yield conn + finally: + if close: + conn.close() + self.push_conn(conn) + diff --git a/src/s3ql/backends/rackspace.py b/src/s3ql/backends/rackspace.py new file mode 100644 index 0000000..041a0f6 --- /dev/null +++ b/src/s3ql/backends/rackspace.py @@ -0,0 +1,43 @@ +''' +rackspace.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from ..logging import logging, QuietError # Ensure use of custom logger class +from . import swiftks +from ..inherit_docstrings import copy_ancestor_docstring +import re + +log = logging.getLogger(__name__) + +class Backend(swiftks.Backend): + """A backend to store data in Rackspace CloudFiles""" + + @copy_ancestor_docstring + def _parse_storage_url(self, storage_url, ssl_context): + + hit = re.match(r'^rackspace://' # Backend + r'([^/:]+)' # Region + r'/([^/]+)' # Bucketname + r'(?:/(.*))?$', # Prefix + storage_url) + if not hit: + raise QuietError('Invalid storage URL', exitcode=2) + + region = hit.group(1) + containername = hit.group(2) + prefix = hit.group(3) or '' + + if ssl_context: + port = 443 + else: + port = 80 + + self.hostname = 'auth.api.rackspacecloud.com' + self.port = port + self.container_name = containername + self.prefix = prefix + self.region = region diff --git a/src/s3ql/backends/s3.py b/src/s3ql/backends/s3.py new file mode 100644 index 0000000..a1e6d20 --- /dev/null +++ b/src/s3ql/backends/s3.py @@ -0,0 +1,163 @@ +''' +s3.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from ..logging import logging, QuietError # Ensure use of custom logger class +from . import s3c +from .s3c import get_S3Error +from .common import NoSuchObject, retry +from ..inherit_docstrings import copy_ancestor_docstring +from xml.sax.saxutils import escape as xml_escape +import re + +log = logging.getLogger(__name__) + +# Maximum number of keys that can be deleted at once +MAX_KEYS = 1000 + +# Pylint goes berserk with false positives +#pylint: disable=E1002,E1101 + +class Backend(s3c.Backend): + """A backend to store data in Amazon S3 + + This class uses standard HTTP connections to connect to S3. + + The backend guarantees get after create consistency, i.e. a newly created + object will be immediately retrievable. Additional consistency guarantees + may or may not be available and can be queried for with instance methods. + """ + + known_options = ((s3c.Backend.known_options | { 'sse', 'rrs', 'ia' }) + - {'dumb-copy', 'disable-expect100'}) + + def __init__(self, storage_url, login, password, options): + super().__init__(storage_url, login, password, options) + + @staticmethod + def _parse_storage_url(storage_url, ssl_context): + hit = re.match(r'^s3s?://([^/]+)(?:/(.*))?$', storage_url) + if not hit: + raise QuietError('Invalid storage URL', exitcode=2) + + bucket_name = hit.group(1) + + # http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/BucketRestrictions.html + if not re.match('^[a-z0-9][a-z0-9.-]{1,60}[a-z0-9]$', bucket_name): + raise QuietError('Invalid bucket name.', exitcode=2) + + # Dots in the bucket cause problems with SSL certificate validation, + # because server certificate is for *.s3.amazonaws.com (which does not + # match e.g. a.b.s3.amazonaws.com). However, when using path based + # bucket selection, AWS issues a redirect to the former + # location. Therefore, S3 buckets with dots in their names cannot be + # accessed securely. This can only be fixed by Amazon by allowing + # path-based bucket selection, or providing a valid SSL certificate for + # endpoints that also matches if the bucket name contains dots. + if '.' in bucket_name and ssl_context: + raise QuietError('Buckets with dots in the name cannot be accessed over SSL.\n' + 'This is purely Amazon\'s fault, see ' + 'https://forums.aws.amazon.com/thread.jspa?threadID=130560', + exitcode=2) + hostname = '%s.s3.amazonaws.com' % bucket_name + + prefix = hit.group(2) or '' + port = 443 if ssl_context else 80 + return (hostname, port, bucket_name, prefix) + + def __str__(self): + return 'Amazon S3 bucket %s, prefix %s' % (self.bucket_name, self.prefix) + + @copy_ancestor_docstring + def delete_multi(self, keys, force=False): + log.debug('started with %s', keys) + + while len(keys) > 0: + tmp = keys[:MAX_KEYS] + try: + self._delete_multi(tmp, force=force) + finally: + keys[:MAX_KEYS] = tmp + + def _set_storage_options(self, headers): + if 'sse' in self.options: + headers['x-amz-server-side-encryption'] = 'AES256' + + if 'ia' in self.options: + sc = 'STANDARD_IA' + elif 'rrs' in self.options: + sc = 'REDUCED_REDUNDANCY' + else: + sc = 'STANDARD' + headers['x-amz-storage-class'] = sc + + @copy_ancestor_docstring + def copy(self, src, dest, metadata=None): + extra_headers = {} + self._set_storage_options(extra_headers) + return super().copy(src, dest, metadata=metadata, + extra_headers=extra_headers) + + @copy_ancestor_docstring + def open_write(self, key, metadata=None, is_compressed=False): + extra_headers = {} + self._set_storage_options(extra_headers) + return super().open_write(key, metadata=metadata, is_compressed=is_compressed, + extra_headers=extra_headers) + + @retry + def _delete_multi(self, keys, force=False): + + body = [ '' ] + esc_prefix = xml_escape(self.prefix) + for key in keys: + body.append('%s%s' % (esc_prefix, xml_escape(key))) + body.append('') + body = '\n'.join(body).encode('utf-8') + headers = { 'content-type': 'text/xml; charset=utf-8' } + + resp = self._do_request('POST', '/', subres='delete', body=body, headers=headers) + try: + root = self._parse_xml_response(resp) + ns_p = self.xml_ns_prefix + + error_tags = root.findall(ns_p + 'Error') + if not error_tags: + # No errors occured, everything has been deleted + del keys[:] + return + + # Some errors occured, so we need to determine what has + # been deleted and what hasn't + offset = len(self.prefix) + for tag in root.findall(ns_p + 'Deleted'): + fullkey = tag.find(ns_p + 'Key').text + assert fullkey.startswith(self.prefix) + keys.remove(fullkey[offset:]) + + if log.isEnabledFor(logging.DEBUG): + for errtag in error_tags: + log.debug('Delete %s failed with %s', + errtag.findtext(ns_p + 'Key')[offset:], + errtag.findtext(ns_p + 'Code')) + + # If *force*, just modify the passed list and return without + # raising an exception, otherwise raise exception for the first error + if force: + return + + errcode = error_tags[0].findtext(ns_p + 'Code') + errmsg = error_tags[0].findtext(ns_p + 'Message') + errkey = error_tags[0].findtext(ns_p + 'Key')[offset:] + + if errcode == 'NoSuchKeyError': + raise NoSuchObject(errkey) + else: + raise get_S3Error(errcode, 'Error deleting %s: %s' % (errkey, errmsg)) + + except: + self.conn.discard() diff --git a/src/s3ql/backends/s3c.py b/src/s3ql/backends/s3c.py new file mode 100644 index 0000000..0a5a493 --- /dev/null +++ b/src/s3ql/backends/s3c.py @@ -0,0 +1,1037 @@ +''' +backends/s3c.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from ..logging import logging, QuietError # Ensure use of custom logger class +from .. import BUFSIZE +from .common import (AbstractBackend, NoSuchObject, retry, AuthorizationError, + AuthenticationError, DanglingStorageURLError, retry_generator, + get_proxy, get_ssl_context, CorruptedObjectError, + checksum_basic_mapping) +from ..inherit_docstrings import (copy_ancestor_docstring, prepend_ancestor_docstring, + ABCDocstMeta) +from io import BytesIO +from shutil import copyfileobj +from dugong import (HTTPConnection, is_temp_network_error, BodyFollowing, CaseInsensitiveDict, + UnsupportedResponse, ConnectionClosed) +from base64 import b64encode, b64decode +from email.utils import parsedate_tz, mktime_tz +from ast import literal_eval +from urllib.parse import urlsplit, quote, unquote +import defusedxml.cElementTree as ElementTree +from itertools import count +import hashlib +import os +import binascii +import hmac +import re +import tempfile +import time +import ssl +import urllib.parse + +C_DAY_NAMES = [ 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun' ] +C_MONTH_NAMES = [ 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec' ] + +XML_CONTENT_RE = re.compile(r'^(?:application|text)/xml(?:;|$)', re.IGNORECASE) + +log = logging.getLogger(__name__) + +class Backend(AbstractBackend, metaclass=ABCDocstMeta): + """A backend to stored data in some S3 compatible storage service. + + The backend guarantees only immediate get after create consistency. + """ + + xml_ns_prefix = '{http://s3.amazonaws.com/doc/2006-03-01/}' + hdr_prefix = 'x-amz-' + known_options = {'no-ssl', 'ssl-ca-path', 'tcp-timeout', + 'dumb-copy', 'disable-expect100'} + + def __init__(self, storage_url, login, password, options): + '''Initialize backend object + + *ssl_context* may be a `ssl.SSLContext` instance or *None*. + ''' + + super().__init__() + + if 'no-ssl' in options: + self.ssl_context = None + else: + self.ssl_context = get_ssl_context(options.get('ssl-ca-path', None)) + + (host, port, bucket_name, prefix) = self._parse_storage_url(storage_url, + self.ssl_context) + + self.options = options + self.bucket_name = bucket_name + self.prefix = prefix + self.hostname = host + self.port = port + self.proxy = get_proxy(self.ssl_context is not None) + self.conn = self._get_conn() + self.password = password + self.login = login + + @property + @copy_ancestor_docstring + def has_native_rename(self): + return False + + # NOTE: ! This function is also used by the swift backend ! + @copy_ancestor_docstring + def reset(self): + if (self.conn is not None and + (self.conn.response_pending() or self.conn._out_remaining)): + log.debug('Resetting state of http connection %d', id(self.conn)) + self.conn.disconnect() + + @staticmethod + def _parse_storage_url(storage_url, ssl_context): + '''Extract information from storage URL + + Return a tuple * (host, port, bucket_name, prefix) * . + ''' + + hit = re.match(r'^[a-zA-Z0-9]+://' # Backend + r'([^/:]+)' # Hostname + r'(?::([0-9]+))?' # Port + r'/([^/]+)' # Bucketname + r'(?:/(.*))?$', # Prefix + storage_url) + if not hit: + raise QuietError('Invalid storage URL', exitcode=2) + + hostname = hit.group(1) + if hit.group(2): + port = int(hit.group(2)) + elif ssl_context: + port = 443 + else: + port = 80 + bucketname = hit.group(3) + prefix = hit.group(4) or '' + + return (hostname, port, bucketname, prefix) + + def _get_conn(self): + '''Return connection to server''' + + conn = HTTPConnection(self.hostname, self.port, proxy=self.proxy, + ssl_context=self.ssl_context) + conn.timeout = int(self.options.get('tcp-timeout', 20)) + return conn + + @staticmethod + def _tag_xmlns_uri(elem): + '''Extract the XML namespace (xmlns) URI from an element''' + if elem.tag[0] == '{': + uri, ignore, tag = elem.tag[1:].partition("}") + else: + uri = None + return uri + + # This method is also used implicitly for the retry handling of + # `gs.Backend._get_access_token`. When modifying this method, do not forget + # to check if this makes it unsuitable for use by `_get_access_token` (in + # that case we will have to implement a custom retry logic there). + @copy_ancestor_docstring + def is_temp_failure(self, exc): #IGNORE:W0613 + if isinstance(exc, (InternalError, BadDigestError, IncompleteBodyError, + RequestTimeoutError, OperationAbortedError, + SlowDownError, ServiceUnavailableError)): + return True + + elif is_temp_network_error(exc): + return True + + # In doubt, we retry on 5xx (Server error). However, there are some + # codes where retry is definitely not desired. For 4xx (client error) we + # do not retry in general, but for 408 (Request Timeout) RFC 2616 + # specifies that the client may repeat the request without + # modifications. + elif (isinstance(exc, HTTPError) and + ((500 <= exc.status <= 599 + and exc.status not in (501,505,508,510,511,523)) + or exc.status == 408)): + return True + + # Temporary workaround for https://bitbucket.org/nikratio/s3ql/issues/87. + # We still need to find a proper string + elif (isinstance(exc, ssl.SSLError) and + str(exc).startswith('[SSL: BAD_WRITE_RETRY]')): + return True + + return False + + # NOTE: ! This function is also used by the swift backend. ! + def _dump_response(self, resp, body=None): + '''Return string representation of server response + + Only the beginning of the response body is read, so this is + mostly useful for debugging. + ''' + + if body is None: + try: + body = self.conn.read(2048) + if body: + self.conn.discard() + except UnsupportedResponse: + log.warning('Unsupported response, trying to retrieve data from raw socket!') + body = self.conn.read_raw(2048) + self.conn.close() + else: + body = body[:2048] + + return '%d %s\n%s\n\n%s' % (resp.status, resp.reason, + '\n'.join('%s: %s' % x for x in resp.headers.items()), + body.decode('utf-8', errors='backslashreplace')) + + # NOTE: ! This function is also used by the swift backend. ! + def _assert_empty_response(self, resp): + '''Assert that current response body is empty''' + + buf = self.conn.read(2048) + if not buf: + return # expected + + # Log the problem + self.conn.discard() + log.error('Unexpected server response. Expected nothing, got:\n' + '%d %s\n%s\n\n%s', resp.status, resp.reason, + '\n'.join('%s: %s' % x for x in resp.headers.items()), + buf) + raise RuntimeError('Unexpected server response') + + @retry + @copy_ancestor_docstring + def delete(self, key, force=False, is_retry=False): + log.debug('started with %s', key) + try: + resp = self._do_request('DELETE', '/%s%s' % (self.prefix, key)) + self._assert_empty_response(resp) + except NoSuchKeyError: + # Server may have deleted the object even though we did not + # receive the response. + if force or is_retry: + pass + else: + raise NoSuchObject(key) + + @retry_generator + @copy_ancestor_docstring + def list(self, prefix='', start_after=''): + log.debug('started with %s, %s', prefix, start_after) + + keys_remaining = True + marker = self.prefix + start_after + prefix = self.prefix + prefix + + while keys_remaining: + log.debug('requesting with marker=%s', marker) + + keys_remaining = None + resp = self._do_request('GET', '/', query_string={ 'prefix': prefix, + 'marker': marker, + 'max-keys': 1000 }) + + if not XML_CONTENT_RE.match(resp.headers['Content-Type']): + raise RuntimeError('unexpected content type: %s' % + resp.headers['Content-Type']) + + try: + itree = iter(ElementTree.iterparse(self.conn, events=("start", "end"))) + (event, root) = next(itree) + + root_xmlns_uri = self._tag_xmlns_uri(root) + if root_xmlns_uri is None: + root_xmlns_prefix = '' + else: + # Validate the XML namespace + root_xmlns_prefix = '{%s}' % (root_xmlns_uri, ) + if root_xmlns_prefix != self.xml_ns_prefix: + log.error('Unexpected server reply to list operation:\n%s', + self._dump_response(resp, body=None)) + raise RuntimeError('List response has %s as root tag, unknown namespace' % root.tag) + + for (event, el) in itree: + if event != 'end': + continue + + if el.tag == root_xmlns_prefix + 'IsTruncated': + keys_remaining = (el.text == 'true') + + elif el.tag == root_xmlns_prefix + 'Contents': + marker = el.findtext(root_xmlns_prefix + 'Key') + yield marker[len(self.prefix):] + root.clear() + + except Exception as exc: + if is_temp_network_error(exc): + # We probably can't use the connection anymore + self.conn.disconnect() + raise + + except GeneratorExit: + # Need to read rest of response + self.conn.discard() + break + + if keys_remaining is None: + raise RuntimeError('Could not parse body') + + @retry + @copy_ancestor_docstring + def lookup(self, key): + log.debug('started with %s', key) + + try: + resp = self._do_request('HEAD', '/%s%s' % (self.prefix, key)) + self._assert_empty_response(resp) + except HTTPError as exc: + if exc.status == 404: + raise NoSuchObject(key) + else: + raise + + return self._extractmeta(resp, key) + + @retry + @copy_ancestor_docstring + def get_size(self, key): + log.debug('started with %s', key) + + try: + resp = self._do_request('HEAD', '/%s%s' % (self.prefix, key)) + self._assert_empty_response(resp) + except HTTPError as exc: + if exc.status == 404: + raise NoSuchObject(key) + else: + raise + + try: + return int(resp.headers['Content-Length']) + except KeyError: + raise RuntimeError('HEAD request did not return Content-Length') + + + @retry + @copy_ancestor_docstring + def open_read(self, key): + try: + resp = self._do_request('GET', '/%s%s' % (self.prefix, key)) + except NoSuchKeyError: + raise NoSuchObject(key) + + try: + meta = self._extractmeta(resp, key) + except (BadDigestError, CorruptedObjectError): + # If there's less than 64 kb of data, read and throw + # away. Otherwise re-establish connection. + if resp.length is not None and resp.length < 64*1024: + self.conn.discard() + else: + self.conn.disconnect() + raise + + return ObjectR(key, resp, self, meta) + + @prepend_ancestor_docstring + def open_write(self, key, metadata=None, is_compressed=False, extra_headers=None): + """ + The returned object will buffer all data and only start the upload + when its `close` method is called. + """ + + log.debug('started with %s', key) + + headers = CaseInsensitiveDict() + if extra_headers is not None: + headers.update(extra_headers) + if metadata is None: + metadata = dict() + self._add_meta_headers(headers, metadata) + + return ObjectW(key, self, headers) + + # NOTE: ! This function is also used by the swift backend. ! + def _add_meta_headers(self, headers, metadata, chunksize=255): + + hdr_count = 0 + length = 0 + for key in metadata.keys(): + if not isinstance(key, str): + raise ValueError('dict keys must be str, not %s' % type(key)) + val = metadata[key] + if (not isinstance(val, (str, bytes, int, float, complex, bool)) + and val is not None): + raise ValueError('value for key %s (%s) is not elementary' % (key, val)) + + if isinstance(val, (bytes, bytearray)): + val = b64encode(val) + + buf = ('%s: %s,' % (repr(key), repr(val))) + buf = quote(buf, safe='!@#$^*()=+/?-_\'"><\\| `.,;:~') + if len(buf) < chunksize: + headers['%smeta-%03d' % (self.hdr_prefix, hdr_count)] = buf + hdr_count += 1 + length += 4 + len(buf) + else: + i = 0 + while i*chunksize < len(buf): + k = '%smeta-%03d' % (self.hdr_prefix, hdr_count) + v = buf[i*chunksize:(i+1)*chunksize] + headers[k] = v + i += 1 + hdr_count += 1 + length += 4 + len(buf) + + if length > 2048: + raise ValueError('Metadata too large') + + assert hdr_count <= 999 + md5 = b64encode(checksum_basic_mapping(metadata)).decode('ascii') + headers[self.hdr_prefix + 'meta-format'] = 'raw2' + headers[self.hdr_prefix + 'meta-md5'] = md5 + + @retry + @copy_ancestor_docstring + def copy(self, src, dest, metadata=None, extra_headers=None): + log.debug('started with %s, %s', src, dest) + + headers = CaseInsensitiveDict() + if extra_headers is not None: + headers.update(extra_headers) + headers[self.hdr_prefix + 'copy-source'] = \ + urllib.parse.quote('/%s/%s%s' % (self.bucket_name, self.prefix, src)) + + if metadata is None: + headers[self.hdr_prefix + 'metadata-directive'] = 'COPY' + else: + headers[self.hdr_prefix + 'metadata-directive'] = 'REPLACE' + self._add_meta_headers(headers, metadata) + + try: + resp = self._do_request('PUT', '/%s%s' % (self.prefix, dest), headers=headers) + except NoSuchKeyError: + raise NoSuchObject(src) + + # When copying, S3 may return error despite a 200 OK status + # http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html + # https://doc.s3.amazonaws.com/proposals/copy.html + if self.options.get('dumb-copy', False): + self.conn.discard() + return + body = self.conn.readall() + root = self._parse_xml_response(resp, body) + + # Some S3 implemenentations do not have a namespace on + # CopyObjectResult. + if root.tag in [self.xml_ns_prefix + 'CopyObjectResult', 'CopyObjectResult']: + return + elif root.tag in [self.xml_ns_prefix + 'Error', 'Error']: + raise get_S3Error(root.findtext('Code'), root.findtext('Message'), + resp.headers) + else: + log.error('Unexpected server reply to copy operation:\n%s', + self._dump_response(resp, body)) + raise RuntimeError('Copy response has %s as root tag' % root.tag) + + @copy_ancestor_docstring + def update_meta(self, key, metadata): + self.copy(key, key, metadata) + + def _do_request(self, method, path, subres=None, query_string=None, + headers=None, body=None): + '''Send request, read and return response object''' + + log.debug('started with %s %s?%s, qs=%s', method, path, subres, query_string) + + if headers is None: + headers = CaseInsensitiveDict() + + if isinstance(body, (bytes, bytearray, memoryview)): + headers['Content-MD5'] = md5sum_b64(body) + + redirect_count = 0 + this_method = method + while True: + resp = self._send_request(this_method, path, headers=headers, subres=subres, + query_string=query_string, body=body) + + if (resp.status < 300 or resp.status > 399): + break + + # Assume redirect + redirect_count += 1 + if redirect_count > 10: + raise RuntimeError('Too many chained redirections') + + # First try location header... + new_url = resp.headers['Location'] + if new_url: + # Discard body + self.conn.discard() + + # Pylint can't infer SplitResult Types + #pylint: disable=E1103 + o = urlsplit(new_url) + if o.scheme: + if self.ssl_context and o.scheme != 'https': + raise RuntimeError('Redirect to non-https URL') + elif not self.ssl_context and o.scheme != 'http': + raise RuntimeError('Redirect to non-http URL') + if o.hostname != self.hostname or o.port != self.port: + self.hostname = o.hostname + self.port = o.port + self.conn.disconnect() + self.conn = self._get_conn() + else: + raise RuntimeError('Redirect to different path on same host') + + # ..but endpoint may also be hidden in message body. + # If we have done a HEAD request, we have to change to GET + # to actually retrieve the body. + elif resp.method == 'HEAD': + log.debug('Switching from HEAD to GET to read redirect body') + this_method = 'GET' + + # Try to read new URL from request body + else: + tree = self._parse_xml_response(resp) + new_url = tree.findtext('Endpoint') + + if not new_url: + raise get_S3Error(tree.findtext('Code'), tree.findtext('Message'), + resp.headers) + + self.hostname = new_url + self.conn.disconnect() + self.conn = self._get_conn() + + # Update method + this_method = method + + log.info('_do_request(): redirected to %s', self.conn.hostname) + + if body and not isinstance(body, (bytes, bytearray, memoryview)): + body.seek(0) + + # At the end, the request should have gone out with the right + # method + if this_method != method: + raise RuntimeError('Dazed and confused - HEAD fails but GET works?') + + # Success + if resp.status >= 200 and resp.status <= 299: + return resp + + # Error + self._parse_error_response(resp) + + def _parse_error_response(self, resp): + '''Handle error response from server + + Try to raise most-specific exception. + ''' + + # Note that even though the final server backend may guarantee to always + # deliver an XML document body with a detailed error message, we may + # also get errors from intermediate proxies. + content_type = resp.headers['Content-Type'] + + # If method == HEAD, server must not return response body + # even in case of errors + if resp.method.upper() == 'HEAD': + assert self.conn.read(1) == b'' + raise HTTPError(resp.status, resp.reason, resp.headers) + + # If not XML, do the best we can + if not XML_CONTENT_RE.match(content_type) or resp.length == 0: + self.conn.discard() + raise HTTPError(resp.status, resp.reason, resp.headers) + + # We don't stream the data into the parser because we want + # to be able to dump a copy if the parsing fails. + body = self.conn.readall() + try: + tree = ElementTree.parse(BytesIO(body)).getroot() + except: + log.error('Unable to parse server response as XML:\n%s', + self._dump_response(resp, body)) + raise + + raise get_S3Error(tree.findtext('Code'), tree.findtext('Message'), resp.headers) + + def _parse_xml_response(self, resp, body=None): + '''Return element tree for XML response''' + + content_type = resp.headers['Content-Type'] + + # AWS S3 sometimes "forgets" to send a Content-Type + # when responding to a multiple delete request. + # https://forums.aws.amazon.com/thread.jspa?threadID=134372 + if content_type is None: + log.error('Server did not provide Content-Type, assuming XML') + elif not XML_CONTENT_RE.match(content_type): + log.error('Unexpected server reply: expected XML, got:\n%s', + self._dump_response(resp)) + raise RuntimeError('Unexpected server response') + + # We don't stream the data into the parser because we want + # to be able to dump a copy if the parsing fails. + if body is None: + body = self.conn.readall() + try: + tree = ElementTree.parse(BytesIO(body)).getroot() + except: + log.error('Unable to parse server response as XML:\n%s', + self._dump_response(resp, body)) + raise + + return tree + + # NOTE: ! This function is also used by the swift backend. ! + @prepend_ancestor_docstring + def clear(self): + """ + This method may not be able to see (and therefore also not delete) + recently uploaded objects. + """ + + # We have to cache keys, because otherwise we can't use the + # http connection to delete keys. + for (no, s3key) in enumerate(list(self)): + if no != 0 and no % 1000 == 0: + log.info('clear(): deleted %d objects so far..', no) + + log.debug('started with %s', s3key) + + # Ignore missing objects when clearing bucket + self.delete(s3key, True) + + def __str__(self): + return 's3c://%s/%s/%s' % (self.hostname, self.bucket_name, self.prefix) + + def _authorize_request(self, method, path, headers, subres): + '''Add authorization information to *headers*''' + + # See http://docs.amazonwebservices.com/AmazonS3/latest/dev/RESTAuthentication.html + + # Date, can't use strftime because it's locale dependent + now = time.gmtime() + headers['Date'] = ('%s, %02d %s %04d %02d:%02d:%02d GMT' + % (C_DAY_NAMES[now.tm_wday], + now.tm_mday, + C_MONTH_NAMES[now.tm_mon - 1], + now.tm_year, now.tm_hour, + now.tm_min, now.tm_sec)) + + auth_strs = [method, '\n'] + + for hdr in ('Content-MD5', 'Content-Type', 'Date'): + if hdr in headers: + auth_strs.append(headers[hdr]) + auth_strs.append('\n') + + for hdr in sorted(x for x in headers if x.lower().startswith('x-amz-')): + val = ' '.join(re.split(r'\s*\n\s*', headers[hdr].strip())) + auth_strs.append('%s:%s\n' % (hdr, val)) + + # Always include bucket name in path for signing + sign_path = urllib.parse.quote('/%s%s' % (self.bucket_name, path)) + auth_strs.append(sign_path) + if subres: + auth_strs.append('?%s' % subres) + + # False positive, hashlib *does* have sha1 member + #pylint: disable=E1101 + auth_str = ''.join(auth_strs).encode() + signature = b64encode(hmac.new(self.password.encode(), auth_str, + hashlib.sha1).digest()).decode() + + headers['Authorization'] = 'AWS %s:%s' % (self.login, signature) + + def _send_request(self, method, path, headers, subres=None, query_string=None, body=None): + '''Add authentication and send request + + Returns the response object. + ''' + + if not isinstance(headers, CaseInsensitiveDict): + headers = CaseInsensitiveDict(headers) + + self._authorize_request(method, path, headers, subres) + + # Construct full path + if not self.hostname.startswith(self.bucket_name): + path = '/%s%s' % (self.bucket_name, path) + path = urllib.parse.quote(path) + if query_string: + s = urllib.parse.urlencode(query_string, doseq=True) + if subres: + path += '?%s&%s' % (subres, s) + else: + path += '?%s' % s + elif subres: + path += '?%s' % subres + + # We can probably remove the assertions at some point and + # call self.conn.read_response() directly + def read_response(): + resp = self.conn.read_response() + assert resp.method == method + assert resp.path == path + return resp + + use_expect_100c = not self.options.get('disable-expect100', False) + try: + log.debug('sending %s %s', method, path) + if body is None or isinstance(body, (bytes, bytearray, memoryview)): + self.conn.send_request(method, path, body=body, headers=headers) + else: + body_len = os.fstat(body.fileno()).st_size + self.conn.send_request(method, path, expect100=use_expect_100c, + headers=headers, body=BodyFollowing(body_len)) + + if use_expect_100c: + resp = read_response() + if resp.status != 100: # Error + return resp + + try: + copyfileobj(body, self.conn, BUFSIZE) + except ConnectionClosed: + # Server closed connection while we were writing body data - + # but we may still be able to read an error response + try: + resp = read_response() + except ConnectionClosed: # No server response available + pass + else: + if resp.status >= 400: # Got error response + return resp + log.warning('Server broke connection during upload, but signaled ' + '%d %s', resp.status, resp.reason) + + # Re-raise first ConnectionClosed exception + raise + + return read_response() + + except Exception as exc: + if is_temp_network_error(exc): + # We probably can't use the connection anymore + self.conn.disconnect() + raise + + @copy_ancestor_docstring + def close(self): + self.conn.disconnect() + + # NOTE: ! This function is also used by the swift backend ! + def _extractmeta(self, resp, obj_key): + '''Extract metadata from HTTP response object''' + + format_ = resp.headers.get('%smeta-format' % self.hdr_prefix, 'raw') + if format_ != 'raw2': # Current + raise CorruptedObjectError('Invalid metadata format: %s' % format_) + + parts = [] + for i in count(): + # Headers is an email.message object, so indexing it + # would also give None instead of KeyError + part = resp.headers.get('%smeta-%03d' % (self.hdr_prefix, i), None) + if part is None: + break + parts.append(part) + buf = unquote(''.join(parts)) + meta = literal_eval('{ %s }' % buf) + + # Decode bytes values + for (k,v) in meta.items(): + if not isinstance(v, bytes): + continue + try: + meta[k] = b64decode(v) + except binascii.Error: + # This should trigger a MD5 mismatch below + meta[k] = None + + # Check MD5. There is a case to be made for treating a mismatch as a + # `CorruptedObjectError` rather than a `BadDigestError`, because the MD5 + # sum is not calculated on-the-fly by the server but stored with the + # object, and therefore does not actually verify what the server has + # sent over the wire. However, it seems more likely for the data to get + # accidentally corrupted in transit than to get accidentally corrupted + # on the server (which hopefully checksums its storage devices). + md5 = b64encode(checksum_basic_mapping(meta)).decode('ascii') + if md5 != resp.headers.get('%smeta-md5' % self.hdr_prefix, None): + log.warning('MD5 mismatch in metadata for %s', obj_key) + raise BadDigestError('BadDigest', 'Meta MD5 for %s does not match' % obj_key) + + return meta + +class ObjectR(object): + '''An S3 object open for reading''' + + # NOTE: This class is used as a base class for the swift backend, + # so changes here should be checked for their effects on other + # backends. + + def __init__(self, key, resp, backend, metadata=None): + self.key = key + self.resp = resp + self.closed = False + self.md5_checked = False + self.backend = backend + self.metadata = metadata + + # False positive, hashlib *does* have md5 member + #pylint: disable=E1101 + self.md5 = hashlib.md5() + + def read(self, size=None): + '''Read up to *size* bytes of object data + + For integrity checking to work, this method has to be called until + it returns an empty string, indicating that all data has been read + (and verified). + ''' + + if size == 0: + return b'' + + # This may raise an exception, in which case we probably can't + # re-use the connection. However, we rely on the caller + # to still close the file-like object, so that we can do + # cleanup in close(). + buf = self.backend.conn.read(size) + self.md5.update(buf) + + # Check MD5 on EOF + # (size == None implies EOF) + if (not buf or size is None) and not self.md5_checked: + etag = self.resp.headers['ETag'].strip('"') + self.md5_checked = True + if etag != self.md5.hexdigest(): + log.warning('MD5 mismatch for %s: %s vs %s', + self.key, etag, self.md5.hexdigest()) + raise BadDigestError('BadDigest', + 'ETag header does not agree with calculated MD5') + return buf + + def __enter__(self): + return self + + def __exit__(self, *a): + self.close() + return False + + def close(self, checksum_warning=True): + '''Close object + + If *checksum_warning* is true, this will generate a warning message if + the object has not been fully read (because in that case the MD5 + checksum cannot be checked). + ''' + + if self.closed: + return + self.closed = True + + # If we have not read all the data, close the entire + # connection (otherwise we loose synchronization) + if not self.md5_checked: + if checksum_warning: + log.warning("Object closed prematurely, can't check MD5, and have to " + "reset connection") + self.backend.conn.disconnect() + + +class ObjectW(object): + '''An S3 object open for writing + + All data is first cached in memory, upload only starts when + the close() method is called. + ''' + + # NOTE: This class is used as a base class for the swift backend, + # so changes here should be checked for their effects on other + # backends. + + def __init__(self, key, backend, headers): + self.key = key + self.backend = backend + self.headers = headers + self.closed = False + self.obj_size = 0 + + # According to http://docs.python.org/3/library/functions.html#open + # the buffer size is typically ~8 kB. We process data in much + # larger chunks, so buffering would only hurt performance. + self.fh = tempfile.TemporaryFile(buffering=0) + + # False positive, hashlib *does* have md5 member + #pylint: disable=E1101 + self.md5 = hashlib.md5() + + def write(self, buf): + '''Write object data''' + + self.fh.write(buf) + self.md5.update(buf) + self.obj_size += len(buf) + + def is_temp_failure(self, exc): + return self.backend.is_temp_failure(exc) + + @retry + def close(self): + '''Close object and upload data''' + + # Access to protected member ok + #pylint: disable=W0212 + + log.debug('started with %s', self.key) + + if self.closed: + # still call fh.close, may have generated an error before + self.fh.close() + return + + self.fh.seek(0) + self.headers['Content-Type'] = 'application/octet-stream' + resp = self.backend._do_request('PUT', '/%s%s' % (self.backend.prefix, self.key), + headers=self.headers, body=self.fh) + etag = resp.headers['ETag'].strip('"') + self.backend._assert_empty_response(resp) + + if etag != self.md5.hexdigest(): + # delete may fail, but we don't want to loose the BadDigest exception + try: + self.backend.delete(self.key) + finally: + raise BadDigestError('BadDigest', 'MD5 mismatch for %s (received: %s, sent: %s)' % + (self.key, etag, self.md5.hexdigest())) + + self.closed = True + self.fh.close() + + def __enter__(self): + return self + + def __exit__(self, *a): + self.close() + return False + + def get_obj_size(self): + if not self.closed: + raise RuntimeError('Object must be closed first.') + return self.obj_size + + +def get_S3Error(code, msg, headers=None): + '''Instantiate most specific S3Error subclass''' + + # Special case + # http://code.google.com/p/s3ql/issues/detail?id=369 + if code == 'Timeout': + code = 'RequestTimeout' + + if code.endswith('Error'): + name = code + else: + name = code + 'Error' + class_ = globals().get(name, S3Error) + + if not issubclass(class_, S3Error): + return S3Error(code, msg, headers) + + return class_(code, msg, headers) + +def md5sum_b64(buf): + '''Return base64 encoded MD5 sum''' + + return b64encode(hashlib.md5(buf).digest()).decode('ascii') + +def _parse_retry_after(header): + '''Parse headers for Retry-After value''' + + hit = re.match(r'^\s*([0-9]+)\s*$', header) + if hit: + val = int(header) + else: + date = parsedate_tz(header) + if date is None: + log.warning('Unable to parse retry-after value: %s', header) + return None + val = mktime_tz(*date) - time.time() + + if val > 300 or val < 0: + log.warning('Ignoring retry-after value of %.3f s, using 1 s instead', val) + val = 1 + + return val + + +class HTTPError(Exception): + ''' + Represents an HTTP error returned by S3. + ''' + + def __init__(self, status, msg, headers=None): + super().__init__() + self.status = status + self.msg = msg + self.headers = headers + + if headers and 'Retry-After' in headers: + self.retry_after = _parse_retry_after(headers['Retry-After']) + else: + self.retry_after = None + + def __str__(self): + return '%d %s' % (self.status, self.msg) + + +class S3Error(Exception): + ''' + Represents an error returned by S3. For possible codes, see + http://docs.amazonwebservices.com/AmazonS3/latest/API/ErrorResponses.html + ''' + + def __init__(self, code, msg, headers=None): + super().__init__(msg) + self.code = code + self.msg = msg + + if headers and 'Retry-After' in headers: + self.retry_after = _parse_retry_after(headers['Retry-After']) + else: + self.retry_after = None + + def __str__(self): + return '%s: %s' % (self.code, self.msg) + +class NoSuchKeyError(S3Error): pass +class AccessDeniedError(S3Error, AuthorizationError): pass +class BadDigestError(S3Error): pass +class IncompleteBodyError(S3Error): pass +class InternalError(S3Error): pass +class InvalidAccessKeyIdError(S3Error, AuthenticationError): pass +class InvalidSecurityError(S3Error, AuthenticationError): pass +class SignatureDoesNotMatchError(S3Error, AuthenticationError): pass +class OperationAbortedError(S3Error): pass +class RequestTimeoutError(S3Error): pass +class SlowDownError(S3Error): pass +class ServiceUnavailableError(S3Error): pass +class RequestTimeTooSkewedError(S3Error): pass +class NoSuchBucketError(S3Error, DanglingStorageURLError): pass diff --git a/src/s3ql/backends/swift.py b/src/s3ql/backends/swift.py new file mode 100644 index 0000000..8365336 --- /dev/null +++ b/src/s3ql/backends/swift.py @@ -0,0 +1,527 @@ +''' +swift.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from ..logging import logging, QuietError # Ensure use of custom logger class +from .. import BUFSIZE +from .common import (AbstractBackend, NoSuchObject, retry, AuthorizationError, + DanglingStorageURLError, retry_generator, get_proxy, + get_ssl_context) +from .s3c import HTTPError, ObjectR, ObjectW, md5sum_b64, BadDigestError +from . import s3c +from ..inherit_docstrings import (copy_ancestor_docstring, prepend_ancestor_docstring, + ABCDocstMeta) +from dugong import (HTTPConnection, BodyFollowing, is_temp_network_error, CaseInsensitiveDict, + ConnectionClosed) +from urllib.parse import urlsplit +import json +import shutil +import re +import os +import urllib.parse +import ssl + +log = logging.getLogger(__name__) + +#: Suffix to use when creating temporary objects +TEMP_SUFFIX = '_tmp$oentuhuo23986konteuh1062$' + +class Backend(AbstractBackend, metaclass=ABCDocstMeta): + """A backend to store data in OpenStack Swift + + The backend guarantees get after create consistency, i.e. a newly created + object will be immediately retrievable. + """ + + hdr_prefix = 'X-Object-' + known_options = {'no-ssl', 'ssl-ca-path', 'tcp-timeout', 'disable-expect100'} + + _add_meta_headers = s3c.Backend._add_meta_headers + _extractmeta = s3c.Backend._extractmeta + _assert_empty_response = s3c.Backend._assert_empty_response + _dump_response = s3c.Backend._dump_response + clear = s3c.Backend.clear + reset = s3c.Backend.reset + + def __init__(self, storage_url, login, password, options): + super().__init__() + self.options = options + self.hostname = None + self.port = None + self.container_name = None + self.prefix = None + self.auth_token = None + self.auth_prefix = None + self.conn = None + self.password = password + self.login = login + + # We may need the context even if no-ssl has been specified, + # because no-ssl applies only to the authentication URL. + self.ssl_context = get_ssl_context(options.get('ssl-ca-path', None)) + + self._parse_storage_url(storage_url, self.ssl_context) + self.proxy = get_proxy(self.ssl_context is not None) + self._container_exists() + + def __str__(self): + return 'swift container %s, prefix %s' % (self.container_name, self.prefix) + + @property + @copy_ancestor_docstring + def has_native_rename(self): + return False + + @retry + def _container_exists(self): + '''Make sure that the container exists''' + + try: + self._do_request('GET', '/', query_string={'limit': 1 }) + self.conn.discard() + except HTTPError as exc: + if exc.status == 404: + raise DanglingStorageURLError(self.container_name) + raise + + def _parse_storage_url(self, storage_url, ssl_context): + '''Init instance variables from storage url''' + + hit = re.match(r'^[a-zA-Z0-9]+://' # Backend + r'([^/:]+)' # Hostname + r'(?::([0-9]+))?' # Port + r'/([^/]+)' # Bucketname + r'(?:/(.*))?$', # Prefix + storage_url) + if not hit: + raise QuietError('Invalid storage URL', exitcode=2) + + hostname = hit.group(1) + if hit.group(2): + port = int(hit.group(2)) + elif ssl_context: + port = 443 + else: + port = 80 + containername = hit.group(3) + prefix = hit.group(4) or '' + + self.hostname = hostname + self.port = port + self.container_name = containername + self.prefix = prefix + + @copy_ancestor_docstring + def is_temp_failure(self, exc): #IGNORE:W0613 + if isinstance(exc, AuthenticationExpired): + return True + + # In doubt, we retry on 5xx (Server error). However, there are some + # codes where retry is definitely not desired. For 4xx (client error) we + # do not retry in general, but for 408 (Request Timeout) RFC 2616 + # specifies that the client may repeat the request without + # modifications. + elif (isinstance(exc, HTTPError) and + ((500 <= exc.status <= 599 + and exc.status not in (501,505,508,510,511,523)) + or exc.status == 408)): + return True + + elif is_temp_network_error(exc): + return True + + # Temporary workaround for https://bitbucket.org/nikratio/s3ql/issues/87. + # We still need to find a proper string + elif (isinstance(exc, ssl.SSLError) and + str(exc).startswith('[SSL: BAD_WRITE_RETRY]')): + return True + + return False + + def _get_conn(self): + '''Obtain connection to server and authentication token''' + + log.debug('started') + + if 'no-ssl' in self.options: + ssl_context = None + else: + ssl_context = self.ssl_context + + headers = CaseInsensitiveDict() + headers['X-Auth-User'] = self.login + headers['X-Auth-Key'] = self.password + + with HTTPConnection(self.hostname, self.port, proxy=self.proxy, + ssl_context=ssl_context) as conn: + conn.timeout = int(self.options.get('tcp-timeout', 20)) + + for auth_path in ('/v1.0', '/auth/v1.0'): + log.debug('GET %s', auth_path) + conn.send_request('GET', auth_path, headers=headers) + resp = conn.read_response() + + if resp.status in (404, 412): + log.debug('auth to %s failed, trying next path', auth_path) + conn.discard() + continue + + elif resp.status == 401: + raise AuthorizationError(resp.reason) + + elif resp.status > 299 or resp.status < 200: + raise HTTPError(resp.status, resp.reason, resp.headers) + + # Pylint can't infer SplitResult Types + #pylint: disable=E1103 + self.auth_token = resp.headers['X-Auth-Token'] + o = urlsplit(resp.headers['X-Storage-Url']) + self.auth_prefix = urllib.parse.unquote(o.path) + if o.scheme == 'https': + ssl_context = self.ssl_context + elif o.scheme == 'http': + ssl_context = None + else: + # fall through to scheme used for authentication + pass + + conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy, + ssl_context=ssl_context) + conn.timeout = int(self.options.get('tcp-timeout', 20)) + return conn + + raise RuntimeError('No valid authentication path found') + + def _do_request(self, method, path, subres=None, query_string=None, + headers=None, body=None): + '''Send request, read and return response object + + This method modifies the *headers* dictionary. + ''' + + log.debug('started with %r, %r, %r, %r, %r, %r', + method, path, subres, query_string, headers, body) + + if headers is None: + headers = CaseInsensitiveDict() + + if isinstance(body, (bytes, bytearray, memoryview)): + headers['Content-MD5'] = md5sum_b64(body) + + if self.conn is None: + log.debug('no active connection, calling _get_conn()') + self.conn = self._get_conn() + + # Construct full path + path = urllib.parse.quote('%s/%s%s' % (self.auth_prefix, self.container_name, path)) + if query_string: + s = urllib.parse.urlencode(query_string, doseq=True) + if subres: + path += '?%s&%s' % (subres, s) + else: + path += '?%s' % s + elif subres: + path += '?%s' % subres + + headers['X-Auth-Token'] = self.auth_token + try: + resp = self._do_request_inner(method, path, body=body, headers=headers) + except Exception as exc: + if is_temp_network_error(exc): + # We probably can't use the connection anymore + self.conn.disconnect() + raise + + # Success + if resp.status >= 200 and resp.status <= 299: + return resp + + # Expired auth token + if resp.status == 401: + log.info('OpenStack auth token seems to have expired, requesting new one.') + self.conn.disconnect() + # Force constructing a new connection with a new token, otherwise + # the connection will be reestablished with the same token. + self.conn = None + raise AuthenticationExpired(resp.reason) + + # If method == HEAD, server must not return response body + # even in case of errors + self.conn.discard() + if method.upper() == 'HEAD': + raise HTTPError(resp.status, resp.reason, resp.headers) + else: + raise HTTPError(resp.status, resp.reason, resp.headers) + + # Including this code directly in _do_request would be very messy since + # we can't `return` the response early, thus the separate method + def _do_request_inner(self, method, path, body, headers): + '''The guts of the _do_request method''' + + log.debug('started with %s %s', method, path) + use_expect_100c = not self.options.get('disable-expect100', False) + + if body is None or isinstance(body, (bytes, bytearray, memoryview)): + self.conn.send_request(method, path, body=body, headers=headers) + return self.conn.read_response() + + body_len = os.fstat(body.fileno()).st_size + self.conn.send_request(method, path, expect100=use_expect_100c, + headers=headers, body=BodyFollowing(body_len)) + + if use_expect_100c: + log.debug('waiting for 100-continue') + resp = self.conn.read_response() + if resp.status != 100: + return resp + + log.debug('writing body data') + try: + shutil.copyfileobj(body, self.conn, BUFSIZE) + except ConnectionClosed: + log.debug('interrupted write, server closed connection') + # Server closed connection while we were writing body data - + # but we may still be able to read an error response + try: + resp = self.conn.read_response() + except ConnectionClosed: # No server response available + log.debug('no response available in buffer') + pass + else: + if resp.status >= 400: # error response + return resp + log.warning('Server broke connection during upload, but signaled ' + '%d %s', resp.status, resp.reason) + + # Re-raise original error + raise + + return self.conn.read_response() + + @retry + @copy_ancestor_docstring + def lookup(self, key): + log.debug('started with %s', key) + if key.endswith(TEMP_SUFFIX): + raise ValueError('Keys must not end with %s' % TEMP_SUFFIX) + + try: + resp = self._do_request('HEAD', '/%s%s' % (self.prefix, key)) + self._assert_empty_response(resp) + except HTTPError as exc: + if exc.status == 404: + raise NoSuchObject(key) + else: + raise + + return self._extractmeta(resp, key) + + @retry + @copy_ancestor_docstring + def get_size(self, key): + if key.endswith(TEMP_SUFFIX): + raise ValueError('Keys must not end with %s' % TEMP_SUFFIX) + log.debug('started with %s', key) + + try: + resp = self._do_request('HEAD', '/%s%s' % (self.prefix, key)) + self._assert_empty_response(resp) + except HTTPError as exc: + if exc.status == 404: + raise NoSuchObject(key) + else: + raise + + try: + return int(resp.headers['Content-Length']) + except KeyError: + raise RuntimeError('HEAD request did not return Content-Length') + + @retry + @copy_ancestor_docstring + def open_read(self, key): + if key.endswith(TEMP_SUFFIX): + raise ValueError('Keys must not end with %s' % TEMP_SUFFIX) + try: + resp = self._do_request('GET', '/%s%s' % (self.prefix, key)) + except HTTPError as exc: + if exc.status == 404: + raise NoSuchObject(key) + raise + + try: + meta = self._extractmeta(resp, key) + except BadDigestError: + # If there's less than 64 kb of data, read and throw + # away. Otherwise re-establish connection. + if resp.length is not None and resp.length < 64*1024: + self.conn.discard() + else: + self.conn.disconnect() + raise + + return ObjectR(key, resp, self, meta) + + @prepend_ancestor_docstring + def open_write(self, key, metadata=None, is_compressed=False): + """ + The returned object will buffer all data and only start the upload + when its `close` method is called. + """ + log.debug('started with %s', key) + + if key.endswith(TEMP_SUFFIX): + raise ValueError('Keys must not end with %s' % TEMP_SUFFIX) + + headers = CaseInsensitiveDict() + if metadata is None: + metadata = dict() + self._add_meta_headers(headers, metadata) + + return ObjectW(key, self, headers) + + @retry + @copy_ancestor_docstring + def delete(self, key, force=False, is_retry=False): + if key.endswith(TEMP_SUFFIX): + raise ValueError('Keys must not end with %s' % TEMP_SUFFIX) + log.debug('started with %s', key) + try: + resp = self._do_request('DELETE', '/%s%s' % (self.prefix, key)) + self._assert_empty_response(resp) + except HTTPError as exc: + # Server may have deleted the object even though we did not + # receive the response. + if exc.status == 404 and not (force or is_retry): + raise NoSuchObject(key) + elif exc.status != 404: + raise + + # We cannot wrap the entire copy() method into a retry() decorator, because + # copy() issues multiple requests. If the server happens to regularly close + # the connection after a request (even though it was processed correctly), + # we'd never make any progress if we always restart from the first request. + # We experimented with adding a retry(fn, args, kwargs) function to wrap + # individual calls, but this doesn't really improve the code because we + # typically also have to wrap e.g. a discard() or assert_empty() call, so we + # end up with having to create an additional function anyway. Having + # anonymous blocks in Python would be very nice here. + @retry + def _copy_helper(self, method, path, headers): + self._do_request(method, path, headers=headers) + self.conn.discard() + + @copy_ancestor_docstring + def copy(self, src, dest, metadata=None): + log.debug('started with %s, %s', src, dest) + if dest.endswith(TEMP_SUFFIX) or src.endswith(TEMP_SUFFIX): + raise ValueError('Keys must not end with %s' % TEMP_SUFFIX) + + headers = CaseInsensitiveDict() + headers['X-Copy-From'] = '/%s/%s%s' % (self.container_name, self.prefix, src) + + if metadata is not None: + # We can't do a direct copy, because during copy we can only update the + # metadata, but not replace it. Therefore, we have to make a full copy + # followed by a separate request to replace the metadata. To avoid an + # inconsistent intermediate state, we use a temporary object. + final_dest = dest + dest = final_dest + TEMP_SUFFIX + headers['X-Delete-After'] = '600' + + try: + self._copy_helper('PUT', '/%s%s' % (self.prefix, dest), headers) + except HTTPError as exc: + if exc.status == 404: + raise NoSuchObject(src) + raise + + if metadata is None: + return + + # Update metadata + headers = CaseInsensitiveDict() + self._add_meta_headers(headers, metadata) + self._copy_helper('POST', '/%s%s' % (self.prefix, dest), headers) + + # Rename object + headers = CaseInsensitiveDict() + headers['X-Copy-From'] = '/%s/%s%s' % (self.container_name, self.prefix, dest) + self._copy_helper('PUT', '/%s%s' % (self.prefix, final_dest), headers) + + @retry + @copy_ancestor_docstring + def update_meta(self, key, metadata): + log.debug('started with %s', key) + headers = CaseInsensitiveDict() + self._add_meta_headers(headers, metadata) + self._do_request('POST', '/%s%s' % (self.prefix, key), headers=headers) + self.conn.discard() + + @retry_generator + @copy_ancestor_docstring + def list(self, prefix='', start_after='', batch_size=5000): + log.debug('started with %s, %s', prefix, start_after) + + keys_remaining = True + marker = self.prefix + start_after + prefix = self.prefix + prefix + + while keys_remaining: + log.debug('requesting with marker=%s', marker) + + try: + resp = self._do_request('GET', '/', query_string={'prefix': prefix, + 'format': 'json', + 'marker': marker, + 'limit': batch_size }) + except HTTPError as exc: + if exc.status == 404: + raise DanglingStorageURLError(self.container_name) + raise + + if resp.status == 204: + return + + hit = re.match('application/json; charset="?(.+?)"?$', + resp.headers['content-type']) + if not hit: + log.error('Unexpected server response. Expected json, got:\n%s', + self._dump_response(resp)) + raise RuntimeError('Unexpected server reply') + + strip = len(self.prefix) + count = 0 + try: + # JSON does not have a streaming API, so we just read + # the entire response in memory. + for dataset in json.loads(self.conn.read().decode(hit.group(1))): + count += 1 + marker = dataset['name'] + if marker.endswith(TEMP_SUFFIX): + continue + yield marker[strip:] + + except GeneratorExit: + self.conn.discard() + break + + keys_remaining = count == batch_size + + @copy_ancestor_docstring + def close(self): + self.conn.disconnect() + +class AuthenticationExpired(Exception): + '''Raised if the provided Authentication Token has expired''' + + def __init__(self, msg): + super().__init__() + self.msg = msg + + def __str__(self): + return 'Auth token expired. Server said: %s' % self.msg diff --git a/src/s3ql/backends/swiftks.py b/src/s3ql/backends/swiftks.py new file mode 100644 index 0000000..2d97cc8 --- /dev/null +++ b/src/s3ql/backends/swiftks.py @@ -0,0 +1,135 @@ +''' +swiftks.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from ..logging import logging, QuietError # Ensure use of custom logger class +from . import swift +from dugong import HTTPConnection, CaseInsensitiveDict +from .common import AuthorizationError, retry, DanglingStorageURLError +from .s3c import HTTPError +from ..inherit_docstrings import copy_ancestor_docstring +from urllib.parse import urlsplit +import json +import re +import urllib.parse + +log = logging.getLogger(__name__) + +class Backend(swift.Backend): + + def __init__(self, storage_url, login, password, options): + self.region = None + super().__init__(storage_url, login, password, options) + + @copy_ancestor_docstring + def _parse_storage_url(self, storage_url, ssl_context): + + hit = re.match(r'^[a-zA-Z0-9]+://' # Backend + r'([^/:]+)' # Hostname + r'(?::([0-9]+))?' # Port + r'/([a-zA-Z0-9._-]+):' # Region + r'([^/]+)' # Bucketname + r'(?:/(.*))?$', # Prefix + storage_url) + if not hit: + raise QuietError('Invalid storage URL', exitcode=2) + + hostname = hit.group(1) + if hit.group(2): + port = int(hit.group(2)) + elif ssl_context: + port = 443 + else: + port = 80 + region = hit.group(3) + containername = hit.group(4) + prefix = hit.group(5) or '' + + self.hostname = hostname + self.port = port + self.container_name = containername + self.prefix = prefix + self.region = region + + @retry + def _get_conn(self): + '''Obtain connection to server and authentication token''' + + log.debug('started') + + if 'no-ssl' in self.options: + ssl_context = None + else: + ssl_context = self.ssl_context + + headers = CaseInsensitiveDict() + headers['Content-Type'] = 'application/json' + headers['Accept'] = 'application/json; charset="utf-8"' + + if ':' in self.login: + (tenant,user) = self.login.split(':') + else: + tenant = None + user = self.login + + auth_body = { 'auth': + { 'passwordCredentials': + { 'username': user, + 'password': self.password } }} + if tenant: + auth_body['auth']['tenantName'] = tenant + + with HTTPConnection(self.hostname, port=self.port, proxy=self.proxy, + ssl_context=ssl_context) as conn: + conn.timeout = int(self.options.get('tcp-timeout', 20)) + + conn.send_request('POST', '/v2.0/tokens', headers=headers, + body=json.dumps(auth_body).encode('utf-8')) + resp = conn.read_response() + + if resp.status == 401: + raise AuthorizationError(resp.reason) + + elif resp.status > 299 or resp.status < 200: + raise HTTPError(resp.status, resp.reason, resp.headers) + + cat = json.loads(conn.read().decode('utf-8')) + self.auth_token = cat['access']['token']['id'] + + avail_regions = [] + for service in cat['access']['serviceCatalog']: + if service['type'] != 'object-store': + continue + + for endpoint in service['endpoints']: + if endpoint['region'] != self.region: + avail_regions.append(endpoint['region']) + continue + + o = urlsplit(endpoint['publicURL']) + self.auth_prefix = urllib.parse.unquote(o.path) + if o.scheme == 'https': + ssl_context = self.ssl_context + elif o.scheme == 'http': + ssl_context = None + else: + # fall through to scheme used for authentication + pass + + conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy, + ssl_context=ssl_context) + conn.timeout = int(self.options.get('tcp-timeout', 20)) + return conn + + if len(avail_regions) < 10: + raise DanglingStorageURLError(self.container_name, + 'No accessible object storage service found in region %s' + ' (available regions: %s)' % (self.region, ', '.join(avail_regions))) + else: + raise DanglingStorageURLError(self.container_name, + 'No accessible object storage service found in region %s' + % self.region) diff --git a/src/s3ql/block_cache.py b/src/s3ql/block_cache.py new file mode 100644 index 0000000..a221d46 --- /dev/null +++ b/src/s3ql/block_cache.py @@ -0,0 +1,978 @@ +''' +block_cache.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from . import BUFSIZE +from .database import NoSuchRowError +from .multi_lock import MultiLock +from .logging import logging # Ensure use of custom logger class +from collections import OrderedDict +from contextlib import contextmanager +from llfuse import lock, lock_released +from queue import Queue, Empty as QueueEmpty, Full as QueueFull +import os +import hashlib +import shutil +import threading +import time +import sys + +# standard logger for this module +log = logging.getLogger(__name__) + +# Special queue entry that signals threads to terminate +QuitSentinel = object() + +# Special queue entry that signals that removal queue should +# be flushed +FlushSentinel = object() + +class NoWorkerThreads(Exception): + ''' + Raised when trying to enqueue an object, but there + are no active consumer threads. + ''' + + pass + +class Distributor(object): + ''' + Distributes objects to consumers. + ''' + + def __init__(self): + super().__init__() + + self.slot = None + self.cv = threading.Condition() + + #: Number of threads waiting to consume an object + self.readers = 0 + + def put(self, obj, timeout=None): + '''Offer *obj* for consumption + + The method blocks until another thread calls `get()` to consume the + object. + + Return `True` if the object was consumed, and `False` if *timeout* was + exceeded without any activity in the queue (this means an individual + invocation may wait for longer than *timeout* if objects from other + threads are being consumed). + ''' + + if obj is None: + raise ValueError("Can't put None into Queue") + + with self.cv: + # Wait until a thread is ready to read + while self.readers == 0 or self.slot is not None: + log.debug('waiting for reader..') + if not self.cv.wait(timeout): + log.debug('timeout, returning') + return False + + log.debug('got reader, enqueueing %s', obj) + self.readers -= 1 + assert self.slot is None + self.slot = obj + self.cv.notify_all() # notify readers + + return True + + def get(self): + '''Consume and return an object + + The method blocks until another thread offers an object by calling the + `put` method. + ''' + with self.cv: + self.readers += 1 + self.cv.notify_all() + while self.slot is None: + log.debug('waiting for writer..') + self.cv.wait() + tmp = self.slot + self.slot = None + self.cv.notify_all() + + return tmp + + +class SimpleEvent(object): + ''' + Like threading.Event, but without any internal flag. Calls + to `wait` always block until some other thread calls + `notify` or `notify_all`. + ''' + + def __init__(self): + super().__init__() + self.__cond = threading.Condition(threading.Lock()) + + def notify_all(self): + self.__cond.acquire() + try: + self.__cond.notify_all() + finally: + self.__cond.release() + + def notify(self): + self.__cond.acquire() + try: + self.__cond.notify() + finally: + self.__cond.release() + + def wait(self, timeout=None): + self.__cond.acquire() + try: + return self.__cond.wait(timeout) + finally: + self.__cond.release() + + +class CacheEntry(object): + """An element in the block cache + + Attributes: + ----------- + + :dirty: entry has been changed since it was last uploaded. + :size: current file size + :pos: current position in file + """ + + __slots__ = [ 'dirty', 'inode', 'blockno', 'last_access', + 'size', 'pos', 'fh', 'removed' ] + + def __init__(self, inode, blockno, filename): + super().__init__() + # Writing 100MB in 128k chunks takes 90ms unbuffered and + # 116ms with 1 MB buffer. Reading time does not depend on + # buffer size. + self.fh = open(filename, "w+b", 0) + self.dirty = False + self.inode = inode + self.blockno = blockno + self.last_access = 0 + self.pos = 0 + self.size = os.fstat(self.fh.fileno()).st_size + + def read(self, size=None): + buf = self.fh.read(size) + self.pos += len(buf) + return buf + + def flush(self): + self.fh.flush() + + def seek(self, off): + if self.pos != off: + self.fh.seek(off) + self.pos = off + + def tell(self): + return self.pos + + def truncate(self, size=None): + self.dirty = True + self.fh.truncate(size) + if size is None: + if self.pos < self.size: + self.size = self.pos + elif size < self.size: + self.size = size + + def write(self, buf): + self.dirty = True + self.fh.write(buf) + self.pos += len(buf) + self.size = max(self.pos, self.size) + + def close(self): + self.fh.close() + + def unlink(self): + os.unlink(self.fh.name) + + def __str__(self): + return ('<%sCacheEntry, inode=%d, blockno=%d>' + % ('Dirty ' if self.dirty else '', self.inode, self.blockno)) + + +class CacheDict(OrderedDict): + ''' + An ordered dictionary designed to store CacheEntries. + + Attributes: + + :max_size: maximum size to which cache can grow + :max_entries: maximum number of entries in cache + :size: current size of all entries together + ''' + + def __init__(self, max_size, max_entries): + super().__init__() + self.max_size = max_size + self.max_entries = max_entries + self.size = 0 + + def remove(self, key): + '''Remove *key* from disk and cache, update size''' + + el = self.pop(key) + el.close() + el.unlink() + self.size -= el.size + + def is_full(self): + return (self.size > self.max_size + or len(self) > self.max_entries) + +class BlockCache(object): + """Provides access to file blocks + + This class manages access to file blocks. It takes care of creation, + uploading, downloading and deduplication. + + This class uses the llfuse global lock. Methods which release the lock have + are marked as such in their docstring. + + Attributes + ---------- + + :path: where cached data is stored + :cache: ordered dictionary of cache entries + :mlock: MultiLock to synchronize access to objects and cache entries + :in_transit: set of cache entries that are currently being uploaded + :to_upload: distributes objects to upload to worker threads + :to_remove: distributes objects to remove to worker threads + :transfer_complete: signals completion of an object upload + :upload_threads: list of threads processing upload queue + :removal_threads: list of threads processing removal queue + :db: Handle to SQL DB + :backend_pool: BackendPool instance + """ + + def __init__(self, backend_pool, db, cachedir, max_size, max_entries=768): + log.debug('Initializing') + + self.path = cachedir + self.db = db + self.backend_pool = backend_pool + self.cache = CacheDict(max_size, max_entries) + self.mlock = MultiLock() + self.in_transit = set() + self.upload_threads = [] + self.removal_threads = [] + self.transfer_completed = SimpleEvent() + + # Will be initialized once threads are available + self.to_upload = None + self.to_remove = None + + if not os.path.exists(self.path): + os.mkdir(self.path) + + def __len__(self): + '''Get number of objects in cache''' + return len(self.cache) + + def init(self, threads=1): + '''Start worker threads''' + + self.to_upload = Distributor() + for _ in range(threads): + t = threading.Thread(target=self._upload_loop) + t.start() + self.upload_threads.append(t) + + self.to_remove = Queue(1000) + for _ in range(10): + t = threading.Thread(target=self._removal_loop) + t.daemon = True # interruption will do no permanent harm + t.start() + self.removal_threads.append(t) + + def _lock_obj(self, obj_id, release_global=False): + '''Acquire lock on *obj*id*''' + + if release_global: + with lock_released: + self.mlock.acquire(obj_id) + else: + self.mlock.acquire(obj_id) + + def _unlock_obj(self, obj_id, release_global=False, noerror=False): + '''Release lock on *obj*id*''' + + if release_global: + with lock_released: + self.mlock.release(obj_id, noerror=noerror) + else: + self.mlock.release(obj_id, noerror=noerror) + + def _lock_entry(self, inode, blockno, release_global=False): + '''Acquire lock on cache entry''' + + if release_global: + with lock_released: + self.mlock.acquire((inode, blockno)) + else: + self.mlock.acquire((inode, blockno)) + + def _unlock_entry(self, inode, blockno, release_global=False, + noerror=False): + '''Release lock on cache entry''' + + if release_global: + with lock_released: + self.mlock.release((inode, blockno), noerror=noerror) + else: + self.mlock.release((inode, blockno), noerror=noerror) + + def destroy(self): + '''Clean up and stop worker threads + + This method should be called without the global lock held. + ''' + + log.debug('clearing cache...') + try: + with lock: + self.clear() + except NoWorkerThreads: + log.error('Unable to flush cache, no upload threads left alive') + + # Signal termination to worker threads. If some of them + # terminated prematurely, continue gracefully. + log.debug('Signaling upload threads...') + try: + for t in self.upload_threads: + self._queue_upload(QuitSentinel) + except NoWorkerThreads: + pass + + log.debug('Signaling removal threads...') + try: + for t in self.removal_threads: + self._queue_removal(QuitSentinel) + except NoWorkerThreads: + pass + + log.debug('waiting for upload threads...') + for t in self.upload_threads: + t.join() + + log.debug('waiting for removal threads...') + for t in self.removal_threads: + t.join() + + assert len(self.in_transit) == 0 + try: + while self.to_remove.get_nowait() is QuitSentinel: + pass + except QueueEmpty: + pass + else: + log.error('Could not complete object removals, ' + 'no removal threads left alive') + + self.to_upload = None + self.to_remove = None + self.upload_threads = None + self.removal_threads = None + + os.rmdir(self.path) + + log.debug('cleanup done.') + + + def _upload_loop(self): + '''Process upload queue''' + + while True: + tmp = self.to_upload.get() + + if tmp is QuitSentinel: + break + + self._do_upload(*tmp) + + + def _do_upload(self, el, obj_id): + '''Upload object''' + + def do_write(fh): + el.seek(0) + while True: + buf = el.read(BUFSIZE) + if not buf: + break + fh.write(buf) + return fh + + try: + with self.backend_pool() as backend: + if log.isEnabledFor(logging.DEBUG): + time_ = time.time() + obj_size = backend.perform_write(do_write, 's3ql_data_%d' + % obj_id).get_obj_size() + time_ = time.time() - time_ + rate = el.size / (1024 ** 2 * time_) if time_ != 0 else 0 + log.debug('uploaded %d bytes in %.3f seconds, %.2f MiB/s', + el.size, time_, rate) + else: + obj_size = backend.perform_write(do_write, 's3ql_data_%d' + % obj_id).get_obj_size() + + with lock: + self.db.execute('UPDATE objects SET size=? WHERE id=?', (obj_size, obj_id)) + el.dirty = False + + except Exception as exc: + log.debug('upload of %d failed: %s', obj_id, exc) + # At this point we have to remove references to this storage object + # from the objects and blocks table to prevent future cache elements + # to be de-duplicated against this (missing) one. However, this may + # already have happened during the attempted upload. The only way to + # avoid this problem is to insert the hash into the blocks table + # *after* successfull upload. But this would open a window without + # de-duplication just to handle the special case of an upload + # failing. + # + # On the other hand, we also want to prevent future deduplication + # against this block: otherwise the next attempt to upload the same + # cache element (by a different upload thread that has not + # encountered problems yet) is guaranteed to link against the + # non-existing block, and the data will be lost. + # + # Therefore, we just set the hash of the missing block to NULL, + # and rely on fsck to pick up the pieces. Note that we cannot + # delete the row from the blocks table, because the id will get + # assigned to a new block, so the inode_blocks entries will + # refer to incorrect data. + # + + with lock: + self.db.execute('UPDATE blocks SET hash=NULL WHERE obj_id=?', + (obj_id,)) + raise + + finally: + self.in_transit.remove(el) + self._unlock_obj(obj_id) + self._unlock_entry(el.inode, el.blockno) + self.transfer_completed.notify_all() + + + def wait(self): + '''Wait until an object has been uploaded + + If there are no objects in transit, return immediately. This method + releases the global lock. + ''' + + # Loop to avoid the race condition of a transfer terminating + # between the call to transfer_in_progress() and wait(). + while True: + if not self.transfer_in_progress(): + return + + with lock_released: + if self.transfer_completed.wait(timeout=5): + return + + def upload(self, el): + '''Upload cache entry `el` asynchronously + + This method releases the global lock. + ''' + + log.debug('started with %s', el) + + # Calculate checksum + with lock_released: + try: + self._lock_entry(el.inode, el.blockno) + assert el not in self.in_transit + if not el.dirty: + log.debug('not dirty, returning') + self._unlock_entry(el.inode, el.blockno) + return + if (el.inode, el.blockno) not in self.cache: + log.debug('%s removed while waiting for lock', el) + self._unlock_entry(el.inode, el.blockno) + return + + self.in_transit.add(el) + sha = hashlib.sha256() + el.seek(0) + while True: + buf = el.read(BUFSIZE) + if not buf: + break + sha.update(buf) + hash_ = sha.digest() + except: + self.in_transit.discard(el) + self._unlock_entry(el.inode, el.blockno) + raise + + obj_lock_taken = False + try: + try: + old_block_id = self.db.get_val('SELECT block_id FROM inode_blocks ' + 'WHERE inode=? AND blockno=?', + (el.inode, el.blockno)) + except NoSuchRowError: + old_block_id = None + + try: + block_id = self.db.get_val('SELECT id FROM blocks WHERE hash=?', (hash_,)) + + # No block with same hash + except NoSuchRowError: + obj_id = self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, -1)') + log.debug('created new object %d', obj_id) + block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, hash, size) ' + 'VALUES(?,?,?,?)', (1, obj_id, hash_, el.size)) + log.debug('created new block %d', block_id) + log.debug('adding to upload queue') + + # Note: we must finish all db transactions before adding to + # in_transit, otherwise commit() may return before all blocks + # are available in db. + self.db.execute('INSERT OR REPLACE INTO inode_blocks (block_id, inode, blockno) ' + 'VALUES(?,?,?)', (block_id, el.inode, el.blockno)) + + with lock_released: + self._lock_obj(obj_id) + obj_lock_taken = True + self._queue_upload((el, obj_id)) + + # There is a block with the same hash + else: + if old_block_id != block_id: + log.debug('(re)linking to %d', block_id) + self.db.execute('UPDATE blocks SET refcount=refcount+1 WHERE id=?', + (block_id,)) + self.db.execute('INSERT OR REPLACE INTO inode_blocks (block_id, inode, blockno) ' + 'VALUES(?,?,?)', (block_id, el.inode, el.blockno)) + + el.dirty = False + self.in_transit.remove(el) + self._unlock_entry(el.inode, el.blockno, release_global=True) + + if old_block_id == block_id: + log.debug('unchanged, block_id=%d', block_id) + return + + except: + self.in_transit.discard(el) + with lock_released: + self._unlock_entry(el.inode, el.blockno, noerror=True) + if obj_lock_taken: + self._unlock_obj(obj_id) + raise + + # Check if we have to remove an old block + if not old_block_id: + log.debug('no old block, returning') + return + + self._deref_block(old_block_id) + + def _queue_upload(self, obj): + '''Put *obj* into upload queue''' + + while True: + if self.to_upload.put(obj, timeout=5): + return + for t in self.upload_threads: + if t.is_alive(): + break + else: + raise NoWorkerThreads('no upload threads') + + def _queue_removal(self, obj): + '''Put *obj* into removal queue''' + + while True: + try: + self.to_remove.put(obj, timeout=5) + except QueueFull: + pass + else: + return + + for t in self.removal_threads: + if t.is_alive(): + break + else: + raise NoWorkerThreads('no removal threads') + + def _deref_block(self, block_id): + '''Decrease reference count for *block_id* + + If reference counter drops to zero, remove block and propagate + to objects table (possibly removing the referenced object + as well). + + This method releases the global lock. + ''' + + refcount = self.db.get_val('SELECT refcount FROM blocks WHERE id=?', (block_id,)) + if refcount > 1: + log.debug('decreased refcount for block: %d', block_id) + self.db.execute('UPDATE blocks SET refcount=refcount-1 WHERE id=?', (block_id,)) + return + + log.debug('removing block %d', block_id) + obj_id = self.db.get_val('SELECT obj_id FROM blocks WHERE id=?', (block_id,)) + self.db.execute('DELETE FROM blocks WHERE id=?', (block_id,)) + (refcount, size) = self.db.get_row('SELECT refcount, size FROM objects WHERE id=?', + (obj_id,)) + if refcount > 1: + log.debug('decreased refcount for obj: %d', obj_id) + self.db.execute('UPDATE objects SET refcount=refcount-1 WHERE id=?', + (obj_id,)) + return + + log.debug('removing object %d', obj_id) + self.db.execute('DELETE FROM objects WHERE id=?', (obj_id,)) + + # Taking the lock ensures that the object is no longer in + # transit itself. We can release it immediately after, because + # the object is no longer in the database. + log.debug('adding %d to removal queue', obj_id) + + with lock_released: + self._lock_obj(obj_id) + self._unlock_obj(obj_id) + + if size == -1: + # size == -1 indicates that object has not yet been uploaded. + # However, since we just acquired a lock on the object, we know + # that the upload must have failed. Therefore, trying to remove + # this object would just give us another error. + return + + self._queue_removal(obj_id) + + + def transfer_in_progress(self): + '''Return True if there are any cache entries being uploaded''' + + return len(self.in_transit) > 0 + + def _removal_loop(self): + '''Process removal queue''' + + # This method may look more complicated than necessary, but + # it ensures that we read as many objects from the queue + # as we can without blocking, and then hand them over to + # the backend all at once. + ids = [] + while True: + try: + log.debug('reading from queue (blocking=%s)', len(ids)==0) + tmp = self.to_remove.get(block=len(ids)==0) + except QueueEmpty: + tmp = FlushSentinel + + if (tmp is FlushSentinel or tmp is QuitSentinel) and ids: + log.debug('removing: %s', ids) + with self.backend_pool() as backend: + backend.delete_multi(['s3ql_data_%d' % i for i in ids]) + ids = [] + else: + ids.append(tmp) + + if tmp is QuitSentinel: + break + + + @contextmanager + def get(self, inode, blockno): + """Get file handle for block `blockno` of `inode` + + This method releases the global lock. The managed block, however, + is executed with the global lock acquired and MUST NOT release + it. This ensures that only one thread is accessing a given block + at a time. + + Note: if `get` and `remove` are called concurrently, then it is + possible that a block that has been requested with `get` and + passed to `remove` for deletion will not be deleted. + """ + + #log.debug('started with %d, %d', inode, blockno) + + if self.cache.is_full(): + self.expire() + + self._lock_entry(inode, blockno, release_global=True) + try: + el = self._get_entry(inode, blockno) + el.last_access = time.time() + oldsize = el.size + try: + yield el + finally: + # Update cachesize. NOTE: this requires that at most one + # thread has access to a cache entry at any time. + self.cache.size += el.size - oldsize + finally: + self._unlock_entry(inode, blockno, release_global=True) + + #log.debug('finished') + + def _get_entry(self, inode, blockno): + '''Get cache entry for `blockno` of `inode` + + Assume that cache entry lock has been acquired. + ''' + + log.debug('started with %d, %d', inode, blockno) + try: + el = self.cache[(inode, blockno)] + + # Not in cache + except KeyError: + filename = os.path.join(self.path, '%d-%d' % (inode, blockno)) + try: + block_id = self.db.get_val('SELECT block_id FROM inode_blocks ' + 'WHERE inode=? AND blockno=?', (inode, blockno)) + + # No corresponding object + except NoSuchRowError: + log.debug('creating new block') + el = CacheEntry(inode, blockno, filename) + self.cache[(inode, blockno)] = el + return el + + # Need to download corresponding object + obj_id = self.db.get_val('SELECT obj_id FROM blocks WHERE id=?', (block_id,)) + log.debug('downloading object %d..', obj_id) + el = CacheEntry(inode, blockno, filename) + try: + def do_read(fh): + el.seek(0) + el.truncate() + shutil.copyfileobj(fh, el, BUFSIZE) + + with lock_released: + # Lock object. This ensures that we wait until the object + # is uploaded. We don't have to worry about deletion, because + # as long as the current cache entry exists, there will always be + # a reference to the object (and we already have a lock on the + # cache entry). + self._lock_obj(obj_id) + self._unlock_obj(obj_id) + with self.backend_pool() as backend: + backend.perform_read(do_read, 's3ql_data_%d' % obj_id) + except: + el.unlink() + el.close() + raise + + self.cache[(inode, blockno)] = el + el.dirty = False # (writing will have set dirty flag) + self.cache.size += el.size + + # In Cache + else: + #log.debug('in cache') + self.cache.move_to_end((inode, blockno), last=True) # move to head + + return el + + def expire(self): + """Perform cache expiry + + This method releases the global lock. + """ + + # Note that we have to make sure that the cache entry is written into + # the database before we remove it from the cache! + + log.debug('started') + + while True: + need_size = self.cache.size - self.cache.max_size + need_entries = len(self.cache) - self.cache.max_entries + + if need_size <= 0 and need_entries <= 0: + break + + # Need to make copy, since we aren't allowed to change dict while + # iterating through it. Look at the comments in CommitThread.run() + # (mount.py) for an estimate of the resulting performance hit. + sth_in_transit = False + for el in list(self.cache.values()): + if need_size <= 0 and need_entries <= 0: + break + + need_entries -= 1 + need_size -= el.size + + if el.dirty: + if el not in self.in_transit: + log.debug('uploading %s..', el) + self.upload(el) # Releases global lock + sth_in_transit = True + continue + + log.debug('removing inode %d, block %d from cache', el.inode, el.blockno) + self._lock_entry(el.inode, el.blockno, release_global=True) + try: + # May have changed while we were waiting for lock + if el.dirty: + log.debug('%s got dirty while waiting for lock', el) + continue + if (el.inode, el.blockno) not in self.cache: + log.debug('%s removed while waiting for lock', el) + continue + self.cache.remove((el.inode, el.blockno)) + finally: + self._unlock_entry(el.inode, el.blockno, release_global=True) + + if sth_in_transit: + log.debug('waiting for transfer threads..') + self.wait() # Releases global lock + + log.debug('finished') + + + def remove(self, inode, start_no, end_no=None): + """Remove blocks for `inode` + + If `end_no` is not specified, remove just the `start_no` block. + Otherwise removes all blocks from `start_no` to, but not including, + `end_no`. + + This method releases the global lock. + """ + + log.debug('started with %d, %d, %s', inode, start_no, end_no) + + if end_no is None: + end_no = start_no + 1 + + for blockno in range(start_no, end_no): + self._lock_entry(inode, blockno, release_global=True) + try: + if (inode, blockno) in self.cache: + log.debug('removing from cache') + self.cache.remove((inode, blockno)) + + try: + block_id = self.db.get_val('SELECT block_id FROM inode_blocks ' + 'WHERE inode=? AND blockno=?', (inode, blockno)) + except NoSuchRowError: + log.debug('block not in db') + continue + + # Detach inode from block + self.db.execute('DELETE FROM inode_blocks WHERE inode=? AND blockno=?', + (inode, blockno)) + + finally: + self._unlock_entry(inode, blockno, release_global=True) + + # Decrease block refcount + self._deref_block(block_id) + + log.debug('finished') + + def flush(self, inode, blockno): + """Flush buffers for given block""" + + try: + el = self.cache[(inode, blockno)] + except KeyError: + return + + el.flush() + + def commit(self): + """Initiate upload of all dirty blocks + + When the method returns, all blocks have been registered + in the database (but the actual uploads may still be + in progress). + + This method releases the global lock. + """ + + # Need to make copy, since we aren't allowed to change dict while + # iterating through it. Look at the comments in CommitThread.run() + # (mount.py) for an estimate of the resulting performance hit. + for el in list(self.cache.values()): + if not el.dirty or el in self.in_transit: + continue + + self.upload(el) # Releases global lock + + def clear(self): + """Clear cache + + This method releases the global lock. + """ + + log.debug('started') + bak = self.cache.max_entries + self.cache.max_entries = 0 + self.expire() # Releases global lock + self.cache.max_entries = bak + log.debug('finished') + + def get_usage(self): + '''Get cache usage information. + + Return a tuple of + + * cache entries + * cache size + * dirty cache entries + * dirty cache size + * pending removals + + This method is O(n) in the number of cache entries. + ''' + + used = self.cache.size + dirty_size = 0 + dirty_cnt = 0 + for el in self.cache.values(): + if el.dirty: + dirty_size += el.size + dirty_cnt += 1 + + if self.to_remove is None: + remove_cnt = 0 + else: + # This is an estimate which may be negative + remove_cnt = max(0, self.to_remove.qsize()) + + return (len(self.cache), used, dirty_cnt, dirty_size, remove_cnt) + + def __del__(self): + if len(self.cache) == 0: + return + + # Force execution of sys.excepthook (exceptions raised + # by __del__ are ignored) + try: + raise RuntimeError("BlockManager instance was destroyed without " + "calling destroy()!") + except RuntimeError: + exc_info = sys.exc_info() + + sys.excepthook(*exc_info) diff --git a/src/s3ql/calc_mro.py b/src/s3ql/calc_mro.py new file mode 100644 index 0000000..bb5bc04 --- /dev/null +++ b/src/s3ql/calc_mro.py @@ -0,0 +1,120 @@ +''' +calc_mro.py - this file is part of S3QL. + +Copyright © 2011 Steven D'Aprano + +This file may be distributed under the terms of the MIT License. + +The original source code was retrieved from +http://code.activestate.com/recipes/577748-calculate-the-mro-of-a-class/ +''' + +def calc_mro(*bases): + """Calculate the Method Resolution Order of bases using the C3 algorithm. + + Suppose you intended creating a class K with the given base classes. This + function returns the MRO which K would have, *excluding* K itself (since + it doesn't yet exist), as if you had actually created the class. + + Another way of looking at this, if you pass a single class K, this will + return the linearization of K (the MRO of K, *including* itself). + """ + seqs = [list(C.__mro__) for C in bases] + [list(bases)] + res = [] + while True: + non_empty = list(filter(None, seqs)) + if not non_empty: + # Nothing left to process, we're done. + return tuple(res) + for seq in non_empty: # Find merge candidates among seq heads. + candidate = seq[0] + not_head = [s for s in non_empty if candidate in s[1:]] + if not_head: + # Reject the candidate. + candidate = None + else: + break + if not candidate: + raise TypeError("inconsistent hierarchy, no C3 MRO is possible") + res.append(candidate) + for seq in non_empty: + # Remove candidate. + if seq[0] == candidate: + del seq[0] + + + +if __name__ == '__main__': + # Run self-tests. Prints nothing if they succeed. + O = object + class SeriousOrderDisagreement: + class X(O): pass + class Y(O): pass + class A(X, Y): pass + class B(Y, X): pass + bases = (A, B) + + try: + x = calc_mro(*SeriousOrderDisagreement.bases) + except TypeError: + pass + else: + print("failed test, mro should have raised but got %s instead" % (x,)) + + class Example0: # Trivial single inheritance case. + class A(O): pass + class B(A): pass + class C(B): pass + class D(C): pass + tester = D + expected = (D, C, B, A, O) + + class Example1: + class F(O): pass + class E(O): pass + class D(O): pass + class C(D, F): pass + class B(D, E): pass + class A(B, C): pass + tester = A + expected = (A, B, C, D, E, F, O) + + class Example2: + class F(O): pass + class E(O): pass + class D(O): pass + class C(D, F): pass + class B(E, D): pass + class A(B, C): pass + tester = A + expected = (A, B, E, C, D, F, O) + + class Example3: + class A(O): pass + class B(O): pass + class C(O): pass + class D(O): pass + class E(O): pass + class K1(A, B, C): pass + class K2(D, B, E): pass + class K3(D, A): pass + class Z(K1, K2, K3): pass + + assert calc_mro(A) == (A, O) + assert calc_mro(B) == (B, O) + assert calc_mro(C) == (C, O) + assert calc_mro(D) == (D, O) + assert calc_mro(E) == (E, O) + assert calc_mro(K1) == (K1, A, B, C, O) + assert calc_mro(K2) == (K2, D, B, E, O) + assert calc_mro(K3) == (K3, D, A, O) + + tester = Z + expected = (Z, K1, K2, K3, D, A, B, C, E, O) + + for example in [Example0, Example1, Example2, Example3]: + # First test that the expected result is the same as what Python + # actually generates. + assert example.expected == example.tester.__mro__ + # Now check the calculated MRO. + assert calc_mro(example.tester) == example.expected diff --git a/src/s3ql/common.py b/src/s3ql/common.py new file mode 100644 index 0000000..09216cb --- /dev/null +++ b/src/s3ql/common.py @@ -0,0 +1,629 @@ +''' +common.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging, QuietError # Ensure use of custom logger class +from . import BUFSIZE, CTRL_NAME, ROOT_INODE +from dugong import HostnameNotResolvable +from getpass import getpass +from ast import literal_eval +from base64 import b64decode, b64encode +import binascii +import configparser +import re +import stat +import threading +import traceback +import sys +import os +import time +import subprocess +import errno +import hashlib +import llfuse +import posixpath +import functools +import contextlib + +log = logging.getLogger(__name__) + +file_system_encoding = sys.getfilesystemencoding() +def path2bytes(s): + return s.encode(file_system_encoding, 'surrogateescape') +def bytes2path(s): + return s.decode(file_system_encoding, 'surrogateescape') + +def get_seq_no(backend): + '''Get current metadata sequence number''' + + from .backends.common import NoSuchObject + + seq_nos = list(backend.list('s3ql_seq_no_')) + if not seq_nos: + # Maybe list result is outdated + seq_nos = [ 's3ql_seq_no_1' ] + + seq_nos = [ int(x[len('s3ql_seq_no_'):]) for x in seq_nos ] + seq_no = max(seq_nos) + + # Make sure that object really exists + while ('s3ql_seq_no_%d' % seq_no) not in backend: + seq_no -= 1 + if seq_no == 0: + raise QuietError('No S3QL file system found at given storage URL.', + exitcode=18) + while ('s3ql_seq_no_%d' % seq_no) in backend: + seq_no += 1 + seq_no -= 1 + + # Delete old seq nos + for i in [ x for x in seq_nos if x < seq_no - 10 ]: + try: + del backend['s3ql_seq_no_%d' % i] + except NoSuchObject: + pass # Key list may not be up to date + + return seq_no + +def is_mounted(storage_url): + '''Try to determine if *storage_url* is mounted + + Note that the result may be wrong.. this is really just + a best-effort guess. + ''' + + match = storage_url + ' ' + if os.path.exists('/proc/mounts'): + with open('/proc/mounts', 'r') as fh: + for line in fh: + if line.startswith(match): + return True + return False + + try: + for line in subprocess.check_output(['mount'], stderr=subprocess.STDOUT, + universal_newlines=True): + if line.startswith(match): + return True + except subprocess.CalledProcessError: + log.warning('Warning! Unable to check if file system is mounted ' + '(/proc/mounts missing and mount call failed)') + + return False + +def inode_for_path(path, conn): + """Return inode of directory entry at `path` + + Raises `KeyError` if the path does not exist. + """ + from .database import NoSuchRowError + + if not isinstance(path, bytes): + raise TypeError('path must be of type bytes') + + # Remove leading and trailing / + path = path.lstrip(b"/").rstrip(b"/") + + # Traverse + inode = ROOT_INODE + for el in path.split(b'/'): + try: + inode = conn.get_val("SELECT inode FROM contents_v WHERE name=? AND parent_inode=?", + (el, inode)) + except NoSuchRowError: + raise KeyError('Path %s does not exist' % path) + + return inode + +def get_path(id_, conn, name=None): + """Return a full path for inode `id_`. + + If `name` is specified, it is appended at the very end of the + path (useful if looking up the path for file name with parent + inode). + """ + + if name is None: + path = list() + else: + if not isinstance(name, bytes): + raise TypeError('name must be of type bytes') + path = [ name ] + + maxdepth = 255 + while id_ != ROOT_INODE: + # This can be ambiguous if directories are hardlinked + (name2, id_) = conn.get_row("SELECT name, parent_inode FROM contents_v " + "WHERE inode=? LIMIT 1", (id_,)) + path.append(name2) + maxdepth -= 1 + if maxdepth == 0: + raise RuntimeError('Failed to resolve name "%s" at inode %d to path', + name, id_) + + path.append(b'') + path.reverse() + + return b'/'.join(path) + + +def _escape(s): + '''Escape '/', '=' and '\0' in s''' + + s = s.replace('=', '=3D') + s = s.replace('/', '=2F') + s = s.replace('\0', '=00') + + return s + +def get_backend_cachedir(storage_url, cachedir): + if not os.path.exists(cachedir): + try: + os.mkdir(cachedir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + except PermissionError: + raise QuietError('No permission to create cache directory (%s)' % cachedir, + exitcode=45) + + if not os.access(cachedir, os.R_OK | os.W_OK | os.X_OK): + raise QuietError('No permission to access cache directory (%s)' % cachedir, + exitcode=45) + + return os.path.join(cachedir, _escape(storage_url)) + + +def sha256_fh(fh): + fh.seek(0) + + # Bogus error about hashlib not having a sha256 member + #pylint: disable=E1101 + sha = hashlib.sha256() + + while True: + buf = fh.read(BUFSIZE) + if not buf: + break + sha.update(buf) + + return sha.digest() + + +def assert_s3ql_fs(path): + '''Raise `QuietError` if *path* is not on an S3QL file system + + Returns name of the S3QL control file. + ''' + + try: + os.stat(path) + except FileNotFoundError: + raise QuietError('%s does not exist' % path) + except OSError as exc: + if exc.errno is errno.ENOTCONN: + raise QuietError('File system appears to have crashed.') + raise + + ctrlfile = os.path.join(path, CTRL_NAME) + if not (CTRL_NAME not in llfuse.listdir(path) + and os.path.exists(ctrlfile)): + raise QuietError('%s is not on an S3QL file system' % path) + + return ctrlfile + + +def assert_fs_owner(path, mountpoint=False): + '''Raise `QuietError` if user is not owner of S3QL fs at *path* + + Implicitly calls `assert_s3ql_fs` first. Returns name of the + S3QL control file. + + If *mountpoint* is True, also call `assert_s3ql_mountpoint`, i.e. + fail if *path* is not the mount point of the file system. + ''' + + if mountpoint: + ctrlfile = assert_s3ql_mountpoint(path) + else: + ctrlfile = assert_s3ql_fs(path) + + if os.stat(ctrlfile).st_uid != os.geteuid() and os.geteuid() != 0: + raise QuietError('Permission denied. %s is was not mounted by you ' + 'and you are not root.' % path) + + return ctrlfile + +def assert_s3ql_mountpoint(mountpoint): + '''Raise QuietError if *mountpoint* is not an S3QL mountpoint + + Implicitly calls `assert_s3ql_fs` first. Returns name of the + S3QL control file. + ''' + + ctrlfile = assert_s3ql_fs(mountpoint) + if not posixpath.ismount(mountpoint): + raise QuietError('%s is not a mount point' % mountpoint) + + return ctrlfile + +def get_backend(options, raw=False): + '''Return backend for given storage-url + + If *raw* is true, don't attempt to unlock and don't wrap into + ComprencBackend. + ''' + + return get_backend_factory(options.storage_url, options.backend_options, + options.authfile, + getattr(options, 'compress', ('lzma', 2)), raw)() + +def get_backend_factory(storage_url, backend_options, authfile, + compress=('lzma', 2), raw=False): + '''Return factory producing backend objects for given storage-url + + If *raw* is true, don't attempt to unlock and don't wrap into + ComprencBackend. + ''' + + from .backends import prefix_map + from .backends.common import (CorruptedObjectError, NoSuchObject, AuthenticationError, + DanglingStorageURLError, AuthorizationError) + from .backends.comprenc import ComprencBackend + + hit = re.match(r'^([a-zA-Z0-9]+)://', storage_url) + if not hit: + raise QuietError('Unable to parse storage url "%s"' % storage_url, + exitcode=2) + + backend = hit.group(1) + try: + backend_class = prefix_map[backend] + except KeyError: + raise QuietError('No such backend: %s' % backend, exitcode=11) + + # Validate backend options + for opt in backend_options.keys(): + if opt not in backend_class.known_options: + raise QuietError('Unknown backend option: %s' % opt, + exitcode=3) + + # Read authfile + config = configparser.ConfigParser() + if os.path.isfile(authfile): + mode = os.stat(authfile).st_mode + if mode & (stat.S_IRGRP | stat.S_IROTH): + raise QuietError("%s has insecure permissions, aborting." % authfile, + exitcode=12) + config.read(authfile) + + backend_login = None + backend_passphrase = None + fs_passphrase = None + for section in config.sections(): + def getopt(name): + try: + return config.get(section, name) + except configparser.NoOptionError: + return None + + pattern = getopt('storage-url') + + if not pattern or not storage_url.startswith(pattern): + continue + + backend_login = getopt('backend-login') or backend_login + backend_passphrase = getopt('backend-password') or backend_passphrase + fs_passphrase = getopt('fs-passphrase') or fs_passphrase + + if not backend_login and backend_class.needs_login: + if sys.stdin.isatty(): + backend_login = getpass("Enter backend login: ") + else: + backend_login = sys.stdin.readline().rstrip() + + if not backend_passphrase and backend_class.needs_login: + if sys.stdin.isatty(): + backend_passphrase = getpass("Enter backend passphrase: ") + else: + backend_passphrase = sys.stdin.readline().rstrip() + + backend = None + try: + backend = backend_class(storage_url, backend_login, backend_passphrase, + backend_options) + + # Do not use backend.lookup(), this would use a HEAD request and + # not provide any useful error messages if something goes wrong + # (e.g. wrong credentials) + backend.fetch('s3ql_passphrase') + + except AuthenticationError: + raise QuietError('Invalid credentials (or skewed system clock?).', + exitcode=14) + + except AuthorizationError: + raise QuietError('No permission to access backend.', + exitcode=15) + + except HostnameNotResolvable: + raise QuietError("Can't connect to backend: unable to resolve hostname", + exitcode=19) + + except DanglingStorageURLError as exc: + raise QuietError(str(exc), exitcode=16) + + except NoSuchObject: + encrypted = False + + else: + encrypted = True + + finally: + if backend is not None: + backend.close() + + if raw: + return lambda: backend_class(storage_url, backend_login, backend_passphrase, + backend_options) + + if encrypted and not fs_passphrase: + if sys.stdin.isatty(): + fs_passphrase = getpass("Enter file system encryption passphrase: ") + else: + fs_passphrase = sys.stdin.readline().rstrip() + elif not encrypted: + fs_passphrase = None + + if fs_passphrase is not None: + fs_passphrase = fs_passphrase.encode('utf-8') + + if not encrypted: + return lambda: ComprencBackend(None, compress, + backend_class(storage_url, backend_login, + backend_passphrase, backend_options)) + + with ComprencBackend(fs_passphrase, compress, backend) as tmp_backend: + try: + data_pw = tmp_backend['s3ql_passphrase'] + except CorruptedObjectError: + raise QuietError('Wrong file system passphrase', exitcode=17) + + # To support upgrade, temporarily store the backend + # passphrase in every backend object. + def factory(): + b = ComprencBackend(data_pw, compress, + backend_class(storage_url, backend_login, + backend_passphrase, backend_options)) + b.fs_passphrase = fs_passphrase + return b + + return factory + +def pretty_print_size(i): + '''Return *i* as string with appropriate suffix (MiB, GiB, etc)''' + + if i < 1024: + return '%d bytes' % i + + if i < 1024**2: + unit = 'KiB' + i /= 1024 + + elif i < 1024**3: + unit = 'MiB' + i /= 1024**2 + + elif i < 1024**4: + unit = 'GiB' + i /= 1024**3 + + else: + unit = 'TB' + i /= 1024**4 + + if i < 10: + form = '%.2f %s' + elif i < 100: + form = '%.1f %s' + else: + form = '%d %s' + + return form % (i, unit) + + +class ExceptionStoringThread(threading.Thread): + def __init__(self): + super().__init__() + self._exc_info = None + self._joined = False + + def run_protected(self): + pass + + def run(self): + try: + self.run_protected() + except: + # This creates a circular reference chain + self._exc_info = sys.exc_info() + + def join_get_exc(self): + self._joined = True + self.join() + return self._exc_info + + def join_and_raise(self): + '''Wait for the thread to finish, raise any occurred exceptions''' + + self._joined = True + if self.is_alive(): + self.join() + + if self._exc_info is not None: + # Break reference chain + exc_info = self._exc_info + self._exc_info = None + raise EmbeddedException(exc_info, self.name) + + def __del__(self): + if not self._joined: + raise RuntimeError("ExceptionStoringThread instance was destroyed " + "without calling join_and_raise()!") + +class EmbeddedException(Exception): + '''Encapsulates an exception that happened in a different thread + ''' + + def __init__(self, exc_info, threadname): + super().__init__() + self.exc_info = exc_info + self.threadname = threadname + + def __str__(self): + return ''.join(['caused by an exception in thread %s.\n' % self.threadname, + 'Original/inner traceback (most recent call last): \n' ] + + traceback.format_exception(*self.exc_info)) + + +class AsyncFn(ExceptionStoringThread): + def __init__(self, fn, *args, **kwargs): + super().__init__() + self.target = fn + self.args = args + self.kwargs = kwargs + + def run_protected(self): + self.target(*self.args, **self.kwargs) + +def split_by_n(seq, n): + '''Yield elements in iterable *seq* in groups of *n*''' + + while seq: + yield seq[:n] + seq = seq[n:] + +def handle_on_return(fn): + '''Provide fresh ExitStack instance in `on_return` argument''' + @functools.wraps(fn) + def wrapper(*a, **kw): + assert 'on_return' not in kw + with contextlib.ExitStack() as on_return: + kw['on_return'] = on_return + return fn(*a, **kw) + return wrapper + +def parse_literal(buf, type_spec): + '''Try to parse *buf* as *type_spec* + + Raise `ValueError` if *buf* does not contain a valid + Python literal, or if the literal does not correspond + to *type_spec*. + + Example use:: + + buf = b'[1, 'a', 3]' + parse_literal(buf, [int, str, int]) + + ''' + + try: + obj = literal_eval(buf.decode()) + except UnicodeDecodeError: + raise ValueError('unable to decode as utf-8') + except (ValueError, SyntaxError): + raise ValueError('unable to parse as python literal') + + if (isinstance(type_spec, list) and type(obj) == list + and [ type(x) for x in obj ] == type_spec): + return obj + elif (isinstance(type_spec, tuple) and type(obj) == tuple + and [ type(x) for x in obj ] == list(type_spec)): + return obj + elif type(obj) == type_spec: + return obj + + raise ValueError('literal has wrong type') + +class ThawError(Exception): + def __str__(self): + return 'Malformed serialization data' + +def thaw_basic_mapping(buf): + '''Reconstruct dict from serialized representation + + *buf* must be a bytes-like object as created by + `freeze_basic_mapping`. Raises `ThawError` if *buf* is not a valid + representation. + + This procedure is safe even if *buf* comes from an untrusted source. + ''' + + try: + d = literal_eval(buf.decode('utf-8')) + except (UnicodeDecodeError, SyntaxError, ValueError): + raise ThawError() + + # Decode bytes values + for (k,v) in d.items(): + if not isinstance(v, bytes): + continue + try: + d[k] = b64decode(v) + except binascii.Error: + raise ThawError() + + return d + +def freeze_basic_mapping(d): + '''Serialize mapping of elementary types + + Keys of *d* must be strings. Values of *d* must be of elementary type (i.e., + `str`, `bytes`, `int`, `float`, `complex`, `bool` or None). + + The output is a bytestream that can be used to reconstruct the mapping. The + bytestream is not guaranteed to be deterministic. Look at + `checksum_basic_mapping` if you need a deterministic bytestream. + ''' + + els = [] + for (k,v) in d.items(): + if not isinstance(k, str): + raise ValueError('key %s must be str, not %s' % (k, type(k))) + + if (not isinstance(v, (str, bytes, bytearray, int, float, complex, bool)) + and v is not None): + raise ValueError('value for key %s (%s) is not elementary' % (k, v)) + + # To avoid wasting space, we b64encode non-ascii byte values. + if isinstance(v, (bytes, bytearray)): + v = b64encode(v) + + # This should be a pretty safe assumption for elementary types, but we + # add an assert just to be safe (Python docs just say that repr makes + # "best effort" to produce something parseable) + (k_repr, v_repr) = (repr(k), repr(v)) + assert (literal_eval(k_repr), literal_eval(v_repr)) == (k, v) + + els.append(('%s: %s' % (k_repr, v_repr))) + + buf = '{ %s }' % ', '.join(els) + return buf.encode('utf-8') + +def load_params(cachepath): + with open(cachepath + '.params' , 'rb') as fh: + return thaw_basic_mapping(fh.read()) + +def save_params(cachepath, param): + with open(cachepath + '.params', 'wb') as fh: + fh.write(freeze_basic_mapping(param)) + + # Fsync to make sure that the updated sequence number is committed to + # disk. Otherwise, a crash immediately after mount could result in both + # the local and remote metadata appearing to be out of date. + fh.flush() + os.fsync(fh.fileno()) + +def time_ns(): + return int(time.time() * 1e9) diff --git a/src/s3ql/cp.py b/src/s3ql/cp.py new file mode 100644 index 0000000..a6e31a4 --- /dev/null +++ b/src/s3ql/cp.py @@ -0,0 +1,92 @@ +''' +cp.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging, setup_logging, QuietError +from .common import assert_fs_owner +from .parse_args import ArgumentParser +import llfuse +import os +import stat +import sys +import textwrap + +log = logging.getLogger(__name__) + +def parse_args(args): + '''Parse command line''' + + parser = ArgumentParser( + description=textwrap.dedent('''\ + Replicates the contents of the directory in the + directory . has to be an existing directory and + must not exist. Both directories have to be within + the same S3QL file system. + + The replication will not take any additional space. Only if one + of directories is modified later on, the modified data will take + additional storage space. + ''')) + + parser.add_debug() + parser.add_quiet() + parser.add_version() + + parser.add_argument('source', help='source directory', + type=(lambda x: x.rstrip('/'))) + parser.add_argument('target', help='target directory', + type=(lambda x: x.rstrip('/'))) + + options = parser.parse_args(args) + + return options + +def main(args=None): + '''Efficiently copy a directory tree''' + + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + if not os.path.exists(options.source): + raise QuietError('Source directory %r does not exist' % options.source) + + if os.path.exists(options.target): + raise QuietError('Target directory %r must not yet exist.' % options.target) + + parent = os.path.dirname(os.path.abspath(options.target)) + if not os.path.exists(parent): + raise QuietError('Target parent %r does not exist' % parent) + + fstat_s = os.stat(options.source) + fstat_p = os.stat(parent) + if not stat.S_ISDIR(fstat_s.st_mode): + raise QuietError('Source %r is not a directory' % options.source) + + if not stat.S_ISDIR(fstat_p.st_mode): + raise QuietError('Target parent %r is not a directory' % parent) + + if fstat_p.st_dev != fstat_s.st_dev: + raise QuietError('Source and target are not on the same file system.') + + if os.path.ismount(options.source): + raise QuietError('%s is a mount point.' % options.source) + + ctrlfile = assert_fs_owner(options.source) + try: + os.mkdir(options.target) + except PermissionError: + raise QuietError('No permission to create target directory') + + fstat_t = os.stat(options.target) + llfuse.setxattr(ctrlfile, 'copy', + ('(%d, %d)' % (fstat_s.st_ino, fstat_t.st_ino)).encode()) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/src/s3ql/ctrl.py b/src/s3ql/ctrl.py new file mode 100644 index 0000000..ab89cc8 --- /dev/null +++ b/src/s3ql/ctrl.py @@ -0,0 +1,99 @@ +''' +ctrl.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging, setup_logging +from .common import assert_fs_owner +from .parse_args import ArgumentParser +import llfuse +import sys +import textwrap + +log = logging.getLogger(__name__) + +def parse_args(args): + '''Parse command line''' + + parser = ArgumentParser( + description='''Control a mounted S3QL File System''', + epilog=textwrap.dedent('''\ + Hint: run `%(prog)s --help` to get help on the additional + arguments that the different actions take.''')) + + pparser = ArgumentParser(add_help=False, epilog=textwrap.dedent('''\ + Hint: run `%(prog)s --help` to get help on other available actions and + optional arguments that can be used with all actions.''')) + pparser.add_argument("mountpoint", metavar='', + type=(lambda x: x.rstrip('/')), + help='Mountpoint of the file system') + + parser.add_debug() + parser.add_quiet() + parser.add_version() + + subparsers = parser.add_subparsers(metavar='', dest='action', + help='may be either of') + subparsers.add_parser('flushcache', help='flush file system cache', + parents=[pparser]) + subparsers.add_parser('upload-meta', help='Upload metadata', + parents=[pparser]) + + sparser = subparsers.add_parser('cachesize', help='Change cache size', + parents=[pparser]) + sparser.add_argument('cachesize', metavar='', type=int, + help='New cache size in KiB') + + sparser = subparsers.add_parser('log', help='Change log level', + parents=[pparser]) + + sparser.add_argument('level', choices=('debug', 'info', 'warn'), + metavar='', + help='Desired new log level for mount.s3ql process. ' + 'Allowed values: %(choices)s') + sparser.add_argument('modules', nargs='*', metavar='', + help='Modules to enable debugging output for. Specify ' + '`all` to enable debugging for all modules.') + + options = parser.parse_args(args) + + if options.action == 'log': + if options.level != 'debug' and options.modules: + parser.error('Modules can only be specified with `debug` logging level.') + if not options.modules: + options.modules = [ 'all' ] + + return options + +def main(args=None): + '''Control a mounted S3QL File System.''' + + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + path = options.mountpoint + + ctrlfile = assert_fs_owner(path, mountpoint=True) + + if options.action == 'flushcache': + llfuse.setxattr(ctrlfile, 's3ql_flushcache!', b'dummy') + + if options.action == 'upload-meta': + llfuse.setxattr(ctrlfile, 'upload-meta', b'dummy') + + elif options.action == 'log': + level = getattr(logging, options.level.upper()) + cmd = ('(%r, %r)' % (level, ','.join(options.modules))).encode() + llfuse.setxattr(ctrlfile, 'logging', cmd) + + elif options.action == 'cachesize': + llfuse.setxattr(ctrlfile, 'cachesize', ('%d' % (options.cachesize * 1024,)).encode()) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/src/s3ql/daemonize.py b/src/s3ql/daemonize.py new file mode 100644 index 0000000..eac8a36 --- /dev/null +++ b/src/s3ql/daemonize.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +''' +daemonize.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. + +The functions in this file are based on the python-daemon module by Ben Finney +. + +The reason for not simply using the module instead is that it does a lot of +staff that we don't need (not a real problem) and some stuff that we must not do +(the real problem). + +This main issue is that python-daemon unconditionally closes all open file +descriptors. We don't want this for S3QL, because we have already opened the +database and log files when we daemonize. I think this is good design, because +it allows us to give a meaningful error message to the user if these files +cannot be opened (if we open them after daemonizing, the user will only see a +vanishing daemon process without any indication what went wrong). + +According to “Advanced Programming in the Unix Environment”, the point of +closing all open file descriptors is only to "prevent the daemon from holding +open any descriptors that it may have inherited from its parent (which could be +a shell or some other process)". In this case the user will have to live with +that. +''' + +from .logging import logging # Ensure use of custom logger class +import os +import sys + +log = logging.getLogger(__name__) + +def daemonize(workdir='/'): + '''Daemonize the process''' + + os.chdir(workdir) + + detach_process_context() + + redirect_stream(sys.stdin, None) + redirect_stream(sys.stdout, None) + redirect_stream(sys.stderr, None) + + +def detach_process_context(): + '''Detach the process context from parent and session. + + Detach from the parent process and session group, allowing the parent to + exit while this process continues running. + + Reference: “Advanced Programming in the Unix Environment”, section 13.3, by + W. Richard Stevens, published 1993 by Addison-Wesley. + ''' + + # Protected member + #pylint: disable=W0212 + + pid = os.fork() + if pid > 0: + os._exit(0) + + os.setsid() + + pid = os.fork() + if pid > 0: + log.info('Daemonizing, new PID is %d', pid) + os._exit(0) + + +def redirect_stream(system_stream, target_stream): + '''Redirect *system_stream* to *target_stream* + + `system_stream` is a standard system stream such as + ``sys.stdout``. `target_stream` is an open file object that should replace + the corresponding system stream object. + + If `target_stream` is ``None``, defaults to opening the operating + system's null device and using its file descriptor. + ''' + + if target_stream is None: + target_fd = os.open(os.devnull, os.O_RDWR) + else: + target_fd = target_stream.fileno() + os.dup2(target_fd, system_stream.fileno()) diff --git a/src/s3ql/database.py b/src/s3ql/database.py new file mode 100644 index 0000000..2744701 --- /dev/null +++ b/src/s3ql/database.py @@ -0,0 +1,210 @@ +''' +database.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. + + +Module Attributes: +----------- + +:initsql: SQL commands that are executed whenever a new + connection is created. +''' + +from .logging import logging, QuietError # Ensure use of custom logger class +import apsw +import os + +log = logging.getLogger(__name__) + +sqlite_ver = tuple([ int(x) for x in apsw.sqlitelibversion().split('.') ]) +if sqlite_ver < (3, 7, 0): + raise QuietError('SQLite version too old, must be 3.7.0 or newer!\n') + + +initsql = ( + # WAL mode causes trouble with e.g. copy_tree, so we don't use it at the moment + # (cf. http://article.gmane.org/gmane.comp.db.sqlite.general/65243). + # However, if we start using it we must initiaze it *before* setting + # locking_mode to EXCLUSIVE, otherwise we can't switch the locking + # mode without first disabling WAL. + 'PRAGMA synchronous = OFF', + 'PRAGMA journal_mode = OFF', + #'PRAGMA synchronous = NORMAL', + #'PRAGMA journal_mode = WAL', + + 'PRAGMA foreign_keys = OFF', + 'PRAGMA locking_mode = EXCLUSIVE', + 'PRAGMA recursize_triggers = on', + 'PRAGMA page_size = 4096', + 'PRAGMA wal_autocheckpoint = 25000', + 'PRAGMA temp_store = FILE', + 'PRAGMA legacy_file_format = off', + ) + +class Connection(object): + ''' + This class wraps an APSW connection object. It should be used instead of any + native APSW cursors. + + It provides methods to directly execute SQL commands and creates apsw + cursors dynamically. + + Instances are not thread safe. They can be passed between threads, + but must not be called concurrently. + + Attributes + ---------- + + :conn: apsw connection object + ''' + + def __init__(self, file_): + self.conn = apsw.Connection(file_) + self.file = file_ + + cur = self.conn.cursor() + + for s in initsql: + cur.execute(s) + + def close(self): + self.conn.close() + + def get_size(self): + '''Return size of database file''' + + if self.file is not None and self.file not in ('', ':memory:'): + return os.path.getsize(self.file) + else: + return 0 + + def query(self, *a, **kw): + '''Return iterator over results of given SQL statement + + If the caller does not retrieve all rows the iterator's close() method + should be called as soon as possible to terminate the SQL statement + (otherwise it may block execution of other statements). To this end, + the iterator may also be used as a context manager. + ''' + + return ResultSet(self.conn.cursor().execute(*a, **kw)) + + def execute(self, *a, **kw): + '''Execute the given SQL statement. Return number of affected rows ''' + + self.conn.cursor().execute(*a, **kw) + return self.changes() + + def rowid(self, *a, **kw): + """Execute SQL statement and return last inserted rowid""" + + self.conn.cursor().execute(*a, **kw) + return self.conn.last_insert_rowid() + + def has_val(self, *a, **kw): + '''Execute statement and check if it gives result rows''' + + res = self.conn.cursor().execute(*a, **kw) + try: + next(res) + except StopIteration: + return False + else: + # Finish the active SQL statement + res.close() + return True + + def get_val(self, *a, **kw): + """Execute statement and return first element of first result row. + + If there is no result row, raises `NoSuchRowError`. If there is more + than one row, raises `NoUniqueValueError`. + """ + + return self.get_row(*a, **kw)[0] + + def get_list(self, *a, **kw): + """Execute select statement and returns result list""" + + return list(self.query(*a, **kw)) + + def get_row(self, *a, **kw): + """Execute select statement and return first row. + + If there are no result rows, raises `NoSuchRowError`. If there is more + than one result row, raises `NoUniqueValueError`. + """ + + res = self.conn.cursor().execute(*a, **kw) + try: + row = next(res) + except StopIteration: + raise NoSuchRowError() + try: + next(res) + except StopIteration: + # Fine, we only wanted one row + pass + else: + # Finish the active SQL statement + res.close() + raise NoUniqueValueError() + + return row + + def last_rowid(self): + """Return rowid most recently inserted in the current thread""" + + return self.conn.last_insert_rowid() + + def changes(self): + """Return number of rows affected by most recent sql statement""" + + return self.conn.changes() + + +class NoUniqueValueError(Exception): + '''Raised if get_val or get_row was called with a query + that generated more than one result row. + ''' + + def __str__(self): + return 'Query generated more than 1 result row' + + +class NoSuchRowError(Exception): + '''Raised if the query did not produce any result rows''' + + def __str__(self): + return 'Query produced 0 result rows' + + +class ResultSet(object): + ''' + Provide iteration over encapsulated apsw cursor. Additionally, + `ResultSet` instances may be used as context managers to terminate + the query before all result rows have been retrieved. + ''' + + def __init__(self, cur): + self.cur = cur + + def __next__(self): + return next(self.cur) + + def __iter__(self): + return self + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.cur.close() + + def close(self): + '''Terminate query''' + + self.cur.close() diff --git a/src/s3ql/deltadump.c b/src/s3ql/deltadump.c new file mode 100644 index 0000000..3343aa6 --- /dev/null +++ b/src/s3ql/deltadump.c @@ -0,0 +1,11558 @@ +/* Generated by Cython 0.24.0a0 */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) + #error Cython requires Python 2.6+ or Python 3.2+. +#else +#define CYTHON_ABI "0_24_0a0" +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#ifdef PYPY_VERSION + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 +#endif +#if !defined(CYTHON_USE_PYLONG_INTERNALS) && CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070000 + #define CYTHON_USE_PYLONG_INTERNALS 1 +#endif +#if CYTHON_USE_PYLONG_INTERNALS + #include "longintrepr.h" + #undef SHIFT + #undef BASE + #undef MASK +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) + #define Py_OptimizeFlag 0 +#endif +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#if PY_MAJOR_VERSION < 3 + #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyClass_Type +#else + #define __Pyx_BUILTIN_MODULE_NAME "builtins" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyType_Type +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) + #define CYTHON_PEP393_ENABLED 1 + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) +#else + #define CYTHON_PEP393_ENABLED 0 + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) + #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) + #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) +#endif +#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) +#else + #define __Pyx_PyString_Format(a, b) PyString_Format(a, b) +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBaseString_Type PyUnicode_Type + #define PyStringObject PyUnicodeObject + #define PyString_Type PyUnicode_Type + #define PyString_Check PyUnicode_Check + #define PyString_CheckExact PyUnicode_CheckExact +#endif +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) + #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) +#else + #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) + #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) +#endif +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#if PY_MAJOR_VERSION >= 3 + #define PyIntObject PyLongObject + #define PyInt_Type PyLong_Type + #define PyInt_Check(op) PyLong_Check(op) + #define PyInt_CheckExact(op) PyLong_CheckExact(op) + #define PyInt_FromString PyLong_FromString + #define PyInt_FromUnicode PyLong_FromUnicode + #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t + #define PyInt_FromSsize_t PyLong_FromSsize_t + #define PyInt_AsLong PyLong_AsLong + #define PyInt_AS_LONG PyLong_AS_LONG + #define PyInt_AsSsize_t PyLong_AsSsize_t + #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask + #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask + #define PyNumber_Int PyNumber_Long +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBoolObject PyLongObject +#endif +#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY + #ifndef PyUnicode_InternFromString + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) + #endif +#endif +#if PY_VERSION_HEX < 0x030200A4 + typedef long Py_hash_t; + #define __Pyx_PyInt_FromHash_t PyInt_FromLong + #define __Pyx_PyInt_AsHash_t PyInt_AsLong +#else + #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t + #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t +#endif +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) +#else + #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) +#endif +#if PY_VERSION_HEX >= 0x030500B1 +#define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) +#elif CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; +} __Pyx_PyAsyncMethodsStruct; +#define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) +#else +#define __Pyx_PyType_AsAsync(obj) NULL +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) + +#ifndef CYTHON_INLINE + #if defined(__GNUC__) + #define CYTHON_INLINE __inline__ + #elif defined(_MSC_VER) + #define CYTHON_INLINE __inline + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_INLINE inline + #else + #define CYTHON_INLINE + #endif +#endif + +#if defined(WIN32) || defined(MS_WINDOWS) + #define _USE_MATH_DEFINES +#endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif + + +#define __PYX_ERR(f_index, lineno, Ln_error) \ +{ \ + __pyx_filename = __pyx_f[f_index]; __pyx_lineno = lineno; __pyx_clineno = __LINE__; goto Ln_error; \ +} + +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#else + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#endif + +#ifndef __PYX_EXTERN_C + #ifdef __cplusplus + #define __PYX_EXTERN_C extern "C" + #else + #define __PYX_EXTERN_C extern + #endif +#endif + +#define __PYX_HAVE__s3ql__deltadump +#define __PYX_HAVE_API__s3ql__deltadump +#include "string.h" +#include "stdio.h" +#include "errno.h" +#include "stdlib.h" +#include "stdint.h" +#include "sys/types.h" +#include "unistd.h" +#include "endian_indep.h" +#include "sqlite3.h" +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#ifdef PYREX_WITHOUT_ASSERTIONS +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; + const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) && defined (_M_X64) + #define __Pyx_sst_abs(value) _abs64(value) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +#define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if PY_MAJOR_VERSION < 3 + #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#else + #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize +#endif +#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#if PY_MAJOR_VERSION < 3 +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) +{ + const Py_UNICODE *u_end = u; + while (*u_end++) ; + return (size_t)(u_end - u - 1); +} +#else +#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen +#endif +#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) +#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +#define __Pyx_NewRef(obj) (Py_INCREF(obj), obj) +#define __Pyx_Owned_Py_None(b) __Pyx_NewRef(Py_None) +#define __Pyx_PyBool_FromLong(b) ((b) ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False)) +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); +#if CYTHON_COMPILING_IN_CPYTHON +#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#else +#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#endif +#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) +#if PY_MAJOR_VERSION >= 3 +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#else +#define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) +#endif +#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII +static int __Pyx_sys_getdefaultencoding_not_ascii; +static int __Pyx_init_sys_getdefaultencoding_params(void) { + PyObject* sys; + PyObject* default_encoding = NULL; + PyObject* ascii_chars_u = NULL; + PyObject* ascii_chars_b = NULL; + const char* default_encoding_c; + sys = PyImport_ImportModule("sys"); + if (!sys) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL); + Py_DECREF(sys); + if (!default_encoding) goto bad; + default_encoding_c = PyBytes_AsString(default_encoding); + if (!default_encoding_c) goto bad; + if (strcmp(default_encoding_c, "ascii") == 0) { + __Pyx_sys_getdefaultencoding_not_ascii = 0; + } else { + char ascii_chars[128]; + int c; + for (c = 0; c < 128; c++) { + ascii_chars[c] = c; + } + __Pyx_sys_getdefaultencoding_not_ascii = 1; + ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); + if (!ascii_chars_u) goto bad; + ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); + if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { + PyErr_Format( + PyExc_ValueError, + "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.", + default_encoding_c); + goto bad; + } + Py_DECREF(ascii_chars_u); + Py_DECREF(ascii_chars_b); + } + Py_DECREF(default_encoding); + return 0; +bad: + Py_XDECREF(default_encoding); + Py_XDECREF(ascii_chars_u); + Py_XDECREF(ascii_chars_b); + return -1; +} +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#else +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT +static char* __PYX_DEFAULT_STRING_ENCODING; +static int __Pyx_init_sys_getdefaultencoding_params(void) { + PyObject* sys; + PyObject* default_encoding = NULL; + char* default_encoding_c; + sys = PyImport_ImportModule("sys"); + if (!sys) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); + Py_DECREF(sys); + if (!default_encoding) goto bad; + default_encoding_c = PyBytes_AsString(default_encoding); + if (!default_encoding_c) goto bad; + __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); + if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; + strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); + Py_DECREF(default_encoding); + return 0; +bad: + Py_XDECREF(default_encoding); + return -1; +} +#endif +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ + +static PyObject *__pyx_m; +static PyObject *__pyx_d; +static PyObject *__pyx_b; +static PyObject *__pyx_empty_tuple; +static PyObject *__pyx_empty_bytes; +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * __pyx_cfilenm= __FILE__; +static const char *__pyx_filename; + + +static const char *__pyx_f[] = { + "src/s3ql/deltadump.pyx", + "type.pxd", +}; + +/*--- Type declarations ---*/ +struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table; +struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table; +struct __pyx_opt_args_4s3ql_9deltadump_raise_from_errno; + +/* "s3ql/deltadump.pyx":136 + * return None + * + * cdef int raise_from_errno(err_class=OSError) except -1: # <<<<<<<<<<<<<< + * '''Raise OSError for current errno value''' + * + */ +struct __pyx_opt_args_4s3ql_9deltadump_raise_from_errno { + int __pyx_n; + PyObject *err_class; +}; + +/* "s3ql/deltadump.pyx":280 + * s3ql_sqlite_options - apsw_sqlite_options)) + * + * def dump_table(table, order, columns, db, fh): # <<<<<<<<<<<<<< + * '''Dump *columns* of *table* into *fh* + * + */ +struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table { + PyObject_HEAD + int *__pyx_v_col_args; + int *__pyx_v_col_types; + FILE *__pyx_v_fp; + int64_t *__pyx_v_int64_prev; + sqlite3 *__pyx_v_sqlite3_db; + sqlite3_stmt *__pyx_v_stmt; +}; + + +/* "s3ql/deltadump.pyx":401 + * fwrite(buf, len_, fp) + * + * def load_table(table, columns, db, fh, trx_rows=5000): # <<<<<<<<<<<<<< + * '''Load *columns* of *table* from *fh* + * + */ +struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table { + PyObject_HEAD + sqlite3_stmt *__pyx_v_begin_stmt; + void *__pyx_v_buf; + int *__pyx_v_col_args; + int *__pyx_v_col_types; + sqlite3_stmt *__pyx_v_commit_stmt; + FILE *__pyx_v_fp; + int64_t *__pyx_v_int64_prev; + sqlite3 *__pyx_v_sqlite3_db; + sqlite3_stmt *__pyx_v_stmt; +}; + + +/* --- Runtime support code (head) --- */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, int); + void (*DECREF)(void*, PyObject*, int); + void (*GOTREF)(void*, PyObject*, int); + void (*GIVEREF)(void*, PyObject*, int); + void* (*SetupContext)(const char*, int, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; +#ifdef WITH_THREAD + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ + } +#else + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) +#endif + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); +#if PY_MAJOR_VERSION < 3 + if (likely(tp->tp_getattr)) + return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); +#endif + return PyObject_GetAttr(obj, attr_name); +} +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = PyThreadState_GET(); +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#endif + +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + +#define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Fast(o, (Py_ssize_t)i, is_list, wraparound, boundscheck) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL) :\ + __Pyx_GetItemInt_Generic(o, to_py_func(i)))) +#define __Pyx_GetItemInt_List(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_List_Fast(o, (Py_ssize_t)i, wraparound, boundscheck) :\ + (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck); +#define __Pyx_GetItemInt_Tuple(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Tuple_Fast(o, (Py_ssize_t)i, wraparound, boundscheck) :\ + (PyErr_SetString(PyExc_IndexError, "tuple index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck); +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, + int is_list, int wraparound, int boundscheck); + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); +#else +#define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL) +#endif + +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ + PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ + const char* function_name); + +#include + +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals); + +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals); + +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070000 +static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name) { + PyObject *res; + PyTypeObject *tp = Py_TYPE(obj); +#if PY_MAJOR_VERSION < 3 + if (unlikely(PyInstance_Check(obj))) + return __Pyx_PyObject_GetAttrStr(obj, attr_name); +#endif + res = _PyType_Lookup(tp, attr_name); + if (likely(res)) { + descrgetfunc f = Py_TYPE(res)->tp_descr_get; + if (!f) { + Py_INCREF(res); + } else { + res = f(res, obj, (PyObject *)tp); + } + } else { + PyErr_SetObject(PyExc_AttributeError, attr_name); + } + return res; +} +#else +#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +#endif + +static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); + +#define __Pyx_CyFunction_USED 1 +#include +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CyFunction_GetClosure(f)\ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#define __Pyx_CyFunction_GetClassObj(f)\ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#define __Pyx_CyFunction_Defaults(type, f)\ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { + PyCFunctionObject func; +#if PY_VERSION_HEX < 0x030500A0 + PyObject *func_weakreflist; +#endif + PyObject *func_dict; + PyObject *func_name; + PyObject *func_qualname; + PyObject *func_doc; + PyObject *func_globals; + PyObject *func_code; + PyObject *func_closure; + PyObject *func_classobj; + void *defaults; + int defaults_pyobjects; + int flags; + PyObject *defaults_tuple; + PyObject *defaults_kwdict; + PyObject *(*defaults_getter)(PyObject *); + PyObject *func_annotations; +} __pyx_CyFunctionObject; +static PyTypeObject *__pyx_CyFunctionType = 0; +#define __Pyx_CyFunction_NewEx(ml, flags, qualname, self, module, globals, code)\ + __Pyx_CyFunction_New(__pyx_CyFunctionType, ml, flags, qualname, self, module, globals, code) +static PyObject *__Pyx_CyFunction_New(PyTypeObject *, PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *self, + PyObject *module, PyObject *globals, + PyObject* code); +static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *m, + size_t size, + int pyobjects); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, + PyObject *dict); +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, + PyObject *dict); +static int __pyx_CyFunction_init(void); + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { + PyListObject* L = (PyListObject*) list; + Py_ssize_t len = Py_SIZE(list); + if (likely(L->allocated > len)) { + Py_INCREF(x); + PyList_SET_ITEM(list, len, x); + Py_SIZE(list) = len+1; + return 0; + } + return PyList_Append(list, x); +} +#else +#define __Pyx_ListComp_Append(L,x) PyList_Append(L,x) +#endif + +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); +#endif + +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); +#endif + +#if CYTHON_COMPILING_IN_CPYTHON +static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, long intval, int inplace); +#else +#define __Pyx_PyInt_EqObjC(op1, op2, intval, inplace)\ + PyObject_RichCompare(op1, op2, Py_EQ) + #endif + +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); + +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); + +typedef struct { + int code_line; + PyCodeObject* code_object; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; +}; +static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static PyCodeObject *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); + +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); + +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); + +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int64_t(int64_t value); + +static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); + +static CYTHON_INLINE int64_t __Pyx_PyInt_As_int64_t(PyObject *); + +static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *); + +static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); + +static int __Pyx_check_binary_version(void); + +#if !defined(__Pyx_PyIdentifier_FromString) +#if PY_MAJOR_VERSION < 3 + #define __Pyx_PyIdentifier_FromString(s) PyString_FromString(s) +#else + #define __Pyx_PyIdentifier_FromString(s) PyUnicode_FromString(s) +#endif +#endif + +static PyObject *__Pyx_ImportModule(const char *name); + +static PyTypeObject *__Pyx_ImportType(const char *module_name, const char *class_name, size_t size, int strict); + +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); + + +/* Module declarations from 'cpython.long' */ + +/* Module declarations from 'libc.string' */ + +/* Module declarations from 'libc.stdio' */ + +/* Module declarations from '__builtin__' */ + +/* Module declarations from 'cpython.type' */ +static PyTypeObject *__pyx_ptype_7cpython_4type_type = 0; + +/* Module declarations from 'cpython' */ + +/* Module declarations from 'cpython.object' */ + +/* Module declarations from 'cpython.exc' */ + +/* Module declarations from 'libc.errno' */ + +/* Module declarations from 'libc.stdlib' */ + +/* Module declarations from 'libc.stdint' */ + +/* Module declarations from 'posix.types' */ + +/* Module declarations from 'posix.unistd' */ + +/* Module declarations from 's3ql.deltadump' */ +static PyTypeObject *__pyx_ptype_4s3ql_9deltadump___pyx_scope_struct__dump_table = 0; +static PyTypeObject *__pyx_ptype_4s3ql_9deltadump___pyx_scope_struct_1_load_table = 0; +static int __pyx_v_4s3ql_9deltadump__INTEGER; +static int __pyx_v_4s3ql_9deltadump__BLOB; +static int __pyx_v_4s3ql_9deltadump__TIME; +static uint8_t __pyx_v_4s3ql_9deltadump_INT8; +static uint8_t __pyx_v_4s3ql_9deltadump_INT16; +static uint8_t __pyx_v_4s3ql_9deltadump_INT32; +static uint8_t __pyx_v_4s3ql_9deltadump_INT64; +static double __pyx_v_4s3ql_9deltadump_time_scale; +static CYTHON_INLINE int __pyx_f_4s3ql_9deltadump_fwrite(const void *, size_t, FILE *); /*proto*/ +static CYTHON_INLINE int __pyx_f_4s3ql_9deltadump_fread(void *, size_t, FILE *); /*proto*/ +static PyObject *__pyx_f_4s3ql_9deltadump_free(void *); /*proto*/ +static int __pyx_f_4s3ql_9deltadump_raise_from_errno(struct __pyx_opt_args_4s3ql_9deltadump_raise_from_errno *__pyx_optional_args); /*proto*/ +static int __pyx_f_4s3ql_9deltadump_fclose(FILE *); /*proto*/ +static void *__pyx_f_4s3ql_9deltadump_calloc(size_t, size_t); /*proto*/ +static int __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(int, int, sqlite3 *); /*proto*/ +static int __pyx_f_4s3ql_9deltadump_prep_columns(PyObject *, int **, int **); /*proto*/ +static FILE *__pyx_f_4s3ql_9deltadump_dup_to_fp(PyObject *, const char *); /*proto*/ +static CYTHON_INLINE int __pyx_f_4s3ql_9deltadump_write_integer(int64_t, FILE *); /*proto*/ +static CYTHON_INLINE int __pyx_f_4s3ql_9deltadump_read_integer(int64_t *, FILE *); /*proto*/ +#define __Pyx_MODULE_NAME "s3ql.deltadump" +int __pyx_module_is_main_s3ql__deltadump = 0; + +/* Implementation of 's3ql.deltadump' */ +static PyObject *__pyx_builtin_OSError; +static PyObject *__pyx_builtin_IOError; +static PyObject *__pyx_builtin_range; +static PyObject *__pyx_builtin_ValueError; +static PyObject *__pyx_builtin_RuntimeError; +static const char __pyx_k_i[] = "i"; +static const char __pyx_k_j[] = "j"; +static const char __pyx_k__4[] = ", "; +static const char __pyx_k__9[] = "?"; +static const char __pyx_k_cm[] = "cm"; +static const char __pyx_k_db[] = "db"; +static const char __pyx_k_fh[] = "fh"; +static const char __pyx_k_fp[] = "fp"; +static const char __pyx_k_os[] = "os"; +static const char __pyx_k_rc[] = "rc"; +static const char __pyx_k_buf[] = "buf"; +static const char __pyx_k_cmd[] = "cmd"; +static const char __pyx_k_idx[] = "idx"; +static const char __pyx_k_len[] = "len_"; +static const char __pyx_k_log[] = "log"; +static const char __pyx_k_sys[] = "sys"; +static const char __pyx_k_tmp[] = "tmp"; +static const char __pyx_k_val[] = "val"; +static const char __pyx_k_BLOB[] = "BLOB"; +static const char __pyx_k_TIME[] = "TIME"; +static const char __pyx_k_apsw[] = "apsw"; +static const char __pyx_k_exit[] = "__exit__"; +static const char __pyx_k_file[] = "file"; +static const char __pyx_k_main[] = "__main__"; +static const char __pyx_k_name[] = "__name__"; +static const char __pyx_k_stmt[] = "stmt"; +static const char __pyx_k_test[] = "__test__"; +static const char __pyx_k_count[] = "count"; +static const char __pyx_k_debug[] = "debug"; +static const char __pyx_k_enter[] = "__enter__"; +static const char __pyx_k_int64[] = "int64"; +static const char __pyx_k_order[] = "order"; +static const char __pyx_k_query[] = "query"; +static const char __pyx_k_range[] = "range"; +static const char __pyx_k_table[] = "table"; +static const char __pyx_k_encode[] = "encode"; +static const char __pyx_k_fileno[] = "fileno"; +static const char __pyx_k_import[] = "__import__"; +static const char __pyx_k_memory[] = ":memory:"; +static const char __pyx_k_pragma[] = "pragma"; +static const char __pyx_k_INTEGER[] = "INTEGER"; +static const char __pyx_k_IOError[] = "IOError"; +static const char __pyx_k_OSError[] = "OSError"; +static const char __pyx_k_columns[] = "columns"; +static const char __pyx_k_get_val[] = "get_val"; +static const char __pyx_k_logging[] = "logging"; +static const char __pyx_k_PRAGMA_s[] = "PRAGMA %s"; +static const char __pyx_k_callback[] = "callback"; +static const char __pyx_k_col_args[] = "col_args"; +static const char __pyx_k_dbfile_b[] = "dbfile_b"; +static const char __pyx_k_trx_rows[] = "trx_rows"; +static const char __pyx_k_ExitStack[] = "ExitStack"; +static const char __pyx_k_col_count[] = "col_count"; +static const char __pyx_k_col_names[] = "col_names"; +static const char __pyx_k_col_types[] = "col_types"; +static const char __pyx_k_getLogger[] = "getLogger"; +static const char __pyx_k_itertools[] = "itertools"; +static const char __pyx_k_row_count[] = "row_count"; +static const char __pyx_k_PRAGMA_s_s[] = "PRAGMA %s = %s"; +static const char __pyx_k_ValueError[] = "ValueError"; +static const char __pyx_k_begin_stmt[] = "begin_stmt"; +static const char __pyx_k_contextlib[] = "contextlib"; +static const char __pyx_k_dump_table[] = "dump_table"; +static const char __pyx_k_int64_prev[] = "int64_prev"; +static const char __pyx_k_load_table[] = "load_table"; +static const char __pyx_k_sqlite3_db[] = "sqlite3_db"; +static const char __pyx_k_commit_stmt[] = "commit_stmt"; +static const char __pyx_k_synchronous[] = "synchronous"; +static const char __pyx_k_RuntimeError[] = "RuntimeError"; +static const char __pyx_k_check_sqlite[] = "check_sqlite"; +static const char __pyx_k_exceptionfor[] = "exceptionfor"; +static const char __pyx_k_foreign_keys[] = "foreign_keys"; +static const char __pyx_k_MAX_BLOB_SIZE[] = "MAX_BLOB_SIZE"; +static const char __pyx_k_s3ql_deltadump[] = "s3ql.deltadump"; +static const char __pyx_k_Corrupted_input[] = "Corrupted input"; +static const char __pyx_k_compile_options[] = "compile_options"; +static const char __pyx_k_surrogateescape[] = "surrogateescape"; +static const char __pyx_k_sqlitelibversion[] = "sqlitelibversion"; +static const char __pyx_k_BEGIN_TRANSACTION[] = "BEGIN TRANSACTION"; +static const char __pyx_k_COMMIT_TRANSACTION[] = "COMMIT TRANSACTION"; +static const char __pyx_k_apsw_sqlite_options[] = "apsw_sqlite_options"; +static const char __pyx_k_apsw_sqlite_version[] = "apsw_sqlite_version"; +static const char __pyx_k_s3ql_sqlite_options[] = "s3ql_sqlite_options"; +static const char __pyx_k_s3ql_sqlite_version[] = "s3ql_sqlite_version"; +static const char __pyx_k_getfilesystemencoding[] = "getfilesystemencoding"; +static const char __pyx_k_Can_t_dump_NULL_values[] = "Can't dump NULL values"; +static const char __pyx_k_Length_d_d_in_column_d[] = "Length %d != %d in column %d"; +static const char __pyx_k_Opening_connection_to_s[] = "Opening connection to %s"; +static const char __pyx_k_INSERT_INTO_s_s_VALUES_s[] = "INSERT INTO %s (%s) VALUES(%s)"; +static const char __pyx_k_dump_table_locals_lambda[] = "dump_table.."; +static const char __pyx_k_load_table_locals_lambda[] = "load_table.."; +static const char __pyx_k_Invalid_type_for_column_d[] = "Invalid type for column %d"; +static const char __pyx_k_SELECT_COUNT_rowid_FROM_s[] = "SELECT COUNT(rowid) FROM %s"; +static const char __pyx_k_SELECT_s_FROM_s_ORDER_BY_s[] = "SELECT %s FROM %s ORDER BY %s "; +static const char __pyx_k_dump_table_s_writing_d_rows[] = "dump_table(%s): writing %d rows"; +static const char __pyx_k_load_table_s_reading_d_rows[] = "load_table(%s): reading %d rows"; +static const char __pyx_k_BLOB_too_large_to_read_d_vs_d[] = "BLOB too large to read (%d vs %d)"; +static const char __pyx_k_home_nikratio_in_progress_s3ql[] = "/home/nikratio/in-progress/s3ql/src/s3ql/deltadump.pyx"; +static const char __pyx_k_Can_not_dump_BLOB_of_size_d_max[] = "Can not dump BLOB of size %d (max: %d)"; +static const char __pyx_k_SQLite_version_mismatch_between[] = "SQLite version mismatch between APSW and S3QL (%s vs %s)"; +static const char __pyx_k_deltadump_pyx_this_file_is_part[] = "\ndeltadump.pyx - this file is part of S3QL (http://s3ql.googlecode.com)\n\nCopyright \302\251 2008 Nikolaus Rath \n\nThis program can be distributed under the terms of the GNU GPLv3.\n"; +static const char __pyx_k_Can_t_access_in_memory_databases[] = "Can't access in-memory databases"; +static const char __pyx_k_SQLite_code_used_by_APSW_was_com[] = "SQLite code used by APSW was compiled with different options than SQLite code available to S3QL! Differing settings: + %s, - %s"; +static const char __pyx_k_apsw_sqlite_compile_options_s_s3[] = "apsw sqlite compile options: %s, s3ql sqlite compile options: %s"; +static const char __pyx_k_apsw_sqlite_version_s_s3ql_sqlit[] = "apsw sqlite version: %s, s3ql sqlite version: %s"; +static PyObject *__pyx_kp_b_BEGIN_TRANSACTION; +static PyObject *__pyx_n_s_BLOB; +static PyObject *__pyx_kp_u_BLOB_too_large_to_read_d_vs_d; +static PyObject *__pyx_kp_b_COMMIT_TRANSACTION; +static PyObject *__pyx_kp_u_Can_not_dump_BLOB_of_size_d_max; +static PyObject *__pyx_kp_u_Can_t_access_in_memory_databases; +static PyObject *__pyx_kp_u_Can_t_dump_NULL_values; +static PyObject *__pyx_kp_u_Corrupted_input; +static PyObject *__pyx_n_s_ExitStack; +static PyObject *__pyx_kp_u_INSERT_INTO_s_s_VALUES_s; +static PyObject *__pyx_n_s_INTEGER; +static PyObject *__pyx_n_s_IOError; +static PyObject *__pyx_kp_u_Invalid_type_for_column_d; +static PyObject *__pyx_kp_u_Length_d_d_in_column_d; +static PyObject *__pyx_n_s_MAX_BLOB_SIZE; +static PyObject *__pyx_n_s_OSError; +static PyObject *__pyx_kp_u_Opening_connection_to_s; +static PyObject *__pyx_kp_u_PRAGMA_s; +static PyObject *__pyx_kp_u_PRAGMA_s_s; +static PyObject *__pyx_n_s_RuntimeError; +static PyObject *__pyx_kp_u_SELECT_COUNT_rowid_FROM_s; +static PyObject *__pyx_kp_u_SELECT_s_FROM_s_ORDER_BY_s; +static PyObject *__pyx_kp_u_SQLite_code_used_by_APSW_was_com; +static PyObject *__pyx_kp_u_SQLite_version_mismatch_between; +static PyObject *__pyx_n_s_TIME; +static PyObject *__pyx_n_s_ValueError; +static PyObject *__pyx_kp_u__4; +static PyObject *__pyx_kp_u__9; +static PyObject *__pyx_n_s_apsw; +static PyObject *__pyx_kp_u_apsw_sqlite_compile_options_s_s3; +static PyObject *__pyx_n_s_apsw_sqlite_options; +static PyObject *__pyx_n_s_apsw_sqlite_version; +static PyObject *__pyx_kp_u_apsw_sqlite_version_s_s3ql_sqlit; +static PyObject *__pyx_n_s_begin_stmt; +static PyObject *__pyx_n_s_buf; +static PyObject *__pyx_n_s_callback; +static PyObject *__pyx_n_s_check_sqlite; +static PyObject *__pyx_n_s_cm; +static PyObject *__pyx_n_s_cmd; +static PyObject *__pyx_n_s_col_args; +static PyObject *__pyx_n_s_col_count; +static PyObject *__pyx_n_s_col_names; +static PyObject *__pyx_n_s_col_types; +static PyObject *__pyx_n_s_columns; +static PyObject *__pyx_n_s_commit_stmt; +static PyObject *__pyx_n_s_compile_options; +static PyObject *__pyx_n_s_contextlib; +static PyObject *__pyx_n_s_count; +static PyObject *__pyx_n_s_db; +static PyObject *__pyx_n_s_dbfile_b; +static PyObject *__pyx_n_s_debug; +static PyObject *__pyx_n_s_dump_table; +static PyObject *__pyx_n_s_dump_table_locals_lambda; +static PyObject *__pyx_kp_u_dump_table_s_writing_d_rows; +static PyObject *__pyx_n_s_encode; +static PyObject *__pyx_n_s_enter; +static PyObject *__pyx_n_s_exceptionfor; +static PyObject *__pyx_n_s_exit; +static PyObject *__pyx_n_s_fh; +static PyObject *__pyx_n_s_file; +static PyObject *__pyx_n_s_fileno; +static PyObject *__pyx_n_u_foreign_keys; +static PyObject *__pyx_n_s_fp; +static PyObject *__pyx_n_s_getLogger; +static PyObject *__pyx_n_s_get_val; +static PyObject *__pyx_n_s_getfilesystemencoding; +static PyObject *__pyx_kp_s_home_nikratio_in_progress_s3ql; +static PyObject *__pyx_n_s_i; +static PyObject *__pyx_n_s_idx; +static PyObject *__pyx_n_s_import; +static PyObject *__pyx_n_s_int64; +static PyObject *__pyx_n_s_int64_prev; +static PyObject *__pyx_n_s_itertools; +static PyObject *__pyx_n_s_j; +static PyObject *__pyx_n_s_len; +static PyObject *__pyx_n_s_load_table; +static PyObject *__pyx_n_s_load_table_locals_lambda; +static PyObject *__pyx_kp_u_load_table_s_reading_d_rows; +static PyObject *__pyx_n_s_log; +static PyObject *__pyx_n_s_logging; +static PyObject *__pyx_n_s_main; +static PyObject *__pyx_kp_u_memory; +static PyObject *__pyx_n_s_name; +static PyObject *__pyx_n_s_order; +static PyObject *__pyx_n_s_os; +static PyObject *__pyx_n_s_pragma; +static PyObject *__pyx_n_s_query; +static PyObject *__pyx_n_s_range; +static PyObject *__pyx_n_s_rc; +static PyObject *__pyx_n_s_row_count; +static PyObject *__pyx_n_s_s3ql_deltadump; +static PyObject *__pyx_n_s_s3ql_sqlite_options; +static PyObject *__pyx_n_s_s3ql_sqlite_version; +static PyObject *__pyx_n_s_sqlite3_db; +static PyObject *__pyx_n_s_sqlitelibversion; +static PyObject *__pyx_n_s_stmt; +static PyObject *__pyx_n_u_surrogateescape; +static PyObject *__pyx_n_u_synchronous; +static PyObject *__pyx_n_s_sys; +static PyObject *__pyx_n_s_table; +static PyObject *__pyx_n_s_test; +static PyObject *__pyx_n_s_tmp; +static PyObject *__pyx_n_s_trx_rows; +static PyObject *__pyx_n_s_val; +static PyObject *__pyx_pf_4s3ql_9deltadump_check_sqlite(CYTHON_UNUSED PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda1(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda2(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda3(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda4(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda5(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_pf_4s3ql_9deltadump_2dump_table(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_table, PyObject *__pyx_v_order, PyObject *__pyx_v_columns, PyObject *__pyx_v_db, PyObject *__pyx_v_fh); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda6(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda7(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda8(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda9(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda10(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda11(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda12(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda13(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda14(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_lambda_funcdef_lambda15(PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_pf_4s3ql_9deltadump_4load_table(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_table, PyObject *__pyx_v_columns, PyObject *__pyx_v_db, PyObject *__pyx_v_fh, PyObject *__pyx_v_trx_rows); /* proto */ +static PyObject *__pyx_tp_new_4s3ql_9deltadump___pyx_scope_struct__dump_table(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_4s3ql_9deltadump___pyx_scope_struct_1_load_table(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_int_0; +static PyObject *__pyx_int_4096; +static PyObject *__pyx_int_5000; +static PyObject *__pyx_k_; +static PyObject *__pyx_tuple__2; +static PyObject *__pyx_tuple__3; +static PyObject *__pyx_tuple__5; +static PyObject *__pyx_tuple__6; +static PyObject *__pyx_tuple__7; +static PyObject *__pyx_tuple__8; +static PyObject *__pyx_tuple__10; +static PyObject *__pyx_tuple__11; +static PyObject *__pyx_tuple__12; +static PyObject *__pyx_tuple__14; +static PyObject *__pyx_tuple__16; +static PyObject *__pyx_codeobj__13; +static PyObject *__pyx_codeobj__15; +static PyObject *__pyx_codeobj__17; + +/* "s3ql/deltadump.pyx":112 + * cdef double time_scale = 1 << 30 + * + * cdef inline int fwrite(const_void * buf, size_t len_, FILE * fp) except -1: # <<<<<<<<<<<<<< + * '''Call libc's fwrite() and raise exception on failure''' + * + */ + +static CYTHON_INLINE int __pyx_f_4s3ql_9deltadump_fwrite(const void *__pyx_v_buf, size_t __pyx_v_len_, FILE *__pyx_v_fp) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + struct __pyx_opt_args_4s3ql_9deltadump_raise_from_errno __pyx_t_3; + __Pyx_RefNannySetupContext("fwrite", 0); + + /* "s3ql/deltadump.pyx":115 + * '''Call libc's fwrite() and raise exception on failure''' + * + * if fwrite_c(buf, len_, 1, fp) != 1: # <<<<<<<<<<<<<< + * raise_from_errno(IOError) + * return 0 + */ + __pyx_t_1 = ((fwrite(__pyx_v_buf, __pyx_v_len_, 1, __pyx_v_fp) != 1) != 0); + if (__pyx_t_1) { + + /* "s3ql/deltadump.pyx":116 + * + * if fwrite_c(buf, len_, 1, fp) != 1: + * raise_from_errno(IOError) # <<<<<<<<<<<<<< + * return 0 + * + */ + __pyx_t_3.__pyx_n = 1; + __pyx_t_3.err_class = __pyx_builtin_IOError; + __pyx_t_2 = __pyx_f_4s3ql_9deltadump_raise_from_errno(&__pyx_t_3); if (unlikely(__pyx_t_2 == -1)) __PYX_ERR(0, 116, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":115 + * '''Call libc's fwrite() and raise exception on failure''' + * + * if fwrite_c(buf, len_, 1, fp) != 1: # <<<<<<<<<<<<<< + * raise_from_errno(IOError) + * return 0 + */ + } + + /* "s3ql/deltadump.pyx":117 + * if fwrite_c(buf, len_, 1, fp) != 1: + * raise_from_errno(IOError) + * return 0 # <<<<<<<<<<<<<< + * + * cdef inline int fread(void * buf, size_t len_, FILE * fp) except -1: + */ + __pyx_r = 0; + goto __pyx_L0; + + /* "s3ql/deltadump.pyx":112 + * cdef double time_scale = 1 << 30 + * + * cdef inline int fwrite(const_void * buf, size_t len_, FILE * fp) except -1: # <<<<<<<<<<<<<< + * '''Call libc's fwrite() and raise exception on failure''' + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("s3ql.deltadump.fwrite", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":119 + * return 0 + * + * cdef inline int fread(void * buf, size_t len_, FILE * fp) except -1: # <<<<<<<<<<<<<< + * '''Call libc's fread() and raise exception on failure''' + * + */ + +static CYTHON_INLINE int __pyx_f_4s3ql_9deltadump_fread(void *__pyx_v_buf, size_t __pyx_v_len_, FILE *__pyx_v_fp) { + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + struct __pyx_opt_args_4s3ql_9deltadump_raise_from_errno __pyx_t_3; + __Pyx_RefNannySetupContext("fread", 0); + + /* "s3ql/deltadump.pyx":122 + * '''Call libc's fread() and raise exception on failure''' + * + * if fread_c(buf, len_, 1, fp) != 1: # <<<<<<<<<<<<<< + * raise_from_errno(IOError) + * return 0 + */ + __pyx_t_1 = ((fread(__pyx_v_buf, __pyx_v_len_, 1, __pyx_v_fp) != 1) != 0); + if (__pyx_t_1) { + + /* "s3ql/deltadump.pyx":123 + * + * if fread_c(buf, len_, 1, fp) != 1: + * raise_from_errno(IOError) # <<<<<<<<<<<<<< + * return 0 + * + */ + __pyx_t_3.__pyx_n = 1; + __pyx_t_3.err_class = __pyx_builtin_IOError; + __pyx_t_2 = __pyx_f_4s3ql_9deltadump_raise_from_errno(&__pyx_t_3); if (unlikely(__pyx_t_2 == -1)) __PYX_ERR(0, 123, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":122 + * '''Call libc's fread() and raise exception on failure''' + * + * if fread_c(buf, len_, 1, fp) != 1: # <<<<<<<<<<<<<< + * raise_from_errno(IOError) + * return 0 + */ + } + + /* "s3ql/deltadump.pyx":124 + * if fread_c(buf, len_, 1, fp) != 1: + * raise_from_errno(IOError) + * return 0 # <<<<<<<<<<<<<< + * + * cdef free(void * ptr): + */ + __pyx_r = 0; + goto __pyx_L0; + + /* "s3ql/deltadump.pyx":119 + * return 0 + * + * cdef inline int fread(void * buf, size_t len_, FILE * fp) except -1: # <<<<<<<<<<<<<< + * '''Call libc's fread() and raise exception on failure''' + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("s3ql.deltadump.fread", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":126 + * return 0 + * + * cdef free(void * ptr): # <<<<<<<<<<<<<< + * '''Call libc.free() + * + */ + +static PyObject *__pyx_f_4s3ql_9deltadump_free(void *__pyx_v_ptr) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("free", 0); + + /* "s3ql/deltadump.pyx":133 + * ''' + * + * free_c(ptr) # <<<<<<<<<<<<<< + * return None + * + */ + free(__pyx_v_ptr); + + /* "s3ql/deltadump.pyx":134 + * + * free_c(ptr) + * return None # <<<<<<<<<<<<<< + * + * cdef int raise_from_errno(err_class=OSError) except -1: + */ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(Py_None); + __pyx_r = Py_None; + goto __pyx_L0; + + /* "s3ql/deltadump.pyx":126 + * return 0 + * + * cdef free(void * ptr): # <<<<<<<<<<<<<< + * '''Call libc.free() + * + */ + + /* function exit code */ + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":136 + * return None + * + * cdef int raise_from_errno(err_class=OSError) except -1: # <<<<<<<<<<<<<< + * '''Raise OSError for current errno value''' + * + */ + +static int __pyx_f_4s3ql_9deltadump_raise_from_errno(struct __pyx_opt_args_4s3ql_9deltadump_raise_from_errno *__pyx_optional_args) { + PyObject *__pyx_v_err_class = __pyx_k_; + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + Py_ssize_t __pyx_t_6; + PyObject *__pyx_t_7 = NULL; + __Pyx_RefNannySetupContext("raise_from_errno", 0); + if (__pyx_optional_args) { + if (__pyx_optional_args->__pyx_n > 0) { + __pyx_v_err_class = __pyx_optional_args->err_class; + } + } + + /* "s3ql/deltadump.pyx":139 + * '''Raise OSError for current errno value''' + * + * raise err_class(errno, PyUnicode_FromString(strerror(errno))) # <<<<<<<<<<<<<< + * + * cdef int fclose(FILE * fp) except -1: + */ + __pyx_t_2 = __Pyx_PyInt_From_int(errno); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 139, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyUnicode_FromString(strerror(errno)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 139, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_INCREF(__pyx_v_err_class); + __pyx_t_4 = __pyx_v_err_class; __pyx_t_5 = NULL; + __pyx_t_6 = 0; + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_4); + if (likely(__pyx_t_5)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_5); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_4, function); + __pyx_t_6 = 1; + } + } + __pyx_t_7 = PyTuple_New(2+__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 139, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (__pyx_t_5) { + __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5); __pyx_t_5 = NULL; + } + __Pyx_GIVEREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_7, 0+__pyx_t_6, __pyx_t_2); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_7, 1+__pyx_t_6, __pyx_t_3); + __pyx_t_2 = 0; + __pyx_t_3 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 139, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(0, 139, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":136 + * return None + * + * cdef int raise_from_errno(err_class=OSError) except -1: # <<<<<<<<<<<<<< + * '''Raise OSError for current errno value''' + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_AddTraceback("s3ql.deltadump.raise_from_errno", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":141 + * raise err_class(errno, PyUnicode_FromString(strerror(errno))) + * + * cdef int fclose(FILE * fp) except -1: # <<<<<<<<<<<<<< + * '''Call libc.fclose() and raise exception on failure''' + * + */ + +static int __pyx_f_4s3ql_9deltadump_fclose(FILE *__pyx_v_fp) { + Py_ssize_t __pyx_v_off; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + __Pyx_RefNannySetupContext("fclose", 0); + + /* "s3ql/deltadump.pyx":149 + * # important, so that we can safely reposition the fd position + * # below (which is necessary in case there is cached input data) + * if fflush(fp) != 0: # <<<<<<<<<<<<<< + * raise_from_errno() + * + */ + __pyx_t_1 = ((fflush(__pyx_v_fp) != 0) != 0); + if (__pyx_t_1) { + + /* "s3ql/deltadump.pyx":150 + * # below (which is necessary in case there is cached input data) + * if fflush(fp) != 0: + * raise_from_errno() # <<<<<<<<<<<<<< + * + * # Reposition FD to position of FILE*, otherwise next read from FD will miss + */ + __pyx_t_2 = __pyx_f_4s3ql_9deltadump_raise_from_errno(NULL); if (unlikely(__pyx_t_2 == -1)) __PYX_ERR(0, 150, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":149 + * # important, so that we can safely reposition the fd position + * # below (which is necessary in case there is cached input data) + * if fflush(fp) != 0: # <<<<<<<<<<<<<< + * raise_from_errno() + * + */ + } + + /* "s3ql/deltadump.pyx":156 + * # the same thing, but this does not seem to be documented so we don't rely + * # on it. + * off = ftell(fp) # <<<<<<<<<<<<<< + * if off == -1: + * raise_from_errno() + */ + __pyx_v_off = ftell(__pyx_v_fp); + + /* "s3ql/deltadump.pyx":157 + * # on it. + * off = ftell(fp) + * if off == -1: # <<<<<<<<<<<<<< + * raise_from_errno() + * + */ + __pyx_t_1 = ((__pyx_v_off == -1L) != 0); + if (__pyx_t_1) { + + /* "s3ql/deltadump.pyx":158 + * off = ftell(fp) + * if off == -1: + * raise_from_errno() # <<<<<<<<<<<<<< + * + * if lseek(fileno(fp), off, SEEK_SET) != off: + */ + __pyx_t_2 = __pyx_f_4s3ql_9deltadump_raise_from_errno(NULL); if (unlikely(__pyx_t_2 == -1)) __PYX_ERR(0, 158, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":157 + * # on it. + * off = ftell(fp) + * if off == -1: # <<<<<<<<<<<<<< + * raise_from_errno() + * + */ + } + + /* "s3ql/deltadump.pyx":160 + * raise_from_errno() + * + * if lseek(fileno(fp), off, SEEK_SET) != off: # <<<<<<<<<<<<<< + * raise_from_errno() + * + */ + __pyx_t_1 = ((lseek(fileno(__pyx_v_fp), __pyx_v_off, SEEK_SET) != __pyx_v_off) != 0); + if (__pyx_t_1) { + + /* "s3ql/deltadump.pyx":161 + * + * if lseek(fileno(fp), off, SEEK_SET) != off: + * raise_from_errno() # <<<<<<<<<<<<<< + * + * if fclose_c(fp) != 0: + */ + __pyx_t_2 = __pyx_f_4s3ql_9deltadump_raise_from_errno(NULL); if (unlikely(__pyx_t_2 == -1)) __PYX_ERR(0, 161, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":160 + * raise_from_errno() + * + * if lseek(fileno(fp), off, SEEK_SET) != off: # <<<<<<<<<<<<<< + * raise_from_errno() + * + */ + } + + /* "s3ql/deltadump.pyx":163 + * raise_from_errno() + * + * if fclose_c(fp) != 0: # <<<<<<<<<<<<<< + * raise_from_errno() + * + */ + __pyx_t_1 = ((fclose(__pyx_v_fp) != 0) != 0); + if (__pyx_t_1) { + + /* "s3ql/deltadump.pyx":164 + * + * if fclose_c(fp) != 0: + * raise_from_errno() # <<<<<<<<<<<<<< + * + * return 0 + */ + __pyx_t_2 = __pyx_f_4s3ql_9deltadump_raise_from_errno(NULL); if (unlikely(__pyx_t_2 == -1)) __PYX_ERR(0, 164, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":163 + * raise_from_errno() + * + * if fclose_c(fp) != 0: # <<<<<<<<<<<<<< + * raise_from_errno() + * + */ + } + + /* "s3ql/deltadump.pyx":166 + * raise_from_errno() + * + * return 0 # <<<<<<<<<<<<<< + * + * cdef void * calloc(size_t cnt, size_t size) except NULL: + */ + __pyx_r = 0; + goto __pyx_L0; + + /* "s3ql/deltadump.pyx":141 + * raise err_class(errno, PyUnicode_FromString(strerror(errno))) + * + * cdef int fclose(FILE * fp) except -1: # <<<<<<<<<<<<<< + * '''Call libc.fclose() and raise exception on failure''' + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("s3ql.deltadump.fclose", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":168 + * return 0 + * + * cdef void * calloc(size_t cnt, size_t size) except NULL: # <<<<<<<<<<<<<< + * '''Call libc.calloc and raise exception on failure''' + * + */ + +static void *__pyx_f_4s3ql_9deltadump_calloc(size_t __pyx_v_cnt, size_t __pyx_v_size) { + void *__pyx_v_ptr; + void *__pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2; + __Pyx_RefNannySetupContext("calloc", 0); + + /* "s3ql/deltadump.pyx":173 + * cdef void * ptr + * + * ptr = calloc_c(cnt, size) # <<<<<<<<<<<<<< + * + * if ptr is NULL: + */ + __pyx_v_ptr = calloc(__pyx_v_cnt, __pyx_v_size); + + /* "s3ql/deltadump.pyx":175 + * ptr = calloc_c(cnt, size) + * + * if ptr is NULL: # <<<<<<<<<<<<<< + * PyErr_NoMemory() + * + */ + __pyx_t_1 = ((__pyx_v_ptr == NULL) != 0); + if (__pyx_t_1) { + + /* "s3ql/deltadump.pyx":176 + * + * if ptr is NULL: + * PyErr_NoMemory() # <<<<<<<<<<<<<< + * + * return ptr + */ + __pyx_t_2 = PyErr_NoMemory(); if (unlikely(__pyx_t_2 == NULL)) __PYX_ERR(0, 176, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":175 + * ptr = calloc_c(cnt, size) + * + * if ptr is NULL: # <<<<<<<<<<<<<< + * PyErr_NoMemory() + * + */ + } + + /* "s3ql/deltadump.pyx":178 + * PyErr_NoMemory() + * + * return ptr # <<<<<<<<<<<<<< + * + * + */ + __pyx_r = __pyx_v_ptr; + goto __pyx_L0; + + /* "s3ql/deltadump.pyx":168 + * return 0 + * + * cdef void * calloc(size_t cnt, size_t size) except NULL: # <<<<<<<<<<<<<< + * '''Call libc.calloc and raise exception on failure''' + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("s3ql.deltadump.calloc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":181 + * + * + * cdef int SQLITE_CHECK_RC(int rc, int success, sqlite3* db) except -1: # <<<<<<<<<<<<<< + * '''Raise correct exception if *rc* != *success*''' + * + */ + +static int __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(int __pyx_v_rc, int __pyx_v_success, sqlite3 *__pyx_v_db) { + PyObject *__pyx_v_exc = NULL; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + __Pyx_RefNannySetupContext("SQLITE_CHECK_RC", 0); + + /* "s3ql/deltadump.pyx":184 + * '''Raise correct exception if *rc* != *success*''' + * + * if rc != success: # <<<<<<<<<<<<<< + * exc = apsw.exceptionfor(rc) + * raise type(exc)(PyUnicode_FromString(sqlite3_errmsg(db))) + */ + __pyx_t_1 = ((__pyx_v_rc != __pyx_v_success) != 0); + if (__pyx_t_1) { + + /* "s3ql/deltadump.pyx":185 + * + * if rc != success: + * exc = apsw.exceptionfor(rc) # <<<<<<<<<<<<<< + * raise type(exc)(PyUnicode_FromString(sqlite3_errmsg(db))) + * + */ + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_apsw); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 185, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_exceptionfor); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 185, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_rc); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 185, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_4); + if (likely(__pyx_t_5)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_5); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_4, function); + } + } + if (!__pyx_t_5) { + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 185, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_2); + } else { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 185, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_5); __pyx_t_5 = NULL; + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 185, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_exc = __pyx_t_2; + __pyx_t_2 = 0; + + /* "s3ql/deltadump.pyx":186 + * if rc != success: + * exc = apsw.exceptionfor(rc) + * raise type(exc)(PyUnicode_FromString(sqlite3_errmsg(db))) # <<<<<<<<<<<<<< + * + * return 0 + */ + __pyx_t_4 = PyUnicode_FromString(sqlite3_errmsg(__pyx_v_db)); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 186, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(((PyObject *)Py_TYPE(__pyx_v_exc))); + __pyx_t_6 = ((PyObject *)Py_TYPE(__pyx_v_exc)); __pyx_t_3 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_6); + if (likely(__pyx_t_3)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_6, function); + } + } + if (!__pyx_t_3) { + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 186, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_2); + } else { + __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 186, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_5, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 186, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(0, 186, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":184 + * '''Raise correct exception if *rc* != *success*''' + * + * if rc != success: # <<<<<<<<<<<<<< + * exc = apsw.exceptionfor(rc) + * raise type(exc)(PyUnicode_FromString(sqlite3_errmsg(db))) + */ + } + + /* "s3ql/deltadump.pyx":188 + * raise type(exc)(PyUnicode_FromString(sqlite3_errmsg(db))) + * + * return 0 # <<<<<<<<<<<<<< + * + * + */ + __pyx_r = 0; + goto __pyx_L0; + + /* "s3ql/deltadump.pyx":181 + * + * + * cdef int SQLITE_CHECK_RC(int rc, int success, sqlite3* db) except -1: # <<<<<<<<<<<<<< + * '''Raise correct exception if *rc* != *success*''' + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("s3ql.deltadump.SQLITE_CHECK_RC", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_exc); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":191 + * + * + * cdef int prep_columns(columns, int** col_types_p, int** col_args_p) except -1: # <<<<<<<<<<<<<< + * '''Allocate col_types and col_args, return number of columns + * + */ + +static int __pyx_f_4s3ql_9deltadump_prep_columns(PyObject *__pyx_v_columns, int **__pyx_v_col_types_p, int **__pyx_v_col_args_p) { + size_t __pyx_v_col_count; + int *__pyx_v_col_types; + int *__pyx_v_col_args; + PyObject *__pyx_v_i = NULL; + int __pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + void *__pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *(*__pyx_t_5)(PyObject *); + PyObject *__pyx_t_6 = NULL; + int __pyx_t_7; + PyObject *__pyx_t_8 = NULL; + int __pyx_t_9; + int __pyx_t_10; + Py_ssize_t __pyx_t_11; + __Pyx_RefNannySetupContext("prep_columns", 0); + + /* "s3ql/deltadump.pyx":201 + * cdef int *col_args + * + * col_count = len(columns) # guaranteed positive # <<<<<<<<<<<<<< + * col_types = < int *> calloc(col_count, sizeof(int)) + * col_args = < int *> calloc(col_count, sizeof(int)) + */ + __pyx_t_1 = PyObject_Length(__pyx_v_columns); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 201, __pyx_L1_error) + __pyx_v_col_count = ((size_t)__pyx_t_1); + + /* "s3ql/deltadump.pyx":202 + * + * col_count = len(columns) # guaranteed positive + * col_types = < int *> calloc(col_count, sizeof(int)) # <<<<<<<<<<<<<< + * col_args = < int *> calloc(col_count, sizeof(int)) + * + */ + __pyx_t_2 = __pyx_f_4s3ql_9deltadump_calloc(__pyx_v_col_count, (sizeof(int))); if (unlikely(__pyx_t_2 == NULL)) __PYX_ERR(0, 202, __pyx_L1_error) + __pyx_v_col_types = ((int *)__pyx_t_2); + + /* "s3ql/deltadump.pyx":203 + * col_count = len(columns) # guaranteed positive + * col_types = < int *> calloc(col_count, sizeof(int)) + * col_args = < int *> calloc(col_count, sizeof(int)) # <<<<<<<<<<<<<< + * + * # Initialize col_args and col_types + */ + __pyx_t_2 = __pyx_f_4s3ql_9deltadump_calloc(__pyx_v_col_count, (sizeof(int))); if (unlikely(__pyx_t_2 == NULL)) __PYX_ERR(0, 203, __pyx_L1_error) + __pyx_v_col_args = ((int *)__pyx_t_2); + + /* "s3ql/deltadump.pyx":206 + * + * # Initialize col_args and col_types + * for i in range(col_count): # <<<<<<<<<<<<<< + * if columns[i][1] not in (BLOB, INTEGER, TIME): + * raise ValueError("Invalid type for column %d" % i) + */ + __pyx_t_3 = __Pyx_PyInt_FromSize_t(__pyx_v_col_count); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 206, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 206, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_t_4, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 206, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (likely(PyList_CheckExact(__pyx_t_3)) || PyTuple_CheckExact(__pyx_t_3)) { + __pyx_t_4 = __pyx_t_3; __Pyx_INCREF(__pyx_t_4); __pyx_t_1 = 0; + __pyx_t_5 = NULL; + } else { + __pyx_t_1 = -1; __pyx_t_4 = PyObject_GetIter(__pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 206, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = Py_TYPE(__pyx_t_4)->tp_iternext; if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 206, __pyx_L1_error) + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + for (;;) { + if (likely(!__pyx_t_5)) { + if (likely(PyList_CheckExact(__pyx_t_4))) { + if (__pyx_t_1 >= PyList_GET_SIZE(__pyx_t_4)) break; + #if CYTHON_COMPILING_IN_CPYTHON + __pyx_t_3 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_1); __Pyx_INCREF(__pyx_t_3); __pyx_t_1++; if (unlikely(0 < 0)) __PYX_ERR(0, 206, __pyx_L1_error) + #else + __pyx_t_3 = PySequence_ITEM(__pyx_t_4, __pyx_t_1); __pyx_t_1++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 206, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + } else { + if (__pyx_t_1 >= PyTuple_GET_SIZE(__pyx_t_4)) break; + #if CYTHON_COMPILING_IN_CPYTHON + __pyx_t_3 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_1); __Pyx_INCREF(__pyx_t_3); __pyx_t_1++; if (unlikely(0 < 0)) __PYX_ERR(0, 206, __pyx_L1_error) + #else + __pyx_t_3 = PySequence_ITEM(__pyx_t_4, __pyx_t_1); __pyx_t_1++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 206, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + } + } else { + __pyx_t_3 = __pyx_t_5(__pyx_t_4); + if (unlikely(!__pyx_t_3)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); + else __PYX_ERR(0, 206, __pyx_L1_error) + } + break; + } + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_XDECREF_SET(__pyx_v_i, __pyx_t_3); + __pyx_t_3 = 0; + + /* "s3ql/deltadump.pyx":207 + * # Initialize col_args and col_types + * for i in range(col_count): + * if columns[i][1] not in (BLOB, INTEGER, TIME): # <<<<<<<<<<<<<< + * raise ValueError("Invalid type for column %d" % i) + * col_types[i] = columns[i][1] + */ + __pyx_t_3 = PyObject_GetItem(__pyx_v_columns, __pyx_v_i); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 207, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = __Pyx_GetItemInt(__pyx_t_3, 1, long, 1, __Pyx_PyInt_From_long, 0, 0, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 207, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_BLOB); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 207, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_8 = PyObject_RichCompare(__pyx_t_6, __pyx_t_3, Py_NE); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 207, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely(__pyx_t_9 < 0)) __PYX_ERR(0, 207, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (__pyx_t_9) { + } else { + __pyx_t_7 = __pyx_t_9; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_INTEGER); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 207, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_3 = PyObject_RichCompare(__pyx_t_6, __pyx_t_8, Py_NE); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 207, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_9 < 0)) __PYX_ERR(0, 207, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (__pyx_t_9) { + } else { + __pyx_t_7 = __pyx_t_9; + goto __pyx_L6_bool_binop_done; + } + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_TIME); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 207, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_8 = PyObject_RichCompare(__pyx_t_6, __pyx_t_3, Py_NE); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 207, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely(__pyx_t_9 < 0)) __PYX_ERR(0, 207, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_7 = __pyx_t_9; + __pyx_L6_bool_binop_done:; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_9 = (__pyx_t_7 != 0); + if (__pyx_t_9) { + + /* "s3ql/deltadump.pyx":208 + * for i in range(col_count): + * if columns[i][1] not in (BLOB, INTEGER, TIME): + * raise ValueError("Invalid type for column %d" % i) # <<<<<<<<<<<<<< + * col_types[i] = columns[i][1] + * + */ + __pyx_t_6 = PyUnicode_Format(__pyx_kp_u_Invalid_type_for_column_d, __pyx_v_i); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 208, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_8 = PyTuple_New(1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 208, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_t_8, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 208, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_Raise(__pyx_t_6, 0, 0, 0); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __PYX_ERR(0, 208, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":207 + * # Initialize col_args and col_types + * for i in range(col_count): + * if columns[i][1] not in (BLOB, INTEGER, TIME): # <<<<<<<<<<<<<< + * raise ValueError("Invalid type for column %d" % i) + * col_types[i] = columns[i][1] + */ + } + + /* "s3ql/deltadump.pyx":209 + * if columns[i][1] not in (BLOB, INTEGER, TIME): + * raise ValueError("Invalid type for column %d" % i) + * col_types[i] = columns[i][1] # <<<<<<<<<<<<<< + * + * if len(columns[i]) == 3: + */ + __pyx_t_6 = PyObject_GetItem(__pyx_v_columns, __pyx_v_i); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 209, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_8 = __Pyx_GetItemInt(__pyx_t_6, 1, long, 1, __Pyx_PyInt_From_long, 0, 0, 1); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 209, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_10 = __Pyx_PyInt_As_int(__pyx_t_8); if (unlikely((__pyx_t_10 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 209, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_v_i); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 209, __pyx_L1_error) + (__pyx_v_col_types[__pyx_t_11]) = __pyx_t_10; + + /* "s3ql/deltadump.pyx":211 + * col_types[i] = columns[i][1] + * + * if len(columns[i]) == 3: # <<<<<<<<<<<<<< + * col_args[i] = columns[i][2] + * else: + */ + __pyx_t_8 = PyObject_GetItem(__pyx_v_columns, __pyx_v_i); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 211, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_11 = PyObject_Length(__pyx_t_8); if (unlikely(__pyx_t_11 == -1)) __PYX_ERR(0, 211, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_9 = ((__pyx_t_11 == 3) != 0); + if (__pyx_t_9) { + + /* "s3ql/deltadump.pyx":212 + * + * if len(columns[i]) == 3: + * col_args[i] = columns[i][2] # <<<<<<<<<<<<<< + * else: + * col_args[i] = 0 + */ + __pyx_t_8 = PyObject_GetItem(__pyx_v_columns, __pyx_v_i); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 212, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_6 = __Pyx_GetItemInt(__pyx_t_8, 2, long, 1, __Pyx_PyInt_From_long, 0, 0, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 212, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_10 = __Pyx_PyInt_As_int(__pyx_t_6); if (unlikely((__pyx_t_10 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 212, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_v_i); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 212, __pyx_L1_error) + (__pyx_v_col_args[__pyx_t_11]) = __pyx_t_10; + + /* "s3ql/deltadump.pyx":211 + * col_types[i] = columns[i][1] + * + * if len(columns[i]) == 3: # <<<<<<<<<<<<<< + * col_args[i] = columns[i][2] + * else: + */ + goto __pyx_L9; + } + + /* "s3ql/deltadump.pyx":214 + * col_args[i] = columns[i][2] + * else: + * col_args[i] = 0 # <<<<<<<<<<<<<< + * + * col_types_p[0] = col_types + */ + /*else*/ { + __pyx_t_11 = __Pyx_PyIndex_AsSsize_t(__pyx_v_i); if (unlikely((__pyx_t_11 == (Py_ssize_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 214, __pyx_L1_error) + (__pyx_v_col_args[__pyx_t_11]) = 0; + } + __pyx_L9:; + + /* "s3ql/deltadump.pyx":206 + * + * # Initialize col_args and col_types + * for i in range(col_count): # <<<<<<<<<<<<<< + * if columns[i][1] not in (BLOB, INTEGER, TIME): + * raise ValueError("Invalid type for column %d" % i) + */ + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":216 + * col_args[i] = 0 + * + * col_types_p[0] = col_types # <<<<<<<<<<<<<< + * col_args_p[0] = col_args + * + */ + (__pyx_v_col_types_p[0]) = __pyx_v_col_types; + + /* "s3ql/deltadump.pyx":217 + * + * col_types_p[0] = col_types + * col_args_p[0] = col_args # <<<<<<<<<<<<<< + * + * # We can safely assume that this fits into an int + */ + (__pyx_v_col_args_p[0]) = __pyx_v_col_args; + + /* "s3ql/deltadump.pyx":220 + * + * # We can safely assume that this fits into an int + * return col_count # <<<<<<<<<<<<<< + * + * cdef FILE* dup_to_fp(fh, const_char* mode) except NULL: + */ + __pyx_r = ((int)__pyx_v_col_count); + goto __pyx_L0; + + /* "s3ql/deltadump.pyx":191 + * + * + * cdef int prep_columns(columns, int** col_types_p, int** col_args_p) except -1: # <<<<<<<<<<<<<< + * '''Allocate col_types and col_args, return number of columns + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("s3ql.deltadump.prep_columns", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_i); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":222 + * return col_count + * + * cdef FILE* dup_to_fp(fh, const_char* mode) except NULL: # <<<<<<<<<<<<<< + * '''Duplicate fd from *fh* and open as FILE*''' + * + */ + +static FILE *__pyx_f_4s3ql_9deltadump_dup_to_fp(PyObject *__pyx_v_fh, const char *__pyx_v_mode) { + int __pyx_v_fd; + FILE *__pyx_v_fp; + FILE *__pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + int __pyx_t_5; + __Pyx_RefNannySetupContext("dup_to_fp", 0); + + /* "s3ql/deltadump.pyx":227 + * cdef int fd + * + * fd = dup(fh.fileno()) # <<<<<<<<<<<<<< + * if fd == -1: + * raise_from_errno() + */ + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_fh, __pyx_n_s_fileno); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 227, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); + if (likely(__pyx_t_3)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_2, function); + } + } + if (__pyx_t_3) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 227, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else { + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 227, __pyx_L1_error) + } + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_4 = __Pyx_PyInt_As_int(__pyx_t_1); if (unlikely((__pyx_t_4 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 227, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_fd = dup(__pyx_t_4); + + /* "s3ql/deltadump.pyx":228 + * + * fd = dup(fh.fileno()) + * if fd == -1: # <<<<<<<<<<<<<< + * raise_from_errno() + * + */ + __pyx_t_5 = ((__pyx_v_fd == -1L) != 0); + if (__pyx_t_5) { + + /* "s3ql/deltadump.pyx":229 + * fd = dup(fh.fileno()) + * if fd == -1: + * raise_from_errno() # <<<<<<<<<<<<<< + * + * fp = fdopen(fd, mode) + */ + __pyx_t_4 = __pyx_f_4s3ql_9deltadump_raise_from_errno(NULL); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(0, 229, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":228 + * + * fd = dup(fh.fileno()) + * if fd == -1: # <<<<<<<<<<<<<< + * raise_from_errno() + * + */ + } + + /* "s3ql/deltadump.pyx":231 + * raise_from_errno() + * + * fp = fdopen(fd, mode) # <<<<<<<<<<<<<< + * if fp == NULL: + * raise_from_errno() + */ + __pyx_v_fp = fdopen(__pyx_v_fd, __pyx_v_mode); + + /* "s3ql/deltadump.pyx":232 + * + * fp = fdopen(fd, mode) + * if fp == NULL: # <<<<<<<<<<<<<< + * raise_from_errno() + * + */ + __pyx_t_5 = ((__pyx_v_fp == NULL) != 0); + if (__pyx_t_5) { + + /* "s3ql/deltadump.pyx":233 + * fp = fdopen(fd, mode) + * if fp == NULL: + * raise_from_errno() # <<<<<<<<<<<<<< + * + * return fp + */ + __pyx_t_4 = __pyx_f_4s3ql_9deltadump_raise_from_errno(NULL); if (unlikely(__pyx_t_4 == -1)) __PYX_ERR(0, 233, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":232 + * + * fp = fdopen(fd, mode) + * if fp == NULL: # <<<<<<<<<<<<<< + * raise_from_errno() + * + */ + } + + /* "s3ql/deltadump.pyx":235 + * raise_from_errno() + * + * return fp # <<<<<<<<<<<<<< + * + * def check_sqlite(): + */ + __pyx_r = __pyx_v_fp; + goto __pyx_L0; + + /* "s3ql/deltadump.pyx":222 + * return col_count + * + * cdef FILE* dup_to_fp(fh, const_char* mode) except NULL: # <<<<<<<<<<<<<< + * '''Duplicate fd from *fh* and open as FILE*''' + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("s3ql.deltadump.dup_to_fp", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":237 + * return fp + * + * def check_sqlite(): # <<<<<<<<<<<<<< + * '''Check if deltadump and apsw module use compatible SQLite code. + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_1check_sqlite(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static char __pyx_doc_4s3ql_9deltadump_check_sqlite[] = "check_sqlite()\nCheck if deltadump and apsw module use compatible SQLite code.\n\n This functions look at versions and compile options of the SQLite\n code used by the *apsw* module and the *deltadump* module. If they\n do not match exactly, a `RuntimeError` is raised.\n\n Only if both modules use the same SQLite version compiled with the\n same options can the database object be shared between *apsw* and\n *deltadump*.\n "; +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_1check_sqlite = {"check_sqlite", (PyCFunction)__pyx_pw_4s3ql_9deltadump_1check_sqlite, METH_NOARGS, __pyx_doc_4s3ql_9deltadump_check_sqlite}; +static PyObject *__pyx_pw_4s3ql_9deltadump_1check_sqlite(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("check_sqlite (wrapper)", 0); + __pyx_r = __pyx_pf_4s3ql_9deltadump_check_sqlite(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_4s3ql_9deltadump_check_sqlite(CYTHON_UNUSED PyObject *__pyx_self) { + const char *__pyx_v_buf; + PyObject *__pyx_v_apsw_sqlite_version = NULL; + PyObject *__pyx_v_s3ql_sqlite_version = NULL; + PyObject *__pyx_v_apsw_sqlite_options = NULL; + PyObject *__pyx_v_s3ql_sqlite_options = NULL; + PyObject *__pyx_v_idx = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + Py_ssize_t __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + int __pyx_t_6; + PyObject *(*__pyx_t_7)(PyObject *); + int __pyx_t_8; + int __pyx_t_9; + __Pyx_RefNannySetupContext("check_sqlite", 0); + + /* "s3ql/deltadump.pyx":251 + * cdef const_char *buf + * + * apsw_sqlite_version = apsw.sqlitelibversion() # <<<<<<<<<<<<<< + * s3ql_sqlite_version = PyUnicode_FromString(sqlite3_libversion()) + * log.debug('apsw sqlite version: %s, ' + */ + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_apsw); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 251, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_sqlitelibversion); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 251, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_2)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + } + } + if (__pyx_t_2) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 251, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 251, __pyx_L1_error) + } + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_apsw_sqlite_version = __pyx_t_1; + __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":252 + * + * apsw_sqlite_version = apsw.sqlitelibversion() + * s3ql_sqlite_version = PyUnicode_FromString(sqlite3_libversion()) # <<<<<<<<<<<<<< + * log.debug('apsw sqlite version: %s, ' + * 's3ql sqlite version: %s', + */ + __pyx_t_1 = PyUnicode_FromString(sqlite3_libversion()); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 252, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_s3ql_sqlite_version = __pyx_t_1; + __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":253 + * apsw_sqlite_version = apsw.sqlitelibversion() + * s3ql_sqlite_version = PyUnicode_FromString(sqlite3_libversion()) + * log.debug('apsw sqlite version: %s, ' # <<<<<<<<<<<<<< + * 's3ql sqlite version: %s', + * apsw_sqlite_version, + */ + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_log); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 253, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_debug); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 253, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "s3ql/deltadump.pyx":256 + * 's3ql sqlite version: %s', + * apsw_sqlite_version, + * s3ql_sqlite_version) # <<<<<<<<<<<<<< + * if apsw_sqlite_version != s3ql_sqlite_version: + * raise RuntimeError('SQLite version mismatch between APSW and S3QL ' + */ + __pyx_t_3 = NULL; + __pyx_t_4 = 0; + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); + if (likely(__pyx_t_3)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_2, function); + __pyx_t_4 = 1; + } + } + __pyx_t_5 = PyTuple_New(3+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 253, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (__pyx_t_3) { + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; + } + __Pyx_INCREF(__pyx_kp_u_apsw_sqlite_version_s_s3ql_sqlit); + __Pyx_GIVEREF(__pyx_kp_u_apsw_sqlite_version_s_s3ql_sqlit); + PyTuple_SET_ITEM(__pyx_t_5, 0+__pyx_t_4, __pyx_kp_u_apsw_sqlite_version_s_s3ql_sqlit); + __Pyx_INCREF(__pyx_v_apsw_sqlite_version); + __Pyx_GIVEREF(__pyx_v_apsw_sqlite_version); + PyTuple_SET_ITEM(__pyx_t_5, 1+__pyx_t_4, __pyx_v_apsw_sqlite_version); + __Pyx_INCREF(__pyx_v_s3ql_sqlite_version); + __Pyx_GIVEREF(__pyx_v_s3ql_sqlite_version); + PyTuple_SET_ITEM(__pyx_t_5, 2+__pyx_t_4, __pyx_v_s3ql_sqlite_version); + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 253, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":257 + * apsw_sqlite_version, + * s3ql_sqlite_version) + * if apsw_sqlite_version != s3ql_sqlite_version: # <<<<<<<<<<<<<< + * raise RuntimeError('SQLite version mismatch between APSW and S3QL ' + * '(%s vs %s)' % (apsw_sqlite_version, s3ql_sqlite_version)) + */ + __pyx_t_1 = PyObject_RichCompare(__pyx_v_apsw_sqlite_version, __pyx_v_s3ql_sqlite_version, Py_NE); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 257, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(0, 257, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_6) { + + /* "s3ql/deltadump.pyx":259 + * if apsw_sqlite_version != s3ql_sqlite_version: + * raise RuntimeError('SQLite version mismatch between APSW and S3QL ' + * '(%s vs %s)' % (apsw_sqlite_version, s3ql_sqlite_version)) # <<<<<<<<<<<<<< + * + * apsw_sqlite_options = set(apsw.compile_options) + */ + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 259, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_apsw_sqlite_version); + __Pyx_GIVEREF(__pyx_v_apsw_sqlite_version); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_apsw_sqlite_version); + __Pyx_INCREF(__pyx_v_s3ql_sqlite_version); + __Pyx_GIVEREF(__pyx_v_s3ql_sqlite_version); + PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_s3ql_sqlite_version); + __pyx_t_2 = PyUnicode_Format(__pyx_kp_u_SQLite_version_mismatch_between, __pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 259, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":258 + * s3ql_sqlite_version) + * if apsw_sqlite_version != s3ql_sqlite_version: + * raise RuntimeError('SQLite version mismatch between APSW and S3QL ' # <<<<<<<<<<<<<< + * '(%s vs %s)' % (apsw_sqlite_version, s3ql_sqlite_version)) + * + */ + __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 258, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_2); + __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 258, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(0, 258, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":257 + * apsw_sqlite_version, + * s3ql_sqlite_version) + * if apsw_sqlite_version != s3ql_sqlite_version: # <<<<<<<<<<<<<< + * raise RuntimeError('SQLite version mismatch between APSW and S3QL ' + * '(%s vs %s)' % (apsw_sqlite_version, s3ql_sqlite_version)) + */ + } + + /* "s3ql/deltadump.pyx":261 + * '(%s vs %s)' % (apsw_sqlite_version, s3ql_sqlite_version)) + * + * apsw_sqlite_options = set(apsw.compile_options) # <<<<<<<<<<<<<< + * s3ql_sqlite_options = set() + * for idx in itertools.count(0): + */ + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_apsw); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 261, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_compile_options); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 261, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = PySet_New(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 261, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_apsw_sqlite_options = ((PyObject*)__pyx_t_2); + __pyx_t_2 = 0; + + /* "s3ql/deltadump.pyx":262 + * + * apsw_sqlite_options = set(apsw.compile_options) + * s3ql_sqlite_options = set() # <<<<<<<<<<<<<< + * for idx in itertools.count(0): + * buf = sqlite3_compileoption_get(idx) + */ + __pyx_t_2 = PySet_New(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 262, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_s3ql_sqlite_options = ((PyObject*)__pyx_t_2); + __pyx_t_2 = 0; + + /* "s3ql/deltadump.pyx":263 + * apsw_sqlite_options = set(apsw.compile_options) + * s3ql_sqlite_options = set() + * for idx in itertools.count(0): # <<<<<<<<<<<<<< + * buf = sqlite3_compileoption_get(idx) + * if buf is NULL: + */ + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_itertools); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 263, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_count); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 263, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 263, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (likely(PyList_CheckExact(__pyx_t_2)) || PyTuple_CheckExact(__pyx_t_2)) { + __pyx_t_1 = __pyx_t_2; __Pyx_INCREF(__pyx_t_1); __pyx_t_4 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_4 = -1; __pyx_t_1 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 263, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_7 = Py_TYPE(__pyx_t_1)->tp_iternext; if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 263, __pyx_L1_error) + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_1))) { + if (__pyx_t_4 >= PyList_GET_SIZE(__pyx_t_1)) break; + #if CYTHON_COMPILING_IN_CPYTHON + __pyx_t_2 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_4); __Pyx_INCREF(__pyx_t_2); __pyx_t_4++; if (unlikely(0 < 0)) __PYX_ERR(0, 263, __pyx_L1_error) + #else + __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_4); __pyx_t_4++; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 263, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #endif + } else { + if (__pyx_t_4 >= PyTuple_GET_SIZE(__pyx_t_1)) break; + #if CYTHON_COMPILING_IN_CPYTHON + __pyx_t_2 = PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_4); __Pyx_INCREF(__pyx_t_2); __pyx_t_4++; if (unlikely(0 < 0)) __PYX_ERR(0, 263, __pyx_L1_error) + #else + __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_4); __pyx_t_4++; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 263, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + #endif + } + } else { + __pyx_t_2 = __pyx_t_7(__pyx_t_1); + if (unlikely(!__pyx_t_2)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); + else __PYX_ERR(0, 263, __pyx_L1_error) + } + break; + } + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_XDECREF_SET(__pyx_v_idx, __pyx_t_2); + __pyx_t_2 = 0; + + /* "s3ql/deltadump.pyx":264 + * s3ql_sqlite_options = set() + * for idx in itertools.count(0): + * buf = sqlite3_compileoption_get(idx) # <<<<<<<<<<<<<< + * if buf is NULL: + * break + */ + __pyx_t_8 = __Pyx_PyInt_As_int(__pyx_v_idx); if (unlikely((__pyx_t_8 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 264, __pyx_L1_error) + __pyx_v_buf = sqlite3_compileoption_get(__pyx_t_8); + + /* "s3ql/deltadump.pyx":265 + * for idx in itertools.count(0): + * buf = sqlite3_compileoption_get(idx) + * if buf is NULL: # <<<<<<<<<<<<<< + * break + * s3ql_sqlite_options.add(PyUnicode_FromString(buf)) + */ + __pyx_t_6 = ((__pyx_v_buf == NULL) != 0); + if (__pyx_t_6) { + + /* "s3ql/deltadump.pyx":266 + * buf = sqlite3_compileoption_get(idx) + * if buf is NULL: + * break # <<<<<<<<<<<<<< + * s3ql_sqlite_options.add(PyUnicode_FromString(buf)) + * + */ + goto __pyx_L5_break; + + /* "s3ql/deltadump.pyx":265 + * for idx in itertools.count(0): + * buf = sqlite3_compileoption_get(idx) + * if buf is NULL: # <<<<<<<<<<<<<< + * break + * s3ql_sqlite_options.add(PyUnicode_FromString(buf)) + */ + } + + /* "s3ql/deltadump.pyx":267 + * if buf is NULL: + * break + * s3ql_sqlite_options.add(PyUnicode_FromString(buf)) # <<<<<<<<<<<<<< + * + * log.debug('apsw sqlite compile options: %s, ' + */ + __pyx_t_2 = PyUnicode_FromString(__pyx_v_buf); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 267, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_9 = PySet_Add(__pyx_v_s3ql_sqlite_options, __pyx_t_2); if (unlikely(__pyx_t_9 == -1)) __PYX_ERR(0, 267, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "s3ql/deltadump.pyx":263 + * apsw_sqlite_options = set(apsw.compile_options) + * s3ql_sqlite_options = set() + * for idx in itertools.count(0): # <<<<<<<<<<<<<< + * buf = sqlite3_compileoption_get(idx) + * if buf is NULL: + */ + } + __pyx_L5_break:; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":269 + * s3ql_sqlite_options.add(PyUnicode_FromString(buf)) + * + * log.debug('apsw sqlite compile options: %s, ' # <<<<<<<<<<<<<< + * 's3ql sqlite compile options: %s', + * apsw_sqlite_options, + */ + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_log); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 269, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_debug); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 269, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "s3ql/deltadump.pyx":272 + * 's3ql sqlite compile options: %s', + * apsw_sqlite_options, + * s3ql_sqlite_options) # <<<<<<<<<<<<<< + * if apsw_sqlite_options != s3ql_sqlite_options: + * raise RuntimeError('SQLite code used by APSW was compiled with different ' + */ + __pyx_t_2 = NULL; + __pyx_t_4 = 0; + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_5); + if (likely(__pyx_t_2)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_5, function); + __pyx_t_4 = 1; + } + } + __pyx_t_3 = PyTuple_New(3+__pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 269, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (__pyx_t_2) { + __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_2); __pyx_t_2 = NULL; + } + __Pyx_INCREF(__pyx_kp_u_apsw_sqlite_compile_options_s_s3); + __Pyx_GIVEREF(__pyx_kp_u_apsw_sqlite_compile_options_s_s3); + PyTuple_SET_ITEM(__pyx_t_3, 0+__pyx_t_4, __pyx_kp_u_apsw_sqlite_compile_options_s_s3); + __Pyx_INCREF(__pyx_v_apsw_sqlite_options); + __Pyx_GIVEREF(__pyx_v_apsw_sqlite_options); + PyTuple_SET_ITEM(__pyx_t_3, 1+__pyx_t_4, __pyx_v_apsw_sqlite_options); + __Pyx_INCREF(__pyx_v_s3ql_sqlite_options); + __Pyx_GIVEREF(__pyx_v_s3ql_sqlite_options); + PyTuple_SET_ITEM(__pyx_t_3, 2+__pyx_t_4, __pyx_v_s3ql_sqlite_options); + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 269, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":273 + * apsw_sqlite_options, + * s3ql_sqlite_options) + * if apsw_sqlite_options != s3ql_sqlite_options: # <<<<<<<<<<<<<< + * raise RuntimeError('SQLite code used by APSW was compiled with different ' + * 'options than SQLite code available to S3QL! ' + */ + __pyx_t_1 = PyObject_RichCompare(__pyx_v_apsw_sqlite_options, __pyx_v_s3ql_sqlite_options, Py_NE); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 273, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(0, 273, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_6) { + + /* "s3ql/deltadump.pyx":277 + * 'options than SQLite code available to S3QL! ' + * 'Differing settings: + %s, - %s' % + * (apsw_sqlite_options - s3ql_sqlite_options, # <<<<<<<<<<<<<< + * s3ql_sqlite_options - apsw_sqlite_options)) + * + */ + __pyx_t_1 = PyNumber_Subtract(__pyx_v_apsw_sqlite_options, __pyx_v_s3ql_sqlite_options); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 277, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + + /* "s3ql/deltadump.pyx":278 + * 'Differing settings: + %s, - %s' % + * (apsw_sqlite_options - s3ql_sqlite_options, + * s3ql_sqlite_options - apsw_sqlite_options)) # <<<<<<<<<<<<<< + * + * def dump_table(table, order, columns, db, fh): + */ + __pyx_t_5 = PyNumber_Subtract(__pyx_v_s3ql_sqlite_options, __pyx_v_apsw_sqlite_options); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 278, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + + /* "s3ql/deltadump.pyx":277 + * 'options than SQLite code available to S3QL! ' + * 'Differing settings: + %s, - %s' % + * (apsw_sqlite_options - s3ql_sqlite_options, # <<<<<<<<<<<<<< + * s3ql_sqlite_options - apsw_sqlite_options)) + * + */ + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 277, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_5); + __pyx_t_1 = 0; + __pyx_t_5 = 0; + + /* "s3ql/deltadump.pyx":276 + * raise RuntimeError('SQLite code used by APSW was compiled with different ' + * 'options than SQLite code available to S3QL! ' + * 'Differing settings: + %s, - %s' % # <<<<<<<<<<<<<< + * (apsw_sqlite_options - s3ql_sqlite_options, + * s3ql_sqlite_options - apsw_sqlite_options)) + */ + __pyx_t_5 = PyUnicode_Format(__pyx_kp_u_SQLite_code_used_by_APSW_was_com, __pyx_t_3); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 276, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "s3ql/deltadump.pyx":274 + * s3ql_sqlite_options) + * if apsw_sqlite_options != s3ql_sqlite_options: + * raise RuntimeError('SQLite code used by APSW was compiled with different ' # <<<<<<<<<<<<<< + * 'options than SQLite code available to S3QL! ' + * 'Differing settings: + %s, - %s' % + */ + __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 274, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_5); + __pyx_t_5 = 0; + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_t_3, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 274, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_Raise(__pyx_t_5, 0, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __PYX_ERR(0, 274, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":273 + * apsw_sqlite_options, + * s3ql_sqlite_options) + * if apsw_sqlite_options != s3ql_sqlite_options: # <<<<<<<<<<<<<< + * raise RuntimeError('SQLite code used by APSW was compiled with different ' + * 'options than SQLite code available to S3QL! ' + */ + } + + /* "s3ql/deltadump.pyx":237 + * return fp + * + * def check_sqlite(): # <<<<<<<<<<<<<< + * '''Check if deltadump and apsw module use compatible SQLite code. + * + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("s3ql.deltadump.check_sqlite", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_apsw_sqlite_version); + __Pyx_XDECREF(__pyx_v_s3ql_sqlite_version); + __Pyx_XDECREF(__pyx_v_apsw_sqlite_options); + __Pyx_XDECREF(__pyx_v_s3ql_sqlite_options); + __Pyx_XDECREF(__pyx_v_idx); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":280 + * s3ql_sqlite_options - apsw_sqlite_options)) + * + * def dump_table(table, order, columns, db, fh): # <<<<<<<<<<<<<< + * '''Dump *columns* of *table* into *fh* + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_3dump_table(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_4s3ql_9deltadump_2dump_table[] = "dump_table(table, order, columns, db, fh)\nDump *columns* of *table* into *fh*\n\n *order* specifies the order in which the rows are written and must be a\n string that can be inserted after the \"ORDER BY\" clause in an SQL SELECT\n statement.\n\n *db* is an `s3ql.Connection` instance for the database.\n\n *columns* must a list of 3-tuples, one for each column that should be\n stored. The first element of the tuple must contain the column name and the\n second element the type of data stored in the column (`INTEGER`, `TIME`\n or `BLOB`). Times will be converted to nanosecond integers.\n\n For integers and seconds, the third tuple element specifies the expected\n change of the values between rows. For blobs it can be either zero\n (indicating variable length columns) or an integer specifying the length of\n the column values in bytes.\n\n This function will open a separate connection to the database, so\n the *db* connection should not be in EXCLUSIVE locking mode.\n (Using a separate connection avoids the requirement on the *apsw*\n and *deltadump* modules be linked against against binary\n compatible SQLite libraries).\n "; +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_3dump_table = {"dump_table", (PyCFunction)__pyx_pw_4s3ql_9deltadump_3dump_table, METH_VARARGS|METH_KEYWORDS, __pyx_doc_4s3ql_9deltadump_2dump_table}; +static PyObject *__pyx_pw_4s3ql_9deltadump_3dump_table(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_table = 0; + PyObject *__pyx_v_order = 0; + PyObject *__pyx_v_columns = 0; + PyObject *__pyx_v_db = 0; + PyObject *__pyx_v_fh = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("dump_table (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_table,&__pyx_n_s_order,&__pyx_n_s_columns,&__pyx_n_s_db,&__pyx_n_s_fh,0}; + PyObject* values[5] = {0,0,0,0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_table)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_order)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("dump_table", 1, 5, 5, 1); __PYX_ERR(0, 280, __pyx_L3_error) + } + case 2: + if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_columns)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("dump_table", 1, 5, 5, 2); __PYX_ERR(0, 280, __pyx_L3_error) + } + case 3: + if (likely((values[3] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_db)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("dump_table", 1, 5, 5, 3); __PYX_ERR(0, 280, __pyx_L3_error) + } + case 4: + if (likely((values[4] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_fh)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("dump_table", 1, 5, 5, 4); __PYX_ERR(0, 280, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "dump_table") < 0)) __PYX_ERR(0, 280, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 5) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + values[4] = PyTuple_GET_ITEM(__pyx_args, 4); + } + __pyx_v_table = values[0]; + __pyx_v_order = values[1]; + __pyx_v_columns = values[2]; + __pyx_v_db = values[3]; + __pyx_v_fh = values[4]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("dump_table", 1, 5, 5, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 280, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("s3ql.deltadump.dump_table", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_4s3ql_9deltadump_2dump_table(__pyx_self, __pyx_v_table, __pyx_v_order, __pyx_v_columns, __pyx_v_db, __pyx_v_fh); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":328 + * SQLITE_OPEN_READONLY, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * SQLITE_CHECK_RC(sqlite3_extended_result_codes(sqlite3_db, 1), + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10dump_table_lambda(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10dump_table_lambda = {"lambda", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10dump_table_lambda, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10dump_table_lambda(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannySetupContext("lambda", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + + /* "s3ql/deltadump.pyx":329 + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), + * SQLITE_OK, sqlite3_db)) # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_extended_result_codes(sqlite3_db, 1), + * SQLITE_OK, sqlite3_db) + */ + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_close(__pyx_cur_scope->__pyx_v_sqlite3_db), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 328, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":328 + * SQLITE_OPEN_READONLY, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * SQLITE_CHECK_RC(sqlite3_extended_result_codes(sqlite3_db, 1), + */ + __pyx_t_2 = __Pyx_PyInt_From_int(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 328, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("s3ql.deltadump.dump_table.lambda", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":335 + * # Get FILE* for buffered reading from *fh* + * fp = dup_to_fp(fh, b'wb') + * cm.callback(lambda: fclose(fp)) # <<<<<<<<<<<<<< + * + * # Allocate col_args and col_types + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10dump_table_1lambda1(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10dump_table_1lambda1 = {"lambda1", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10dump_table_1lambda1, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10dump_table_1lambda1(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda1 (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda1(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda1(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannySetupContext("lambda1", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_fclose(__pyx_cur_scope->__pyx_v_fp); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 335, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyInt_From_int(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("s3ql.deltadump.dump_table.lambda1", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":339 + * # Allocate col_args and col_types + * col_count = prep_columns(columns, &col_types, &col_args) + * cm.callback(lambda: free(col_args)) # <<<<<<<<<<<<<< + * cm.callback(lambda: free(col_types)) + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10dump_table_2lambda2(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10dump_table_2lambda2 = {"lambda2", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10dump_table_2lambda2, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10dump_table_2lambda2(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda2 (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda2(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda2(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("lambda2", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_free(__pyx_cur_scope->__pyx_v_col_args); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 339, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("s3ql.deltadump.dump_table.lambda2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":340 + * col_count = prep_columns(columns, &col_types, &col_args) + * cm.callback(lambda: free(col_args)) + * cm.callback(lambda: free(col_types)) # <<<<<<<<<<<<<< + * + * # Allocate int64_prev + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10dump_table_3lambda3(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10dump_table_3lambda3 = {"lambda3", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10dump_table_3lambda3, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10dump_table_3lambda3(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda3 (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda3(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda3(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("lambda3", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_free(__pyx_cur_scope->__pyx_v_col_types); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 340, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("s3ql.deltadump.dump_table.lambda3", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":344 + * # Allocate int64_prev + * int64_prev = calloc( len(columns), sizeof(int64_t)) + * cm.callback(lambda: free(int64_prev)) # <<<<<<<<<<<<<< + * + * # Prepare statement + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10dump_table_4lambda4(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10dump_table_4lambda4 = {"lambda4", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10dump_table_4lambda4, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10dump_table_4lambda4(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda4 (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda4(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda4(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("lambda4", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_free(__pyx_cur_scope->__pyx_v_int64_prev); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 344, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("s3ql.deltadump.dump_table.lambda4", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":352 + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(stmt), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10dump_table_5lambda5(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10dump_table_5lambda5 = {"lambda5", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10dump_table_5lambda5, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10dump_table_5lambda5(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda5 (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda5(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda5(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannySetupContext("lambda5", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + + /* "s3ql/deltadump.pyx":353 + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(stmt), + * SQLITE_OK, sqlite3_db)) # <<<<<<<<<<<<<< + * + * row_count = db.get_val("SELECT COUNT(rowid) FROM %s" % table) + */ + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_finalize(__pyx_cur_scope->__pyx_v_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 352, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":352 + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(stmt), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * + */ + __pyx_t_2 = __Pyx_PyInt_From_int(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 352, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("s3ql.deltadump.dump_table.lambda5", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":280 + * s3ql_sqlite_options - apsw_sqlite_options)) + * + * def dump_table(table, order, columns, db, fh): # <<<<<<<<<<<<<< + * '''Dump *columns* of *table* into *fh* + * + */ + +static PyObject *__pyx_pf_4s3ql_9deltadump_2dump_table(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_table, PyObject *__pyx_v_order, PyObject *__pyx_v_columns, PyObject *__pyx_v_db, PyObject *__pyx_v_fh) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *__pyx_cur_scope; + int __pyx_v_col_count; + int __pyx_v_rc; + int __pyx_v_i; + size_t __pyx_v_len_; + int64_t __pyx_v_int64; + int64_t __pyx_v_tmp; + const void *__pyx_v_buf; + int64_t __pyx_v_row_count; + PyObject *__pyx_v_cm = NULL; + PyObject *__pyx_v_dbfile_b = NULL; + PyObject *__pyx_v_col_names = NULL; + PyObject *__pyx_v_query = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + Py_ssize_t __pyx_t_10; + PyObject *__pyx_t_11 = NULL; + const char *__pyx_t_12; + int __pyx_t_13; + FILE *__pyx_t_14; + void *__pyx_t_15; + PyObject *(*__pyx_t_16)(PyObject *); + char *__pyx_t_17; + int64_t __pyx_t_18; + int __pyx_t_19; + int __pyx_t_20; + PyObject *__pyx_t_21 = NULL; + int __pyx_t_22; + __Pyx_RefNannySetupContext("dump_table", 0); + __pyx_cur_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *)__pyx_tp_new_4s3ql_9deltadump___pyx_scope_struct__dump_table(__pyx_ptype_4s3ql_9deltadump___pyx_scope_struct__dump_table, __pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __Pyx_RefNannyFinishContext(); + return NULL; + } + __Pyx_GOTREF(__pyx_cur_scope); + + /* "s3ql/deltadump.pyx":318 + * cdef int64_t row_count + * + * if db.file == ':memory:': # <<<<<<<<<<<<<< + * raise ValueError("Can't access in-memory databases") + * + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_db, __pyx_n_s_file); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 318, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = (__Pyx_PyUnicode_Equals(__pyx_t_1, __pyx_kp_u_memory, Py_EQ)); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 318, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":319 + * + * if db.file == ':memory:': + * raise ValueError("Can't access in-memory databases") # <<<<<<<<<<<<<< + * + * with ExitStack() as cm: + */ + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 319, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(0, 319, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":318 + * cdef int64_t row_count + * + * if db.file == ':memory:': # <<<<<<<<<<<<<< + * raise ValueError("Can't access in-memory databases") + * + */ + } + + /* "s3ql/deltadump.pyx":321 + * raise ValueError("Can't access in-memory databases") + * + * with ExitStack() as cm: # <<<<<<<<<<<<<< + * # Get SQLite connection + * log.debug('Opening connection to %s', db.file) + */ + /*with:*/ { + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ExitStack); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_4)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + } + } + if (__pyx_t_4) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } else { + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 321, __pyx_L1_error) + } + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_5 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_exit); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_4 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 321, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_4); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_4, function); + } + } + if (__pyx_t_6) { + __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_6); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 321, __pyx_L4_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else { + __pyx_t_3 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 321, __pyx_L4_error) + } + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __pyx_t_3; + __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + /*try:*/ { + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_7, &__pyx_t_8, &__pyx_t_9); + __Pyx_XGOTREF(__pyx_t_7); + __Pyx_XGOTREF(__pyx_t_8); + __Pyx_XGOTREF(__pyx_t_9); + /*try:*/ { + __pyx_v_cm = __pyx_t_4; + __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":323 + * with ExitStack() as cm: + * # Get SQLite connection + * log.debug('Opening connection to %s', db.file) # <<<<<<<<<<<<<< + * dbfile_b = db.file.encode(sys.getfilesystemencoding(), 'surrogateescape') + * SQLITE_CHECK_RC(sqlite3_open_v2(dbfile_b, &sqlite3_db, + */ + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_log); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 323, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_debug); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 323, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_db, __pyx_n_s_file); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 323, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = NULL; + __pyx_t_10 = 0; + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + __pyx_t_10 = 1; + } + } + __pyx_t_11 = PyTuple_New(2+__pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 323, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + if (__pyx_t_6) { + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_t_6); __pyx_t_6 = NULL; + } + __Pyx_INCREF(__pyx_kp_u_Opening_connection_to_s); + __Pyx_GIVEREF(__pyx_kp_u_Opening_connection_to_s); + PyTuple_SET_ITEM(__pyx_t_11, 0+__pyx_t_10, __pyx_kp_u_Opening_connection_to_s); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_11, 1+__pyx_t_10, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_11, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 323, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":324 + * # Get SQLite connection + * log.debug('Opening connection to %s', db.file) + * dbfile_b = db.file.encode(sys.getfilesystemencoding(), 'surrogateescape') # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_open_v2(dbfile_b, &sqlite3_db, + * SQLITE_OPEN_READONLY, NULL), + */ + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_db, __pyx_n_s_file); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 324, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_encode); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 324, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_sys); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 324, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_getfilesystemencoding); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 324, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_6); + if (likely(__pyx_t_1)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_1); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_6, function); + } + } + if (__pyx_t_1) { + __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 324, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else { + __pyx_t_3 = __Pyx_PyObject_CallNoArg(__pyx_t_6); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 324, __pyx_L8_error) + } + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = NULL; + __pyx_t_10 = 0; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + __pyx_t_10 = 1; + } + } + __pyx_t_1 = PyTuple_New(2+__pyx_t_10); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 324, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + if (__pyx_t_6) { + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6); __pyx_t_6 = NULL; + } + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_1, 0+__pyx_t_10, __pyx_t_3); + __Pyx_INCREF(__pyx_n_u_surrogateescape); + __Pyx_GIVEREF(__pyx_n_u_surrogateescape); + PyTuple_SET_ITEM(__pyx_t_1, 1+__pyx_t_10, __pyx_n_u_surrogateescape); + __pyx_t_3 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 324, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_v_dbfile_b = __pyx_t_4; + __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":325 + * log.debug('Opening connection to %s', db.file) + * dbfile_b = db.file.encode(sys.getfilesystemencoding(), 'surrogateescape') + * SQLITE_CHECK_RC(sqlite3_open_v2(dbfile_b, &sqlite3_db, # <<<<<<<<<<<<<< + * SQLITE_OPEN_READONLY, NULL), + * SQLITE_OK, sqlite3_db) + */ + __pyx_t_12 = __Pyx_PyObject_AsString(__pyx_v_dbfile_b); if (unlikely((!__pyx_t_12) && PyErr_Occurred())) __PYX_ERR(0, 325, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":327 + * SQLITE_CHECK_RC(sqlite3_open_v2(dbfile_b, &sqlite3_db, + * SQLITE_OPEN_READONLY, NULL), + * SQLITE_OK, sqlite3_db) # <<<<<<<<<<<<<< + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), + * SQLITE_OK, sqlite3_db)) + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_open_v2(__pyx_t_12, (&__pyx_cur_scope->__pyx_v_sqlite3_db), SQLITE_OPEN_READONLY, NULL), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 325, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":328 + * SQLITE_OPEN_READONLY, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * SQLITE_CHECK_RC(sqlite3_extended_result_codes(sqlite3_db, 1), + */ + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 328, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10dump_table_lambda, 0, __pyx_n_s_dump_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 328, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_3)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + } + } + if (!__pyx_t_3) { + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 328, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 328, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); __pyx_t_3 = NULL; + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 328, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":330 + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), + * SQLITE_OK, sqlite3_db)) + * SQLITE_CHECK_RC(sqlite3_extended_result_codes(sqlite3_db, 1), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db) + * + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_extended_result_codes(__pyx_cur_scope->__pyx_v_sqlite3_db, 1), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 330, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":334 + * + * # Get FILE* for buffered reading from *fh* + * fp = dup_to_fp(fh, b'wb') # <<<<<<<<<<<<<< + * cm.callback(lambda: fclose(fp)) + * + */ + __pyx_t_14 = __pyx_f_4s3ql_9deltadump_dup_to_fp(__pyx_v_fh, ((const char *)"wb")); if (unlikely(__pyx_t_14 == NULL)) __PYX_ERR(0, 334, __pyx_L8_error) + __pyx_cur_scope->__pyx_v_fp = __pyx_t_14; + + /* "s3ql/deltadump.pyx":335 + * # Get FILE* for buffered reading from *fh* + * fp = dup_to_fp(fh, b'wb') + * cm.callback(lambda: fclose(fp)) # <<<<<<<<<<<<<< + * + * # Allocate col_args and col_types + */ + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 335, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_6 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10dump_table_1lambda1, 0, __pyx_n_s_dump_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 335, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_1 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_1)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_1); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + } + } + if (!__pyx_t_1) { + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_t_6); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 335, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else { + __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 335, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); __pyx_t_1 = NULL; + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_3, 0+1, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 335, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":338 + * + * # Allocate col_args and col_types + * col_count = prep_columns(columns, &col_types, &col_args) # <<<<<<<<<<<<<< + * cm.callback(lambda: free(col_args)) + * cm.callback(lambda: free(col_types)) + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_prep_columns(__pyx_v_columns, (&__pyx_cur_scope->__pyx_v_col_types), (&__pyx_cur_scope->__pyx_v_col_args)); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 338, __pyx_L8_error) + __pyx_v_col_count = __pyx_t_13; + + /* "s3ql/deltadump.pyx":339 + * # Allocate col_args and col_types + * col_count = prep_columns(columns, &col_types, &col_args) + * cm.callback(lambda: free(col_args)) # <<<<<<<<<<<<<< + * cm.callback(lambda: free(col_types)) + * + */ + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 339, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_3 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10dump_table_2lambda2, 0, __pyx_n_s_dump_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 339, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + } + } + if (!__pyx_t_6) { + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 339, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else { + __pyx_t_1 = PyTuple_New(1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 339, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6); __pyx_t_6 = NULL; + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_1, 0+1, __pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 339, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":340 + * col_count = prep_columns(columns, &col_types, &col_args) + * cm.callback(lambda: free(col_args)) + * cm.callback(lambda: free(col_types)) # <<<<<<<<<<<<<< + * + * # Allocate int64_prev + */ + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 340, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10dump_table_3lambda3, 0, __pyx_n_s_dump_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 340, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_3)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + } + } + if (!__pyx_t_3) { + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 340, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 340, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); __pyx_t_3 = NULL; + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 340, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":343 + * + * # Allocate int64_prev + * int64_prev = calloc( len(columns), sizeof(int64_t)) # <<<<<<<<<<<<<< + * cm.callback(lambda: free(int64_prev)) + * + */ + __pyx_t_10 = PyObject_Length(__pyx_v_columns); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 343, __pyx_L8_error) + __pyx_t_15 = __pyx_f_4s3ql_9deltadump_calloc(((size_t)__pyx_t_10), (sizeof(int64_t))); if (unlikely(__pyx_t_15 == NULL)) __PYX_ERR(0, 343, __pyx_L8_error) + __pyx_cur_scope->__pyx_v_int64_prev = ((int64_t *)__pyx_t_15); + + /* "s3ql/deltadump.pyx":344 + * # Allocate int64_prev + * int64_prev = calloc( len(columns), sizeof(int64_t)) + * cm.callback(lambda: free(int64_prev)) # <<<<<<<<<<<<<< + * + * # Prepare statement + */ + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 344, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_6 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10dump_table_4lambda4, 0, __pyx_n_s_dump_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 344, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_1 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_1)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_1); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + } + } + if (!__pyx_t_1) { + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_t_6); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 344, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else { + __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 344, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); __pyx_t_1 = NULL; + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_3, 0+1, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 344, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":347 + * + * # Prepare statement + * col_names = [ x[0] for x in columns ] # <<<<<<<<<<<<<< + * query = ("SELECT %s FROM %s ORDER BY %s " % + * (', '.join(col_names), table, order)).encode('utf-8') + */ + { /* enter inner scope */ + PyObject *__pyx_7genexpr__pyx_v_x = NULL; + __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 347, __pyx_L18_error) + __Pyx_GOTREF(__pyx_t_4); + if (likely(PyList_CheckExact(__pyx_v_columns)) || PyTuple_CheckExact(__pyx_v_columns)) { + __pyx_t_11 = __pyx_v_columns; __Pyx_INCREF(__pyx_t_11); __pyx_t_10 = 0; + __pyx_t_16 = NULL; + } else { + __pyx_t_10 = -1; __pyx_t_11 = PyObject_GetIter(__pyx_v_columns); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 347, __pyx_L18_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_16 = Py_TYPE(__pyx_t_11)->tp_iternext; if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 347, __pyx_L18_error) + } + for (;;) { + if (likely(!__pyx_t_16)) { + if (likely(PyList_CheckExact(__pyx_t_11))) { + if (__pyx_t_10 >= PyList_GET_SIZE(__pyx_t_11)) break; + #if CYTHON_COMPILING_IN_CPYTHON + __pyx_t_3 = PyList_GET_ITEM(__pyx_t_11, __pyx_t_10); __Pyx_INCREF(__pyx_t_3); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 347, __pyx_L18_error) + #else + __pyx_t_3 = PySequence_ITEM(__pyx_t_11, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 347, __pyx_L18_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + } else { + if (__pyx_t_10 >= PyTuple_GET_SIZE(__pyx_t_11)) break; + #if CYTHON_COMPILING_IN_CPYTHON + __pyx_t_3 = PyTuple_GET_ITEM(__pyx_t_11, __pyx_t_10); __Pyx_INCREF(__pyx_t_3); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 347, __pyx_L18_error) + #else + __pyx_t_3 = PySequence_ITEM(__pyx_t_11, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 347, __pyx_L18_error) + __Pyx_GOTREF(__pyx_t_3); + #endif + } + } else { + __pyx_t_3 = __pyx_t_16(__pyx_t_11); + if (unlikely(!__pyx_t_3)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); + else __PYX_ERR(0, 347, __pyx_L18_error) + } + break; + } + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_XDECREF_SET(__pyx_7genexpr__pyx_v_x, __pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_GetItemInt(__pyx_7genexpr__pyx_v_x, 0, long, 1, __Pyx_PyInt_From_long, 0, 0, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 347, __pyx_L18_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_4, (PyObject*)__pyx_t_3))) __PYX_ERR(0, 347, __pyx_L18_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_XDECREF(__pyx_7genexpr__pyx_v_x); + goto __pyx_L21_exit_scope; + __pyx_L18_error:; + __Pyx_XDECREF(__pyx_7genexpr__pyx_v_x); + goto __pyx_L8_error; + __pyx_L21_exit_scope:; + } /* exit inner scope */ + __pyx_v_col_names = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":349 + * col_names = [ x[0] for x in columns ] + * query = ("SELECT %s FROM %s ORDER BY %s " % + * (', '.join(col_names), table, order)).encode('utf-8') # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + * SQLITE_OK, sqlite3_db) + */ + __pyx_t_4 = PyUnicode_Join(__pyx_kp_u__4, __pyx_v_col_names); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 349, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_11 = PyTuple_New(3); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 349, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_t_4); + __Pyx_INCREF(__pyx_v_table); + __Pyx_GIVEREF(__pyx_v_table); + PyTuple_SET_ITEM(__pyx_t_11, 1, __pyx_v_table); + __Pyx_INCREF(__pyx_v_order); + __Pyx_GIVEREF(__pyx_v_order); + PyTuple_SET_ITEM(__pyx_t_11, 2, __pyx_v_order); + __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":348 + * # Prepare statement + * col_names = [ x[0] for x in columns ] + * query = ("SELECT %s FROM %s ORDER BY %s " % # <<<<<<<<<<<<<< + * (', '.join(col_names), table, order)).encode('utf-8') + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + */ + __pyx_t_4 = PyUnicode_Format(__pyx_kp_u_SELECT_s_FROM_s_ORDER_BY_s, __pyx_t_11); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 348, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "s3ql/deltadump.pyx":349 + * col_names = [ x[0] for x in columns ] + * query = ("SELECT %s FROM %s ORDER BY %s " % + * (', '.join(col_names), table, order)).encode('utf-8') # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + * SQLITE_OK, sqlite3_db) + */ + __pyx_t_11 = PyUnicode_AsUTF8String(((PyObject*)__pyx_t_4)); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 349, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_v_query = __pyx_t_11; + __pyx_t_11 = 0; + + /* "s3ql/deltadump.pyx":350 + * query = ("SELECT %s FROM %s ORDER BY %s " % + * (', '.join(col_names), table, order)).encode('utf-8') + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(stmt), + */ + __pyx_t_17 = __Pyx_PyObject_AsString(__pyx_v_query); if (unlikely((!__pyx_t_17) && PyErr_Occurred())) __PYX_ERR(0, 350, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":351 + * (', '.join(col_names), table, order)).encode('utf-8') + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + * SQLITE_OK, sqlite3_db) # <<<<<<<<<<<<<< + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(stmt), + * SQLITE_OK, sqlite3_db)) + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_prepare_v2(__pyx_cur_scope->__pyx_v_sqlite3_db, __pyx_t_17, -1, (&__pyx_cur_scope->__pyx_v_stmt), NULL), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 350, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":352 + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(stmt), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * + */ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 352, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_3 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10dump_table_5lambda5, 0, __pyx_n_s_dump_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 352, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_4); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_4, function); + } + } + if (!__pyx_t_6) { + __pyx_t_11 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 352, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_11); + } else { + __pyx_t_1 = PyTuple_New(1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 352, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6); __pyx_t_6 = NULL; + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_1, 0+1, __pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_1, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 352, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "s3ql/deltadump.pyx":355 + * SQLITE_OK, sqlite3_db)) + * + * row_count = db.get_val("SELECT COUNT(rowid) FROM %s" % table) # <<<<<<<<<<<<<< + * log.debug('dump_table(%s): writing %d rows', table, row_count) + * write_integer(row_count, fp) + */ + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_db, __pyx_n_s_get_val); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 355, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = PyUnicode_Format(__pyx_kp_u_SELECT_COUNT_rowid_FROM_s, __pyx_v_table); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 355, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4); + if (likely(__pyx_t_3)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_4, function); + } + } + if (!__pyx_t_3) { + __pyx_t_11 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 355, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_11); + } else { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 355, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); __pyx_t_3 = NULL; + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 355, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_18 = __Pyx_PyInt_As_int64_t(__pyx_t_11); if (unlikely((__pyx_t_18 == (int64_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 355, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_v_row_count = __pyx_t_18; + + /* "s3ql/deltadump.pyx":356 + * + * row_count = db.get_val("SELECT COUNT(rowid) FROM %s" % table) + * log.debug('dump_table(%s): writing %d rows', table, row_count) # <<<<<<<<<<<<<< + * write_integer(row_count, fp) + * + */ + __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_log); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 356, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_debug); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 356, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyInt_From_int64_t(__pyx_v_row_count); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 356, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = NULL; + __pyx_t_10 = 0; + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_6); + if (likely(__pyx_t_1)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_1); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_6, function); + __pyx_t_10 = 1; + } + } + __pyx_t_3 = PyTuple_New(3+__pyx_t_10); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 356, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + if (__pyx_t_1) { + __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); __pyx_t_1 = NULL; + } + __Pyx_INCREF(__pyx_kp_u_dump_table_s_writing_d_rows); + __Pyx_GIVEREF(__pyx_kp_u_dump_table_s_writing_d_rows); + PyTuple_SET_ITEM(__pyx_t_3, 0+__pyx_t_10, __pyx_kp_u_dump_table_s_writing_d_rows); + __Pyx_INCREF(__pyx_v_table); + __Pyx_GIVEREF(__pyx_v_table); + PyTuple_SET_ITEM(__pyx_t_3, 1+__pyx_t_10, __pyx_v_table); + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_3, 2+__pyx_t_10, __pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_3, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 356, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + + /* "s3ql/deltadump.pyx":357 + * row_count = db.get_val("SELECT COUNT(rowid) FROM %s" % table) + * log.debug('dump_table(%s): writing %d rows', table, row_count) + * write_integer(row_count, fp) # <<<<<<<<<<<<<< + * + * # Iterate through rows + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_write_integer(__pyx_v_row_count, __pyx_cur_scope->__pyx_v_fp); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 357, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":360 + * + * # Iterate through rows + * while True: # <<<<<<<<<<<<<< + * rc = sqlite3_step(stmt) + * if rc == SQLITE_DONE: + */ + while (1) { + + /* "s3ql/deltadump.pyx":361 + * # Iterate through rows + * while True: + * rc = sqlite3_step(stmt) # <<<<<<<<<<<<<< + * if rc == SQLITE_DONE: + * break + */ + __pyx_v_rc = sqlite3_step(__pyx_cur_scope->__pyx_v_stmt); + + /* "s3ql/deltadump.pyx":362 + * while True: + * rc = sqlite3_step(stmt) + * if rc == SQLITE_DONE: # <<<<<<<<<<<<<< + * break + * SQLITE_CHECK_RC(rc, SQLITE_ROW, sqlite3_db) + */ + __pyx_t_2 = ((__pyx_v_rc == SQLITE_DONE) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":363 + * rc = sqlite3_step(stmt) + * if rc == SQLITE_DONE: + * break # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(rc, SQLITE_ROW, sqlite3_db) + * + */ + goto __pyx_L23_break; + + /* "s3ql/deltadump.pyx":362 + * while True: + * rc = sqlite3_step(stmt) + * if rc == SQLITE_DONE: # <<<<<<<<<<<<<< + * break + * SQLITE_CHECK_RC(rc, SQLITE_ROW, sqlite3_db) + */ + } + + /* "s3ql/deltadump.pyx":364 + * if rc == SQLITE_DONE: + * break + * SQLITE_CHECK_RC(rc, SQLITE_ROW, sqlite3_db) # <<<<<<<<<<<<<< + * + * for i in range(col_count): + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(__pyx_v_rc, SQLITE_ROW, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 364, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":366 + * SQLITE_CHECK_RC(rc, SQLITE_ROW, sqlite3_db) + * + * for i in range(col_count): # <<<<<<<<<<<<<< + * if sqlite3_column_type(stmt, i) is SQLITE_NULL: + * raise ValueError("Can't dump NULL values") + */ + __pyx_t_13 = __pyx_v_col_count; + for (__pyx_t_19 = 0; __pyx_t_19 < __pyx_t_13; __pyx_t_19+=1) { + __pyx_v_i = __pyx_t_19; + + /* "s3ql/deltadump.pyx":367 + * + * for i in range(col_count): + * if sqlite3_column_type(stmt, i) is SQLITE_NULL: # <<<<<<<<<<<<<< + * raise ValueError("Can't dump NULL values") + * + */ + __pyx_t_2 = ((sqlite3_column_type(__pyx_cur_scope->__pyx_v_stmt, __pyx_v_i) == SQLITE_NULL) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":368 + * for i in range(col_count): + * if sqlite3_column_type(stmt, i) is SQLITE_NULL: + * raise ValueError("Can't dump NULL values") # <<<<<<<<<<<<<< + * + * if col_types[i] == _INTEGER: + */ + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 368, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_Raise(__pyx_t_11, 0, 0, 0); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __PYX_ERR(0, 368, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":367 + * + * for i in range(col_count): + * if sqlite3_column_type(stmt, i) is SQLITE_NULL: # <<<<<<<<<<<<<< + * raise ValueError("Can't dump NULL values") + * + */ + } + + /* "s3ql/deltadump.pyx":370 + * raise ValueError("Can't dump NULL values") + * + * if col_types[i] == _INTEGER: # <<<<<<<<<<<<<< + * int64 = sqlite3_column_int64(stmt, i) + * tmp = int64 + */ + __pyx_t_2 = (((__pyx_cur_scope->__pyx_v_col_types[__pyx_v_i]) == __pyx_v_4s3ql_9deltadump__INTEGER) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":371 + * + * if col_types[i] == _INTEGER: + * int64 = sqlite3_column_int64(stmt, i) # <<<<<<<<<<<<<< + * tmp = int64 + * int64 -= int64_prev[i] + col_args[i] + */ + __pyx_v_int64 = sqlite3_column_int64(__pyx_cur_scope->__pyx_v_stmt, __pyx_v_i); + + /* "s3ql/deltadump.pyx":372 + * if col_types[i] == _INTEGER: + * int64 = sqlite3_column_int64(stmt, i) + * tmp = int64 # <<<<<<<<<<<<<< + * int64 -= int64_prev[i] + col_args[i] + * int64_prev[i] = tmp + */ + __pyx_v_tmp = __pyx_v_int64; + + /* "s3ql/deltadump.pyx":373 + * int64 = sqlite3_column_int64(stmt, i) + * tmp = int64 + * int64 -= int64_prev[i] + col_args[i] # <<<<<<<<<<<<<< + * int64_prev[i] = tmp + * write_integer(int64, fp) + */ + __pyx_v_int64 = (__pyx_v_int64 - ((__pyx_cur_scope->__pyx_v_int64_prev[__pyx_v_i]) + (__pyx_cur_scope->__pyx_v_col_args[__pyx_v_i]))); + + /* "s3ql/deltadump.pyx":374 + * tmp = int64 + * int64 -= int64_prev[i] + col_args[i] + * int64_prev[i] = tmp # <<<<<<<<<<<<<< + * write_integer(int64, fp) + * + */ + (__pyx_cur_scope->__pyx_v_int64_prev[__pyx_v_i]) = __pyx_v_tmp; + + /* "s3ql/deltadump.pyx":375 + * int64 -= int64_prev[i] + col_args[i] + * int64_prev[i] = tmp + * write_integer(int64, fp) # <<<<<<<<<<<<<< + * + * elif col_types[i] == _TIME: + */ + __pyx_t_20 = __pyx_f_4s3ql_9deltadump_write_integer(__pyx_v_int64, __pyx_cur_scope->__pyx_v_fp); if (unlikely(__pyx_t_20 == -1)) __PYX_ERR(0, 375, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":370 + * raise ValueError("Can't dump NULL values") + * + * if col_types[i] == _INTEGER: # <<<<<<<<<<<<<< + * int64 = sqlite3_column_int64(stmt, i) + * tmp = int64 + */ + goto __pyx_L28; + } + + /* "s3ql/deltadump.pyx":377 + * write_integer(int64, fp) + * + * elif col_types[i] == _TIME: # <<<<<<<<<<<<<< + * int64 = (sqlite3_column_double(stmt, i) * time_scale) + * tmp = int64 + */ + __pyx_t_2 = (((__pyx_cur_scope->__pyx_v_col_types[__pyx_v_i]) == __pyx_v_4s3ql_9deltadump__TIME) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":378 + * + * elif col_types[i] == _TIME: + * int64 = (sqlite3_column_double(stmt, i) * time_scale) # <<<<<<<<<<<<<< + * tmp = int64 + * int64 -= int64_prev[i] + col_args[i] + */ + __pyx_v_int64 = ((int64_t)(sqlite3_column_double(__pyx_cur_scope->__pyx_v_stmt, __pyx_v_i) * __pyx_v_4s3ql_9deltadump_time_scale)); + + /* "s3ql/deltadump.pyx":379 + * elif col_types[i] == _TIME: + * int64 = (sqlite3_column_double(stmt, i) * time_scale) + * tmp = int64 # <<<<<<<<<<<<<< + * int64 -= int64_prev[i] + col_args[i] + * int64_prev[i] = tmp + */ + __pyx_v_tmp = __pyx_v_int64; + + /* "s3ql/deltadump.pyx":380 + * int64 = (sqlite3_column_double(stmt, i) * time_scale) + * tmp = int64 + * int64 -= int64_prev[i] + col_args[i] # <<<<<<<<<<<<<< + * int64_prev[i] = tmp + * write_integer(int64, fp) + */ + __pyx_v_int64 = (__pyx_v_int64 - ((__pyx_cur_scope->__pyx_v_int64_prev[__pyx_v_i]) + (__pyx_cur_scope->__pyx_v_col_args[__pyx_v_i]))); + + /* "s3ql/deltadump.pyx":381 + * tmp = int64 + * int64 -= int64_prev[i] + col_args[i] + * int64_prev[i] = tmp # <<<<<<<<<<<<<< + * write_integer(int64, fp) + * + */ + (__pyx_cur_scope->__pyx_v_int64_prev[__pyx_v_i]) = __pyx_v_tmp; + + /* "s3ql/deltadump.pyx":382 + * int64 -= int64_prev[i] + col_args[i] + * int64_prev[i] = tmp + * write_integer(int64, fp) # <<<<<<<<<<<<<< + * + * elif col_types[i] == _BLOB: + */ + __pyx_t_20 = __pyx_f_4s3ql_9deltadump_write_integer(__pyx_v_int64, __pyx_cur_scope->__pyx_v_fp); if (unlikely(__pyx_t_20 == -1)) __PYX_ERR(0, 382, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":377 + * write_integer(int64, fp) + * + * elif col_types[i] == _TIME: # <<<<<<<<<<<<<< + * int64 = (sqlite3_column_double(stmt, i) * time_scale) + * tmp = int64 + */ + goto __pyx_L28; + } + + /* "s3ql/deltadump.pyx":384 + * write_integer(int64, fp) + * + * elif col_types[i] == _BLOB: # <<<<<<<<<<<<<< + * buf = sqlite3_column_blob(stmt, i) + * rc = sqlite3_column_bytes(stmt, i) + */ + __pyx_t_2 = (((__pyx_cur_scope->__pyx_v_col_types[__pyx_v_i]) == __pyx_v_4s3ql_9deltadump__BLOB) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":385 + * + * elif col_types[i] == _BLOB: + * buf = sqlite3_column_blob(stmt, i) # <<<<<<<<<<<<<< + * rc = sqlite3_column_bytes(stmt, i) + * if rc > MAX_BLOB_SIZE: + */ + __pyx_v_buf = sqlite3_column_blob(__pyx_cur_scope->__pyx_v_stmt, __pyx_v_i); + + /* "s3ql/deltadump.pyx":386 + * elif col_types[i] == _BLOB: + * buf = sqlite3_column_blob(stmt, i) + * rc = sqlite3_column_bytes(stmt, i) # <<<<<<<<<<<<<< + * if rc > MAX_BLOB_SIZE: + * raise ValueError('Can not dump BLOB of size %d (max: %d)', + */ + __pyx_v_rc = sqlite3_column_bytes(__pyx_cur_scope->__pyx_v_stmt, __pyx_v_i); + + /* "s3ql/deltadump.pyx":387 + * buf = sqlite3_column_blob(stmt, i) + * rc = sqlite3_column_bytes(stmt, i) + * if rc > MAX_BLOB_SIZE: # <<<<<<<<<<<<<< + * raise ValueError('Can not dump BLOB of size %d (max: %d)', + * rc, MAX_BLOB_SIZE) + */ + __pyx_t_11 = __Pyx_PyInt_From_int(__pyx_v_rc); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 387, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_MAX_BLOB_SIZE); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 387, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_3 = PyObject_RichCompare(__pyx_t_11, __pyx_t_6, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 387, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 387, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":389 + * if rc > MAX_BLOB_SIZE: + * raise ValueError('Can not dump BLOB of size %d (max: %d)', + * rc, MAX_BLOB_SIZE) # <<<<<<<<<<<<<< + * # Safe to cast now + * len_ = rc + */ + __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_rc); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 389, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_MAX_BLOB_SIZE); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 389, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + + /* "s3ql/deltadump.pyx":388 + * rc = sqlite3_column_bytes(stmt, i) + * if rc > MAX_BLOB_SIZE: + * raise ValueError('Can not dump BLOB of size %d (max: %d)', # <<<<<<<<<<<<<< + * rc, MAX_BLOB_SIZE) + * # Safe to cast now + */ + __pyx_t_11 = PyTuple_New(3); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 388, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_INCREF(__pyx_kp_u_Can_not_dump_BLOB_of_size_d_max); + __Pyx_GIVEREF(__pyx_kp_u_Can_not_dump_BLOB_of_size_d_max); + PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_kp_u_Can_not_dump_BLOB_of_size_d_max); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_11, 1, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_11, 2, __pyx_t_6); + __pyx_t_3 = 0; + __pyx_t_6 = 0; + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_t_11, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 388, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_Raise(__pyx_t_6, 0, 0, 0); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __PYX_ERR(0, 388, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":387 + * buf = sqlite3_column_blob(stmt, i) + * rc = sqlite3_column_bytes(stmt, i) + * if rc > MAX_BLOB_SIZE: # <<<<<<<<<<<<<< + * raise ValueError('Can not dump BLOB of size %d (max: %d)', + * rc, MAX_BLOB_SIZE) + */ + } + + /* "s3ql/deltadump.pyx":391 + * rc, MAX_BLOB_SIZE) + * # Safe to cast now + * len_ = rc # <<<<<<<<<<<<<< + * if col_args[i] == 0: + * write_integer(rc - int64_prev[i], fp) + */ + __pyx_v_len_ = ((size_t)__pyx_v_rc); + + /* "s3ql/deltadump.pyx":392 + * # Safe to cast now + * len_ = rc + * if col_args[i] == 0: # <<<<<<<<<<<<<< + * write_integer(rc - int64_prev[i], fp) + * int64_prev[i] = rc + */ + __pyx_t_2 = (((__pyx_cur_scope->__pyx_v_col_args[__pyx_v_i]) == 0) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":393 + * len_ = rc + * if col_args[i] == 0: + * write_integer(rc - int64_prev[i], fp) # <<<<<<<<<<<<<< + * int64_prev[i] = rc + * elif rc != col_args[i]: + */ + __pyx_t_20 = __pyx_f_4s3ql_9deltadump_write_integer((__pyx_v_rc - (__pyx_cur_scope->__pyx_v_int64_prev[__pyx_v_i])), __pyx_cur_scope->__pyx_v_fp); if (unlikely(__pyx_t_20 == -1)) __PYX_ERR(0, 393, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":394 + * if col_args[i] == 0: + * write_integer(rc - int64_prev[i], fp) + * int64_prev[i] = rc # <<<<<<<<<<<<<< + * elif rc != col_args[i]: + * raise ValueError("Length %d != %d in column %d" % (len_, col_args[i], i)) + */ + (__pyx_cur_scope->__pyx_v_int64_prev[__pyx_v_i]) = __pyx_v_rc; + + /* "s3ql/deltadump.pyx":392 + * # Safe to cast now + * len_ = rc + * if col_args[i] == 0: # <<<<<<<<<<<<<< + * write_integer(rc - int64_prev[i], fp) + * int64_prev[i] = rc + */ + goto __pyx_L30; + } + + /* "s3ql/deltadump.pyx":395 + * write_integer(rc - int64_prev[i], fp) + * int64_prev[i] = rc + * elif rc != col_args[i]: # <<<<<<<<<<<<<< + * raise ValueError("Length %d != %d in column %d" % (len_, col_args[i], i)) + * + */ + __pyx_t_2 = ((__pyx_v_rc != (__pyx_cur_scope->__pyx_v_col_args[__pyx_v_i])) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":396 + * int64_prev[i] = rc + * elif rc != col_args[i]: + * raise ValueError("Length %d != %d in column %d" % (len_, col_args[i], i)) # <<<<<<<<<<<<<< + * + * if len_ != 0: + */ + __pyx_t_6 = __Pyx_PyInt_FromSize_t(__pyx_v_len_); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 396, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_11 = __Pyx_PyInt_From_int((__pyx_cur_scope->__pyx_v_col_args[__pyx_v_i])); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 396, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_i); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 396, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = PyTuple_New(3); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 396, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_6); + __Pyx_GIVEREF(__pyx_t_11); + PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_t_11); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_4, 2, __pyx_t_3); + __pyx_t_6 = 0; + __pyx_t_11 = 0; + __pyx_t_3 = 0; + __pyx_t_3 = PyUnicode_Format(__pyx_kp_u_Length_d_d_in_column_d, __pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 396, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 396, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_t_4, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 396, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 396, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":395 + * write_integer(rc - int64_prev[i], fp) + * int64_prev[i] = rc + * elif rc != col_args[i]: # <<<<<<<<<<<<<< + * raise ValueError("Length %d != %d in column %d" % (len_, col_args[i], i)) + * + */ + } + __pyx_L30:; + + /* "s3ql/deltadump.pyx":398 + * raise ValueError("Length %d != %d in column %d" % (len_, col_args[i], i)) + * + * if len_ != 0: # <<<<<<<<<<<<<< + * fwrite(buf, len_, fp) + * + */ + __pyx_t_2 = ((__pyx_v_len_ != 0) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":399 + * + * if len_ != 0: + * fwrite(buf, len_, fp) # <<<<<<<<<<<<<< + * + * def load_table(table, columns, db, fh, trx_rows=5000): + */ + __pyx_t_20 = __pyx_f_4s3ql_9deltadump_fwrite(__pyx_v_buf, __pyx_v_len_, __pyx_cur_scope->__pyx_v_fp); if (unlikely(__pyx_t_20 == -1)) __PYX_ERR(0, 399, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":398 + * raise ValueError("Length %d != %d in column %d" % (len_, col_args[i], i)) + * + * if len_ != 0: # <<<<<<<<<<<<<< + * fwrite(buf, len_, fp) + * + */ + } + + /* "s3ql/deltadump.pyx":384 + * write_integer(int64, fp) + * + * elif col_types[i] == _BLOB: # <<<<<<<<<<<<<< + * buf = sqlite3_column_blob(stmt, i) + * rc = sqlite3_column_bytes(stmt, i) + */ + } + __pyx_L28:; + } + } + __pyx_L23_break:; + + /* "s3ql/deltadump.pyx":321 + * raise ValueError("Can't access in-memory databases") + * + * with ExitStack() as cm: # <<<<<<<<<<<<<< + * # Get SQLite connection + * log.debug('Opening connection to %s', db.file) + */ + } + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + goto __pyx_L15_try_end; + __pyx_L8_error:; + __Pyx_PyThreadState_assign + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + /*except:*/ { + __Pyx_AddTraceback("s3ql.deltadump.dump_table", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_3, &__pyx_t_4, &__pyx_t_11) < 0) __PYX_ERR(0, 321, __pyx_L10_except_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_6 = PyTuple_Pack(3, __pyx_t_3, __pyx_t_4, __pyx_t_11); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 321, __pyx_L10_except_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_21 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_6, NULL); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_21)) __PYX_ERR(0, 321, __pyx_L10_except_error) + __Pyx_GOTREF(__pyx_t_21); + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_21); + __Pyx_DECREF(__pyx_t_21); __pyx_t_21 = 0; + if (__pyx_t_2 < 0) __PYX_ERR(0, 321, __pyx_L10_except_error) + __pyx_t_22 = ((!(__pyx_t_2 != 0)) != 0); + if (__pyx_t_22) { + __Pyx_GIVEREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_11); + __Pyx_ErrRestoreWithState(__pyx_t_3, __pyx_t_4, __pyx_t_11); + __pyx_t_3 = 0; __pyx_t_4 = 0; __pyx_t_11 = 0; + __PYX_ERR(0, 321, __pyx_L10_except_error) + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + goto __pyx_L9_exception_handled; + } + __pyx_L10_except_error:; + __Pyx_PyThreadState_assign + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_XGIVEREF(__pyx_t_9); + __Pyx_ExceptionReset(__pyx_t_7, __pyx_t_8, __pyx_t_9); + goto __pyx_L1_error; + __pyx_L9_exception_handled:; + __Pyx_PyThreadState_assign + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_XGIVEREF(__pyx_t_9); + __Pyx_ExceptionReset(__pyx_t_7, __pyx_t_8, __pyx_t_9); + __pyx_L15_try_end:; + } + } + /*finally:*/ { + /*normal exit:*/{ + if (__pyx_t_5) { + __pyx_t_9 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_tuple__6, NULL); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + } + goto __pyx_L7; + } + __pyx_L7:; + } + goto __pyx_L35; + __pyx_L4_error:; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + goto __pyx_L1_error; + __pyx_L35:; + } + + /* "s3ql/deltadump.pyx":280 + * s3ql_sqlite_options - apsw_sqlite_options)) + * + * def dump_table(table, order, columns, db, fh): # <<<<<<<<<<<<<< + * '''Dump *columns* of *table* into *fh* + * + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_AddTraceback("s3ql.deltadump.dump_table", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_cm); + __Pyx_XDECREF(__pyx_v_dbfile_b); + __Pyx_XDECREF(__pyx_v_col_names); + __Pyx_XDECREF(__pyx_v_query); + __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":401 + * fwrite(buf, len_, fp) + * + * def load_table(table, columns, db, fh, trx_rows=5000): # <<<<<<<<<<<<<< + * '''Load *columns* of *table* from *fh* + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_5load_table(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_4s3ql_9deltadump_4load_table[] = "load_table(table, columns, db, fh, trx_rows=5000)\nLoad *columns* of *table* from *fh*\n\n *db* is an `s3ql.Connection` instance for the database.\n\n *columns* must be the same list of 3-tuples that was passed to\n `dump_table` when creating the dump stored in *fh*.\n\n This function will open a separate connection to the database, so\n the *db* connection should not be in EXCLUSIVE locking mode.\n (Using a separate connection avoids the requirement on the *apsw*\n and *deltadump* modules be linked against against binary\n compatible SQLite libraries).\n\n When writing into the table, a new transaction will be started\n every *trx_rows* rows.\n "; +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_5load_table = {"load_table", (PyCFunction)__pyx_pw_4s3ql_9deltadump_5load_table, METH_VARARGS|METH_KEYWORDS, __pyx_doc_4s3ql_9deltadump_4load_table}; +static PyObject *__pyx_pw_4s3ql_9deltadump_5load_table(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_table = 0; + PyObject *__pyx_v_columns = 0; + PyObject *__pyx_v_db = 0; + PyObject *__pyx_v_fh = 0; + PyObject *__pyx_v_trx_rows = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("load_table (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_table,&__pyx_n_s_columns,&__pyx_n_s_db,&__pyx_n_s_fh,&__pyx_n_s_trx_rows,0}; + PyObject* values[5] = {0,0,0,0,0}; + values[4] = ((PyObject *)__pyx_int_5000); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_table)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_columns)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("load_table", 0, 4, 5, 1); __PYX_ERR(0, 401, __pyx_L3_error) + } + case 2: + if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_db)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("load_table", 0, 4, 5, 2); __PYX_ERR(0, 401, __pyx_L3_error) + } + case 3: + if (likely((values[3] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_fh)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("load_table", 0, 4, 5, 3); __PYX_ERR(0, 401, __pyx_L3_error) + } + case 4: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_trx_rows); + if (value) { values[4] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "load_table") < 0)) __PYX_ERR(0, 401, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 5: values[4] = PyTuple_GET_ITEM(__pyx_args, 4); + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_table = values[0]; + __pyx_v_columns = values[1]; + __pyx_v_db = values[2]; + __pyx_v_fh = values[3]; + __pyx_v_trx_rows = values[4]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("load_table", 0, 4, 5, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(0, 401, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("s3ql.deltadump.load_table", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_4s3ql_9deltadump_4load_table(__pyx_self, __pyx_v_table, __pyx_v_columns, __pyx_v_db, __pyx_v_fh, __pyx_v_trx_rows); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":441 + * SQLITE_OPEN_READWRITE, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * SQLITE_CHECK_RC(sqlite3_extended_result_codes(sqlite3_db, 1), + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_lambda6(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10load_table_lambda6 = {"lambda6", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10load_table_lambda6, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_lambda6(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda6 (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda6(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda6(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannySetupContext("lambda6", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + + /* "s3ql/deltadump.pyx":442 + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), + * SQLITE_OK, sqlite3_db)) # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_extended_result_codes(sqlite3_db, 1), + * SQLITE_OK, sqlite3_db) + */ + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_close(__pyx_cur_scope->__pyx_v_sqlite3_db), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 441, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":441 + * SQLITE_OPEN_READWRITE, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * SQLITE_CHECK_RC(sqlite3_extended_result_codes(sqlite3_db, 1), + */ + __pyx_t_2 = __Pyx_PyInt_From_int(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 441, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("s3ql.deltadump.load_table.lambda6", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":462 + * # Get FILE* for buffered reading from *fh* + * fp = dup_to_fp(fh, b'rb') + * cm.callback(lambda: fclose(fp)) # <<<<<<<<<<<<<< + * + * # Allocate col_args and col_types + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_1lambda7(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10load_table_1lambda7 = {"lambda7", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10load_table_1lambda7, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_1lambda7(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda7 (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda7(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda7(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannySetupContext("lambda7", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_fclose(__pyx_cur_scope->__pyx_v_fp); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 462, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyInt_From_int(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 462, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("s3ql.deltadump.load_table.lambda7", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":466 + * # Allocate col_args and col_types + * col_count = prep_columns(columns, &col_types, &col_args) + * cm.callback(lambda: free(col_args)) # <<<<<<<<<<<<<< + * cm.callback(lambda: free(col_types)) + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_2lambda8(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10load_table_2lambda8 = {"lambda8", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10load_table_2lambda8, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_2lambda8(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda8 (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda8(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda8(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("lambda8", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_free(__pyx_cur_scope->__pyx_v_col_args); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 466, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("s3ql.deltadump.load_table.lambda8", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":467 + * col_count = prep_columns(columns, &col_types, &col_args) + * cm.callback(lambda: free(col_args)) + * cm.callback(lambda: free(col_types)) # <<<<<<<<<<<<<< + * + * # Allocate int64_prev + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_3lambda9(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10load_table_3lambda9 = {"lambda9", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10load_table_3lambda9, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_3lambda9(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda9 (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda9(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda9(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("lambda9", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_free(__pyx_cur_scope->__pyx_v_col_types); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 467, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("s3ql.deltadump.load_table.lambda9", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":471 + * # Allocate int64_prev + * int64_prev = calloc( len(columns), sizeof(int64_t)) + * cm.callback(lambda: free(int64_prev)) # <<<<<<<<<<<<<< + * + * # Prepare INSERT statement + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_4lambda10(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10load_table_4lambda10 = {"lambda10", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10load_table_4lambda10, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_4lambda10(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda10 (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda10(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda10(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("lambda10", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_free(__pyx_cur_scope->__pyx_v_int64_prev); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 471, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("s3ql.deltadump.load_table.lambda10", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":480 + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(stmt), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_5lambda11(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10load_table_5lambda11 = {"lambda11", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10load_table_5lambda11, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_5lambda11(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda11 (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda11(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda11(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannySetupContext("lambda11", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + + /* "s3ql/deltadump.pyx":481 + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(stmt), + * SQLITE_OK, sqlite3_db)) # <<<<<<<<<<<<<< + * + * # Prepare BEGIN statement + */ + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_finalize(__pyx_cur_scope->__pyx_v_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 480, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":480 + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(stmt), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * + */ + __pyx_t_2 = __Pyx_PyInt_From_int(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 480, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("s3ql.deltadump.load_table.lambda11", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":487 + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &begin_stmt, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(begin_stmt), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_6lambda12(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10load_table_6lambda12 = {"lambda12", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10load_table_6lambda12, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_6lambda12(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda12 (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda12(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda12(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannySetupContext("lambda12", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + + /* "s3ql/deltadump.pyx":488 + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(begin_stmt), + * SQLITE_OK, sqlite3_db)) # <<<<<<<<<<<<<< + * + * # Prepare COMMIT statement + */ + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_finalize(__pyx_cur_scope->__pyx_v_begin_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 487, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":487 + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &begin_stmt, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(begin_stmt), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * + */ + __pyx_t_2 = __Pyx_PyInt_From_int(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 487, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("s3ql.deltadump.load_table.lambda12", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":494 + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &commit_stmt, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(commit_stmt), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_7lambda13(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10load_table_7lambda13 = {"lambda13", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10load_table_7lambda13, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_7lambda13(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda13 (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda13(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda13(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannySetupContext("lambda13", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + + /* "s3ql/deltadump.pyx":495 + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(commit_stmt), + * SQLITE_OK, sqlite3_db)) # <<<<<<<<<<<<<< + * + * buf = calloc(MAX_BLOB_SIZE, 1) + */ + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_finalize(__pyx_cur_scope->__pyx_v_commit_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 494, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":494 + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &commit_stmt, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(commit_stmt), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * + */ + __pyx_t_2 = __Pyx_PyInt_From_int(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 494, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("s3ql.deltadump.load_table.lambda13", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":498 + * + * buf = calloc(MAX_BLOB_SIZE, 1) + * cm.callback(lambda: free(buf)) # <<<<<<<<<<<<<< + * read_integer(&row_count, fp) + * log.debug('load_table(%s): reading %d rows', table, row_count) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_8lambda14(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10load_table_8lambda14 = {"lambda14", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10load_table_8lambda14, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_8lambda14(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda14 (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda14(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda14(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("lambda14", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_free(__pyx_cur_scope->__pyx_v_buf); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 498, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("s3ql.deltadump.load_table.lambda14", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":504 + * # Start transaction + * SQLITE_CHECK_RC(sqlite3_step(begin_stmt), SQLITE_DONE, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_step(commit_stmt), # <<<<<<<<<<<<<< + * SQLITE_DONE, sqlite3_db)) + * SQLITE_CHECK_RC(sqlite3_reset(begin_stmt), SQLITE_OK, sqlite3_db) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_9lambda15(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyMethodDef __pyx_mdef_4s3ql_9deltadump_10load_table_9lambda15 = {"lambda15", (PyCFunction)__pyx_pw_4s3ql_9deltadump_10load_table_9lambda15, METH_NOARGS, 0}; +static PyObject *__pyx_pw_4s3ql_9deltadump_10load_table_9lambda15(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("lambda15 (wrapper)", 0); + __pyx_r = __pyx_lambda_funcdef_lambda15(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_lambda_funcdef_lambda15(PyObject *__pyx_self) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_cur_scope; + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannySetupContext("lambda15", 0); + __pyx_outer_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_XDECREF(__pyx_r); + + /* "s3ql/deltadump.pyx":505 + * SQLITE_CHECK_RC(sqlite3_step(begin_stmt), SQLITE_DONE, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_step(commit_stmt), + * SQLITE_DONE, sqlite3_db)) # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_reset(begin_stmt), SQLITE_OK, sqlite3_db) + * + */ + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_step(__pyx_cur_scope->__pyx_v_commit_stmt), SQLITE_DONE, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 504, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":504 + * # Start transaction + * SQLITE_CHECK_RC(sqlite3_step(begin_stmt), SQLITE_DONE, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_step(commit_stmt), # <<<<<<<<<<<<<< + * SQLITE_DONE, sqlite3_db)) + * SQLITE_CHECK_RC(sqlite3_reset(begin_stmt), SQLITE_OK, sqlite3_db) + */ + __pyx_t_2 = __Pyx_PyInt_From_int(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 504, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("s3ql.deltadump.load_table.lambda15", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":401 + * fwrite(buf, len_, fp) + * + * def load_table(table, columns, db, fh, trx_rows=5000): # <<<<<<<<<<<<<< + * '''Load *columns* of *table* from *fh* + * + */ + +static PyObject *__pyx_pf_4s3ql_9deltadump_4load_table(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_table, PyObject *__pyx_v_columns, PyObject *__pyx_v_db, PyObject *__pyx_v_fh, PyObject *__pyx_v_trx_rows) { + struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_cur_scope; + int __pyx_v_col_count; + int __pyx_v_rc; + int __pyx_v_len_; + int __pyx_v_i; + int __pyx_v_j; + int64_t __pyx_v_row_count; + int64_t __pyx_v_int64; + int64_t __pyx_v_tmp; + PyObject *__pyx_v_cm = NULL; + PyObject *__pyx_v_dbfile_b = NULL; + PyObject *__pyx_v_pragma = NULL; + PyObject *__pyx_v_val = NULL; + PyObject *__pyx_v_cmd = NULL; + PyObject *__pyx_v_col_names = NULL; + PyObject *__pyx_v_query = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + Py_ssize_t __pyx_t_10; + PyObject *__pyx_t_11 = NULL; + const char *__pyx_t_12; + int __pyx_t_13; + PyObject *__pyx_t_14 = NULL; + char *__pyx_t_15; + int __pyx_t_16; + char const *__pyx_t_17; + PyObject *__pyx_t_18 = NULL; + PyObject *__pyx_t_19 = NULL; + PyObject *__pyx_t_20 = NULL; + PyObject *__pyx_t_21 = NULL; + PyObject *__pyx_t_22 = NULL; + PyObject *__pyx_t_23 = NULL; + int __pyx_t_24; + FILE *__pyx_t_25; + void *__pyx_t_26; + PyObject *(*__pyx_t_27)(PyObject *); + size_t __pyx_t_28; + int64_t __pyx_t_29; + int __pyx_t_30; + int __pyx_t_31; + __Pyx_RefNannySetupContext("load_table", 0); + __pyx_cur_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *)__pyx_tp_new_4s3ql_9deltadump___pyx_scope_struct_1_load_table(__pyx_ptype_4s3ql_9deltadump___pyx_scope_struct_1_load_table, __pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __Pyx_RefNannyFinishContext(); + return NULL; + } + __Pyx_GOTREF(__pyx_cur_scope); + + /* "s3ql/deltadump.pyx":431 + * cdef int64_t row_count, int64, tmp + * + * if db.file == ':memory:': # <<<<<<<<<<<<<< + * raise ValueError("Can't access in-memory databases") + * + */ + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_db, __pyx_n_s_file); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 431, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = (__Pyx_PyUnicode_Equals(__pyx_t_1, __pyx_kp_u_memory, Py_EQ)); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 431, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":432 + * + * if db.file == ':memory:': + * raise ValueError("Can't access in-memory databases") # <<<<<<<<<<<<<< + * + * with ExitStack() as cm: + */ + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_ValueError, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(0, 432, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":431 + * cdef int64_t row_count, int64, tmp + * + * if db.file == ':memory:': # <<<<<<<<<<<<<< + * raise ValueError("Can't access in-memory databases") + * + */ + } + + /* "s3ql/deltadump.pyx":434 + * raise ValueError("Can't access in-memory databases") + * + * with ExitStack() as cm: # <<<<<<<<<<<<<< + * # Get SQLite connection + * log.debug('Opening connection to %s', db.file) + */ + /*with:*/ { + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ExitStack); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 434, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_4)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + } + } + if (__pyx_t_4) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 434, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } else { + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 434, __pyx_L1_error) + } + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_5 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_exit); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 434, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_4 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 434, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_4); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_4, function); + } + } + if (__pyx_t_6) { + __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_6); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 434, __pyx_L4_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else { + __pyx_t_3 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 434, __pyx_L4_error) + } + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = __pyx_t_3; + __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + /*try:*/ { + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_7, &__pyx_t_8, &__pyx_t_9); + __Pyx_XGOTREF(__pyx_t_7); + __Pyx_XGOTREF(__pyx_t_8); + __Pyx_XGOTREF(__pyx_t_9); + /*try:*/ { + __pyx_v_cm = __pyx_t_4; + __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":436 + * with ExitStack() as cm: + * # Get SQLite connection + * log.debug('Opening connection to %s', db.file) # <<<<<<<<<<<<<< + * dbfile_b = db.file.encode(sys.getfilesystemencoding(), 'surrogateescape') + * SQLITE_CHECK_RC(sqlite3_open_v2(dbfile_b, &sqlite3_db, + */ + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_log); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 436, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_debug); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 436, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_db, __pyx_n_s_file); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 436, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = NULL; + __pyx_t_10 = 0; + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + __pyx_t_10 = 1; + } + } + __pyx_t_11 = PyTuple_New(2+__pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 436, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + if (__pyx_t_6) { + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_t_6); __pyx_t_6 = NULL; + } + __Pyx_INCREF(__pyx_kp_u_Opening_connection_to_s); + __Pyx_GIVEREF(__pyx_kp_u_Opening_connection_to_s); + PyTuple_SET_ITEM(__pyx_t_11, 0+__pyx_t_10, __pyx_kp_u_Opening_connection_to_s); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_11, 1+__pyx_t_10, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_11, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 436, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":437 + * # Get SQLite connection + * log.debug('Opening connection to %s', db.file) + * dbfile_b = db.file.encode(sys.getfilesystemencoding(), 'surrogateescape') # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_open_v2(dbfile_b, &sqlite3_db, + * SQLITE_OPEN_READWRITE, NULL), + */ + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_db, __pyx_n_s_file); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 437, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_encode); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 437, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_sys); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 437, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_getfilesystemencoding); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 437, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_6); + if (likely(__pyx_t_1)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_1); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_6, function); + } + } + if (__pyx_t_1) { + __pyx_t_3 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 437, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else { + __pyx_t_3 = __Pyx_PyObject_CallNoArg(__pyx_t_6); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 437, __pyx_L8_error) + } + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = NULL; + __pyx_t_10 = 0; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + __pyx_t_10 = 1; + } + } + __pyx_t_1 = PyTuple_New(2+__pyx_t_10); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 437, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + if (__pyx_t_6) { + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6); __pyx_t_6 = NULL; + } + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_1, 0+__pyx_t_10, __pyx_t_3); + __Pyx_INCREF(__pyx_n_u_surrogateescape); + __Pyx_GIVEREF(__pyx_n_u_surrogateescape); + PyTuple_SET_ITEM(__pyx_t_1, 1+__pyx_t_10, __pyx_n_u_surrogateescape); + __pyx_t_3 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 437, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_v_dbfile_b = __pyx_t_4; + __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":438 + * log.debug('Opening connection to %s', db.file) + * dbfile_b = db.file.encode(sys.getfilesystemencoding(), 'surrogateescape') + * SQLITE_CHECK_RC(sqlite3_open_v2(dbfile_b, &sqlite3_db, # <<<<<<<<<<<<<< + * SQLITE_OPEN_READWRITE, NULL), + * SQLITE_OK, sqlite3_db) + */ + __pyx_t_12 = __Pyx_PyObject_AsString(__pyx_v_dbfile_b); if (unlikely((!__pyx_t_12) && PyErr_Occurred())) __PYX_ERR(0, 438, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":440 + * SQLITE_CHECK_RC(sqlite3_open_v2(dbfile_b, &sqlite3_db, + * SQLITE_OPEN_READWRITE, NULL), + * SQLITE_OK, sqlite3_db) # <<<<<<<<<<<<<< + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), + * SQLITE_OK, sqlite3_db)) + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_open_v2(__pyx_t_12, (&__pyx_cur_scope->__pyx_v_sqlite3_db), SQLITE_OPEN_READWRITE, NULL), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 438, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":441 + * SQLITE_OPEN_READWRITE, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * SQLITE_CHECK_RC(sqlite3_extended_result_codes(sqlite3_db, 1), + */ + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 441, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_lambda6, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 441, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_3)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + } + } + if (!__pyx_t_3) { + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 441, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 441, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); __pyx_t_3 = NULL; + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 441, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":443 + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), + * SQLITE_OK, sqlite3_db)) + * SQLITE_CHECK_RC(sqlite3_extended_result_codes(sqlite3_db, 1), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db) + * + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_extended_result_codes(__pyx_cur_scope->__pyx_v_sqlite3_db, 1), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 443, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":447 + * + * # Copy settings + * for pragma in ('synchronous', 'foreign_keys'): # <<<<<<<<<<<<<< + * val = db.get_val('PRAGMA %s' % pragma) + * cmd = ('PRAGMA %s = %s' % (pragma, val)).encode('utf-8') + */ + __pyx_t_4 = __pyx_tuple__8; __Pyx_INCREF(__pyx_t_4); __pyx_t_10 = 0; + for (;;) { + if (__pyx_t_10 >= 2) break; + #if CYTHON_COMPILING_IN_CPYTHON + __pyx_t_11 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_10); __Pyx_INCREF(__pyx_t_11); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 447, __pyx_L8_error) + #else + __pyx_t_11 = PySequence_ITEM(__pyx_t_4, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 447, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + #endif + __Pyx_XDECREF_SET(__pyx_v_pragma, ((PyObject*)__pyx_t_11)); + __pyx_t_11 = 0; + + /* "s3ql/deltadump.pyx":448 + * # Copy settings + * for pragma in ('synchronous', 'foreign_keys'): + * val = db.get_val('PRAGMA %s' % pragma) # <<<<<<<<<<<<<< + * cmd = ('PRAGMA %s = %s' % (pragma, val)).encode('utf-8') + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, cmd, -1, &stmt, NULL), + */ + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_db, __pyx_n_s_get_val); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 448, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_1 = PyUnicode_Format(__pyx_kp_u_PRAGMA_s, __pyx_v_pragma); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 448, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_6); + if (likely(__pyx_t_3)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_6, function); + } + } + if (!__pyx_t_3) { + __pyx_t_11 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_t_1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 448, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_11); + } else { + __pyx_t_14 = PyTuple_New(1+1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 448, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_3); __pyx_t_3 = NULL; + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_14, 0+1, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_14, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 448, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF_SET(__pyx_v_val, __pyx_t_11); + __pyx_t_11 = 0; + + /* "s3ql/deltadump.pyx":449 + * for pragma in ('synchronous', 'foreign_keys'): + * val = db.get_val('PRAGMA %s' % pragma) + * cmd = ('PRAGMA %s = %s' % (pragma, val)).encode('utf-8') # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, cmd, -1, &stmt, NULL), + * SQLITE_OK, sqlite3_db) + */ + __pyx_t_11 = PyTuple_New(2); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 449, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_INCREF(__pyx_v_pragma); + __Pyx_GIVEREF(__pyx_v_pragma); + PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_v_pragma); + __Pyx_INCREF(__pyx_v_val); + __Pyx_GIVEREF(__pyx_v_val); + PyTuple_SET_ITEM(__pyx_t_11, 1, __pyx_v_val); + __pyx_t_6 = PyUnicode_Format(__pyx_kp_u_PRAGMA_s_s, __pyx_t_11); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 449, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_t_11 = PyUnicode_AsUTF8String(((PyObject*)__pyx_t_6)); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 449, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF_SET(__pyx_v_cmd, __pyx_t_11); + __pyx_t_11 = 0; + + /* "s3ql/deltadump.pyx":450 + * val = db.get_val('PRAGMA %s' % pragma) + * cmd = ('PRAGMA %s = %s' % (pragma, val)).encode('utf-8') + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, cmd, -1, &stmt, NULL), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db) + * try: + */ + __pyx_t_15 = __Pyx_PyObject_AsString(__pyx_v_cmd); if (unlikely((!__pyx_t_15) && PyErr_Occurred())) __PYX_ERR(0, 450, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":451 + * cmd = ('PRAGMA %s = %s' % (pragma, val)).encode('utf-8') + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, cmd, -1, &stmt, NULL), + * SQLITE_OK, sqlite3_db) # <<<<<<<<<<<<<< + * try: + * rc = sqlite3_step(stmt) + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_prepare_v2(__pyx_cur_scope->__pyx_v_sqlite3_db, __pyx_t_15, -1, (&__pyx_cur_scope->__pyx_v_stmt), NULL), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 450, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":452 + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, cmd, -1, &stmt, NULL), + * SQLITE_OK, sqlite3_db) + * try: # <<<<<<<<<<<<<< + * rc = sqlite3_step(stmt) + * if rc == SQLITE_ROW: + */ + /*try:*/ { + + /* "s3ql/deltadump.pyx":453 + * SQLITE_OK, sqlite3_db) + * try: + * rc = sqlite3_step(stmt) # <<<<<<<<<<<<<< + * if rc == SQLITE_ROW: + * rc = sqlite3_step(stmt) + */ + __pyx_v_rc = sqlite3_step(__pyx_cur_scope->__pyx_v_stmt); + + /* "s3ql/deltadump.pyx":454 + * try: + * rc = sqlite3_step(stmt) + * if rc == SQLITE_ROW: # <<<<<<<<<<<<<< + * rc = sqlite3_step(stmt) + * SQLITE_CHECK_RC(rc, SQLITE_DONE, sqlite3_db) + */ + __pyx_t_2 = ((__pyx_v_rc == SQLITE_ROW) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":455 + * rc = sqlite3_step(stmt) + * if rc == SQLITE_ROW: + * rc = sqlite3_step(stmt) # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(rc, SQLITE_DONE, sqlite3_db) + * finally: + */ + __pyx_v_rc = sqlite3_step(__pyx_cur_scope->__pyx_v_stmt); + + /* "s3ql/deltadump.pyx":454 + * try: + * rc = sqlite3_step(stmt) + * if rc == SQLITE_ROW: # <<<<<<<<<<<<<< + * rc = sqlite3_step(stmt) + * SQLITE_CHECK_RC(rc, SQLITE_DONE, sqlite3_db) + */ + } + + /* "s3ql/deltadump.pyx":456 + * if rc == SQLITE_ROW: + * rc = sqlite3_step(stmt) + * SQLITE_CHECK_RC(rc, SQLITE_DONE, sqlite3_db) # <<<<<<<<<<<<<< + * finally: + * SQLITE_CHECK_RC(sqlite3_finalize(stmt), SQLITE_OK, sqlite3_db) + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(__pyx_v_rc, SQLITE_DONE, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 456, __pyx_L21_error) + } + + /* "s3ql/deltadump.pyx":458 + * SQLITE_CHECK_RC(rc, SQLITE_DONE, sqlite3_db) + * finally: + * SQLITE_CHECK_RC(sqlite3_finalize(stmt), SQLITE_OK, sqlite3_db) # <<<<<<<<<<<<<< + * + * # Get FILE* for buffered reading from *fh* + */ + /*finally:*/ { + /*normal exit:*/{ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_finalize(__pyx_cur_scope->__pyx_v_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 458, __pyx_L8_error) + goto __pyx_L22; + } + /*exception exit:*/{ + __Pyx_PyThreadState_declare + __pyx_L21_error:; + __pyx_t_18 = 0; __pyx_t_19 = 0; __pyx_t_20 = 0; __pyx_t_21 = 0; __pyx_t_22 = 0; __pyx_t_23 = 0; + __Pyx_PyThreadState_assign + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_11); __pyx_t_11 = 0; + if (PY_MAJOR_VERSION >= 3) __Pyx_ExceptionSwap(&__pyx_t_21, &__pyx_t_22, &__pyx_t_23); + if ((PY_MAJOR_VERSION < 3) || unlikely(__Pyx_GetException(&__pyx_t_18, &__pyx_t_19, &__pyx_t_20) < 0)) __Pyx_ErrFetch(&__pyx_t_18, &__pyx_t_19, &__pyx_t_20); + __Pyx_XGOTREF(__pyx_t_18); + __Pyx_XGOTREF(__pyx_t_19); + __Pyx_XGOTREF(__pyx_t_20); + __Pyx_XGOTREF(__pyx_t_21); + __Pyx_XGOTREF(__pyx_t_22); + __Pyx_XGOTREF(__pyx_t_23); + __pyx_t_13 = __pyx_lineno; __pyx_t_16 = __pyx_clineno; __pyx_t_17 = __pyx_filename; + { + __pyx_t_24 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_finalize(__pyx_cur_scope->__pyx_v_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_24 == -1)) __PYX_ERR(0, 458, __pyx_L27_error) + } + __Pyx_PyThreadState_assign + if (PY_MAJOR_VERSION >= 3) { + __Pyx_XGIVEREF(__pyx_t_21); + __Pyx_XGIVEREF(__pyx_t_22); + __Pyx_XGIVEREF(__pyx_t_23); + __Pyx_ExceptionReset(__pyx_t_21, __pyx_t_22, __pyx_t_23); + } + __Pyx_XGIVEREF(__pyx_t_18); + __Pyx_XGIVEREF(__pyx_t_19); + __Pyx_XGIVEREF(__pyx_t_20); + __Pyx_ErrRestore(__pyx_t_18, __pyx_t_19, __pyx_t_20); + __pyx_t_18 = 0; __pyx_t_19 = 0; __pyx_t_20 = 0; __pyx_t_21 = 0; __pyx_t_22 = 0; __pyx_t_23 = 0; + __pyx_lineno = __pyx_t_13; __pyx_clineno = __pyx_t_16; __pyx_filename = __pyx_t_17; + goto __pyx_L8_error; + __pyx_L27_error:; + __Pyx_PyThreadState_assign + if (PY_MAJOR_VERSION >= 3) { + __Pyx_XGIVEREF(__pyx_t_21); + __Pyx_XGIVEREF(__pyx_t_22); + __Pyx_XGIVEREF(__pyx_t_23); + __Pyx_ExceptionReset(__pyx_t_21, __pyx_t_22, __pyx_t_23); + } + __Pyx_XDECREF(__pyx_t_18); __pyx_t_18 = 0; + __Pyx_XDECREF(__pyx_t_19); __pyx_t_19 = 0; + __Pyx_XDECREF(__pyx_t_20); __pyx_t_20 = 0; + __pyx_t_21 = 0; __pyx_t_22 = 0; __pyx_t_23 = 0; + goto __pyx_L8_error; + } + __pyx_L22:; + } + + /* "s3ql/deltadump.pyx":447 + * + * # Copy settings + * for pragma in ('synchronous', 'foreign_keys'): # <<<<<<<<<<<<<< + * val = db.get_val('PRAGMA %s' % pragma) + * cmd = ('PRAGMA %s = %s' % (pragma, val)).encode('utf-8') + */ + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":461 + * + * # Get FILE* for buffered reading from *fh* + * fp = dup_to_fp(fh, b'rb') # <<<<<<<<<<<<<< + * cm.callback(lambda: fclose(fp)) + * + */ + __pyx_t_25 = __pyx_f_4s3ql_9deltadump_dup_to_fp(__pyx_v_fh, ((const char *)"rb")); if (unlikely(__pyx_t_25 == NULL)) __PYX_ERR(0, 461, __pyx_L8_error) + __pyx_cur_scope->__pyx_v_fp = __pyx_t_25; + + /* "s3ql/deltadump.pyx":462 + * # Get FILE* for buffered reading from *fh* + * fp = dup_to_fp(fh, b'rb') + * cm.callback(lambda: fclose(fp)) # <<<<<<<<<<<<<< + * + * # Allocate col_args and col_types + */ + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 462, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_6 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_1lambda7, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 462, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_14 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_14 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_14)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_14); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + } + } + if (!__pyx_t_14) { + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_t_6); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 462, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else { + __pyx_t_1 = PyTuple_New(1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 462, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_14); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_14); __pyx_t_14 = NULL; + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_1, 0+1, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 462, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":465 + * + * # Allocate col_args and col_types + * col_count = prep_columns(columns, &col_types, &col_args) # <<<<<<<<<<<<<< + * cm.callback(lambda: free(col_args)) + * cm.callback(lambda: free(col_types)) + */ + __pyx_t_16 = __pyx_f_4s3ql_9deltadump_prep_columns(__pyx_v_columns, (&__pyx_cur_scope->__pyx_v_col_types), (&__pyx_cur_scope->__pyx_v_col_args)); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 465, __pyx_L8_error) + __pyx_v_col_count = __pyx_t_16; + + /* "s3ql/deltadump.pyx":466 + * # Allocate col_args and col_types + * col_count = prep_columns(columns, &col_types, &col_args) + * cm.callback(lambda: free(col_args)) # <<<<<<<<<<<<<< + * cm.callback(lambda: free(col_types)) + * + */ + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 466, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_2lambda8, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 466, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + } + } + if (!__pyx_t_6) { + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 466, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else { + __pyx_t_14 = PyTuple_New(1+1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 466, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_6); __pyx_t_6 = NULL; + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_14, 0+1, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_14, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 466, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":467 + * col_count = prep_columns(columns, &col_types, &col_args) + * cm.callback(lambda: free(col_args)) + * cm.callback(lambda: free(col_types)) # <<<<<<<<<<<<<< + * + * # Allocate int64_prev + */ + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 467, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_14 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_3lambda9, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 467, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_1 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_1)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_1); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + } + } + if (!__pyx_t_1) { + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_t_14); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 467, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 467, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_1); __pyx_t_1 = NULL; + __Pyx_GIVEREF(__pyx_t_14); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_14); + __pyx_t_14 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 467, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":470 + * + * # Allocate int64_prev + * int64_prev = calloc( len(columns), sizeof(int64_t)) # <<<<<<<<<<<<<< + * cm.callback(lambda: free(int64_prev)) + * + */ + __pyx_t_10 = PyObject_Length(__pyx_v_columns); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 470, __pyx_L8_error) + __pyx_t_26 = __pyx_f_4s3ql_9deltadump_calloc(((size_t)__pyx_t_10), (sizeof(int64_t))); if (unlikely(__pyx_t_26 == NULL)) __PYX_ERR(0, 470, __pyx_L8_error) + __pyx_cur_scope->__pyx_v_int64_prev = ((int64_t *)__pyx_t_26); + + /* "s3ql/deltadump.pyx":471 + * # Allocate int64_prev + * int64_prev = calloc( len(columns), sizeof(int64_t)) + * cm.callback(lambda: free(int64_prev)) # <<<<<<<<<<<<<< + * + * # Prepare INSERT statement + */ + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 471, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_6 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_4lambda10, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 471, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_14 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_14 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_14)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_14); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + } + } + if (!__pyx_t_14) { + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_t_6); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 471, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else { + __pyx_t_1 = PyTuple_New(1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 471, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_14); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_14); __pyx_t_14 = NULL; + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_1, 0+1, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 471, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":474 + * + * # Prepare INSERT statement + * col_names = [ x[0] for x in columns ] # <<<<<<<<<<<<<< + * query = ("INSERT INTO %s (%s) VALUES(%s)" + * % (table, ', '.join(col_names), + */ + { /* enter inner scope */ + PyObject *__pyx_8genexpr1__pyx_v_x = NULL; + __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 474, __pyx_L30_error) + __Pyx_GOTREF(__pyx_t_4); + if (likely(PyList_CheckExact(__pyx_v_columns)) || PyTuple_CheckExact(__pyx_v_columns)) { + __pyx_t_11 = __pyx_v_columns; __Pyx_INCREF(__pyx_t_11); __pyx_t_10 = 0; + __pyx_t_27 = NULL; + } else { + __pyx_t_10 = -1; __pyx_t_11 = PyObject_GetIter(__pyx_v_columns); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 474, __pyx_L30_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_27 = Py_TYPE(__pyx_t_11)->tp_iternext; if (unlikely(!__pyx_t_27)) __PYX_ERR(0, 474, __pyx_L30_error) + } + for (;;) { + if (likely(!__pyx_t_27)) { + if (likely(PyList_CheckExact(__pyx_t_11))) { + if (__pyx_t_10 >= PyList_GET_SIZE(__pyx_t_11)) break; + #if CYTHON_COMPILING_IN_CPYTHON + __pyx_t_1 = PyList_GET_ITEM(__pyx_t_11, __pyx_t_10); __Pyx_INCREF(__pyx_t_1); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 474, __pyx_L30_error) + #else + __pyx_t_1 = PySequence_ITEM(__pyx_t_11, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 474, __pyx_L30_error) + __Pyx_GOTREF(__pyx_t_1); + #endif + } else { + if (__pyx_t_10 >= PyTuple_GET_SIZE(__pyx_t_11)) break; + #if CYTHON_COMPILING_IN_CPYTHON + __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_11, __pyx_t_10); __Pyx_INCREF(__pyx_t_1); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 474, __pyx_L30_error) + #else + __pyx_t_1 = PySequence_ITEM(__pyx_t_11, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 474, __pyx_L30_error) + __Pyx_GOTREF(__pyx_t_1); + #endif + } + } else { + __pyx_t_1 = __pyx_t_27(__pyx_t_11); + if (unlikely(!__pyx_t_1)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear(); + else __PYX_ERR(0, 474, __pyx_L30_error) + } + break; + } + __Pyx_GOTREF(__pyx_t_1); + } + __Pyx_XDECREF_SET(__pyx_8genexpr1__pyx_v_x, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_GetItemInt(__pyx_8genexpr1__pyx_v_x, 0, long, 1, __Pyx_PyInt_From_long, 0, 0, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 474, __pyx_L30_error) + __Pyx_GOTREF(__pyx_t_1); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_4, (PyObject*)__pyx_t_1))) __PYX_ERR(0, 474, __pyx_L30_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_XDECREF(__pyx_8genexpr1__pyx_v_x); + goto __pyx_L33_exit_scope; + __pyx_L30_error:; + __Pyx_XDECREF(__pyx_8genexpr1__pyx_v_x); + goto __pyx_L8_error; + __pyx_L33_exit_scope:; + } /* exit inner scope */ + __pyx_v_col_names = ((PyObject*)__pyx_t_4); + __pyx_t_4 = 0; + + /* "s3ql/deltadump.pyx":476 + * col_names = [ x[0] for x in columns ] + * query = ("INSERT INTO %s (%s) VALUES(%s)" + * % (table, ', '.join(col_names), # <<<<<<<<<<<<<< + * ', '.join('?' * col_count))).encode('utf-8') + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + */ + __pyx_t_4 = PyUnicode_Join(__pyx_kp_u__4, __pyx_v_col_names); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 476, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + + /* "s3ql/deltadump.pyx":477 + * query = ("INSERT INTO %s (%s) VALUES(%s)" + * % (table, ', '.join(col_names), + * ', '.join('?' * col_count))).encode('utf-8') # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + * SQLITE_OK, sqlite3_db) + */ + __pyx_t_11 = __Pyx_PyInt_From_int(__pyx_v_col_count); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 477, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_1 = PyNumber_Multiply(__pyx_kp_u__9, __pyx_t_11); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 477, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_t_11 = PyUnicode_Join(__pyx_kp_u__4, __pyx_t_1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 477, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":476 + * col_names = [ x[0] for x in columns ] + * query = ("INSERT INTO %s (%s) VALUES(%s)" + * % (table, ', '.join(col_names), # <<<<<<<<<<<<<< + * ', '.join('?' * col_count))).encode('utf-8') + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + */ + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 476, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_v_table); + __Pyx_GIVEREF(__pyx_v_table); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_table); + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_11); + PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_t_11); + __pyx_t_4 = 0; + __pyx_t_11 = 0; + __pyx_t_11 = PyUnicode_Format(__pyx_kp_u_INSERT_INTO_s_s_VALUES_s, __pyx_t_1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 476, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":477 + * query = ("INSERT INTO %s (%s) VALUES(%s)" + * % (table, ', '.join(col_names), + * ', '.join('?' * col_count))).encode('utf-8') # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + * SQLITE_OK, sqlite3_db) + */ + __pyx_t_1 = PyUnicode_AsUTF8String(((PyObject*)__pyx_t_11)); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 477, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_v_query = __pyx_t_1; + __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":478 + * % (table, ', '.join(col_names), + * ', '.join('?' * col_count))).encode('utf-8') + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(stmt), + */ + __pyx_t_15 = __Pyx_PyObject_AsString(__pyx_v_query); if (unlikely((!__pyx_t_15) && PyErr_Occurred())) __PYX_ERR(0, 478, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":479 + * ', '.join('?' * col_count))).encode('utf-8') + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + * SQLITE_OK, sqlite3_db) # <<<<<<<<<<<<<< + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(stmt), + * SQLITE_OK, sqlite3_db)) + */ + __pyx_t_16 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_prepare_v2(__pyx_cur_scope->__pyx_v_sqlite3_db, __pyx_t_15, -1, (&__pyx_cur_scope->__pyx_v_stmt), NULL), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 478, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":480 + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(stmt), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * + */ + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 480, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_4 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_5lambda11, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 480, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + } + } + if (!__pyx_t_6) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 480, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else { + __pyx_t_14 = PyTuple_New(1+1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 480, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_6); __pyx_t_6 = NULL; + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_14, 0+1, __pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_14, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 480, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":484 + * + * # Prepare BEGIN statement + * query = b'BEGIN TRANSACTION' # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &begin_stmt, NULL), + * SQLITE_OK, sqlite3_db) + */ + __Pyx_INCREF(__pyx_kp_b_BEGIN_TRANSACTION); + __Pyx_DECREF_SET(__pyx_v_query, __pyx_kp_b_BEGIN_TRANSACTION); + + /* "s3ql/deltadump.pyx":485 + * # Prepare BEGIN statement + * query = b'BEGIN TRANSACTION' + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &begin_stmt, NULL), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(begin_stmt), + */ + __pyx_t_15 = __Pyx_PyObject_AsString(__pyx_v_query); if (unlikely((!__pyx_t_15) && PyErr_Occurred())) __PYX_ERR(0, 485, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":486 + * query = b'BEGIN TRANSACTION' + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &begin_stmt, NULL), + * SQLITE_OK, sqlite3_db) # <<<<<<<<<<<<<< + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(begin_stmt), + * SQLITE_OK, sqlite3_db)) + */ + __pyx_t_16 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_prepare_v2(__pyx_cur_scope->__pyx_v_sqlite3_db, __pyx_t_15, -1, (&__pyx_cur_scope->__pyx_v_begin_stmt), NULL), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 485, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":487 + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &begin_stmt, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(begin_stmt), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * + */ + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 487, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_14 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_6lambda12, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 487, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_4 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_4)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + } + } + if (!__pyx_t_4) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_t_14); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 487, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 487, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; + __Pyx_GIVEREF(__pyx_t_14); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_14); + __pyx_t_14 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 487, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":491 + * + * # Prepare COMMIT statement + * query = b'COMMIT TRANSACTION' # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &commit_stmt, NULL), + * SQLITE_OK, sqlite3_db) + */ + __Pyx_INCREF(__pyx_kp_b_COMMIT_TRANSACTION); + __Pyx_DECREF_SET(__pyx_v_query, __pyx_kp_b_COMMIT_TRANSACTION); + + /* "s3ql/deltadump.pyx":492 + * # Prepare COMMIT statement + * query = b'COMMIT TRANSACTION' + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &commit_stmt, NULL), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(commit_stmt), + */ + __pyx_t_15 = __Pyx_PyObject_AsString(__pyx_v_query); if (unlikely((!__pyx_t_15) && PyErr_Occurred())) __PYX_ERR(0, 492, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":493 + * query = b'COMMIT TRANSACTION' + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &commit_stmt, NULL), + * SQLITE_OK, sqlite3_db) # <<<<<<<<<<<<<< + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(commit_stmt), + * SQLITE_OK, sqlite3_db)) + */ + __pyx_t_16 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_prepare_v2(__pyx_cur_scope->__pyx_v_sqlite3_db, __pyx_t_15, -1, (&__pyx_cur_scope->__pyx_v_commit_stmt), NULL), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 492, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":494 + * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &commit_stmt, NULL), + * SQLITE_OK, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(commit_stmt), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db)) + * + */ + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 494, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_6 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_7lambda13, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 494, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_14 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_14 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_14)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_14); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + } + } + if (!__pyx_t_14) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 494, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else { + __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 494, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_14); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_14); __pyx_t_14 = NULL; + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 494, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":497 + * SQLITE_OK, sqlite3_db)) + * + * buf = calloc(MAX_BLOB_SIZE, 1) # <<<<<<<<<<<<<< + * cm.callback(lambda: free(buf)) + * read_integer(&row_count, fp) + */ + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_MAX_BLOB_SIZE); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 497, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_28 = __Pyx_PyInt_As_size_t(__pyx_t_1); if (unlikely((__pyx_t_28 == (size_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 497, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_26 = __pyx_f_4s3ql_9deltadump_calloc(__pyx_t_28, 1); if (unlikely(__pyx_t_26 == NULL)) __PYX_ERR(0, 497, __pyx_L8_error) + __pyx_cur_scope->__pyx_v_buf = __pyx_t_26; + + /* "s3ql/deltadump.pyx":498 + * + * buf = calloc(MAX_BLOB_SIZE, 1) + * cm.callback(lambda: free(buf)) # <<<<<<<<<<<<<< + * read_integer(&row_count, fp) + * log.debug('load_table(%s): reading %d rows', table, row_count) + */ + __pyx_t_11 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 498, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_4 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_8lambda14, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 498, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_11); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_11, function); + } + } + if (!__pyx_t_6) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_11, __pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 498, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else { + __pyx_t_14 = PyTuple_New(1+1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 498, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_6); __pyx_t_6 = NULL; + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_14, 0+1, __pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_14, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 498, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + } + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":499 + * buf = calloc(MAX_BLOB_SIZE, 1) + * cm.callback(lambda: free(buf)) + * read_integer(&row_count, fp) # <<<<<<<<<<<<<< + * log.debug('load_table(%s): reading %d rows', table, row_count) + * + */ + __pyx_t_16 = __pyx_f_4s3ql_9deltadump_read_integer((&__pyx_v_row_count), __pyx_cur_scope->__pyx_v_fp); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 499, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":500 + * cm.callback(lambda: free(buf)) + * read_integer(&row_count, fp) + * log.debug('load_table(%s): reading %d rows', table, row_count) # <<<<<<<<<<<<<< + * + * # Start transaction + */ + __pyx_t_11 = __Pyx_GetModuleGlobalName(__pyx_n_s_log); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 500, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_14 = __Pyx_PyObject_GetAttrStr(__pyx_t_11, __pyx_n_s_debug); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 500, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + __pyx_t_11 = __Pyx_PyInt_From_int64_t(__pyx_v_row_count); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 500, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_4 = NULL; + __pyx_t_10 = 0; + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_14))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_14); + if (likely(__pyx_t_4)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_14); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_14, function); + __pyx_t_10 = 1; + } + } + __pyx_t_6 = PyTuple_New(3+__pyx_t_10); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 500, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + if (__pyx_t_4) { + __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; + } + __Pyx_INCREF(__pyx_kp_u_load_table_s_reading_d_rows); + __Pyx_GIVEREF(__pyx_kp_u_load_table_s_reading_d_rows); + PyTuple_SET_ITEM(__pyx_t_6, 0+__pyx_t_10, __pyx_kp_u_load_table_s_reading_d_rows); + __Pyx_INCREF(__pyx_v_table); + __Pyx_GIVEREF(__pyx_v_table); + PyTuple_SET_ITEM(__pyx_t_6, 1+__pyx_t_10, __pyx_v_table); + __Pyx_GIVEREF(__pyx_t_11); + PyTuple_SET_ITEM(__pyx_t_6, 2+__pyx_t_10, __pyx_t_11); + __pyx_t_11 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_14, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 500, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":503 + * + * # Start transaction + * SQLITE_CHECK_RC(sqlite3_step(begin_stmt), SQLITE_DONE, sqlite3_db) # <<<<<<<<<<<<<< + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_step(commit_stmt), + * SQLITE_DONE, sqlite3_db)) + */ + __pyx_t_16 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_step(__pyx_cur_scope->__pyx_v_begin_stmt), SQLITE_DONE, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 503, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":504 + * # Start transaction + * SQLITE_CHECK_RC(sqlite3_step(begin_stmt), SQLITE_DONE, sqlite3_db) + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_step(commit_stmt), # <<<<<<<<<<<<<< + * SQLITE_DONE, sqlite3_db)) + * SQLITE_CHECK_RC(sqlite3_reset(begin_stmt), SQLITE_OK, sqlite3_db) + */ + __pyx_t_14 = __Pyx_PyObject_GetAttrStr(__pyx_v_cm, __pyx_n_s_callback); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 504, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_6 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_9lambda15, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 504, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_11 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_14))) { + __pyx_t_11 = PyMethod_GET_SELF(__pyx_t_14); + if (likely(__pyx_t_11)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_14); + __Pyx_INCREF(__pyx_t_11); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_14, function); + } + } + if (!__pyx_t_11) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_14, __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 504, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else { + __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 504, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_11); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_11); __pyx_t_11 = NULL; + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_14, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 504, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":506 + * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_step(commit_stmt), + * SQLITE_DONE, sqlite3_db)) + * SQLITE_CHECK_RC(sqlite3_reset(begin_stmt), SQLITE_OK, sqlite3_db) # <<<<<<<<<<<<<< + * + * # Iterate through rows + */ + __pyx_t_16 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_reset(__pyx_cur_scope->__pyx_v_begin_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_16 == -1)) __PYX_ERR(0, 506, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":509 + * + * # Iterate through rows + * for i in range(row_count): # <<<<<<<<<<<<<< + * for j in range(col_count): + * if col_types[j] == _INTEGER: + */ + __pyx_t_29 = __pyx_v_row_count; + for (__pyx_t_16 = 0; __pyx_t_16 < __pyx_t_29; __pyx_t_16+=1) { + __pyx_v_i = __pyx_t_16; + + /* "s3ql/deltadump.pyx":510 + * # Iterate through rows + * for i in range(row_count): + * for j in range(col_count): # <<<<<<<<<<<<<< + * if col_types[j] == _INTEGER: + * read_integer(&int64, fp) + */ + __pyx_t_13 = __pyx_v_col_count; + for (__pyx_t_24 = 0; __pyx_t_24 < __pyx_t_13; __pyx_t_24+=1) { + __pyx_v_j = __pyx_t_24; + + /* "s3ql/deltadump.pyx":511 + * for i in range(row_count): + * for j in range(col_count): + * if col_types[j] == _INTEGER: # <<<<<<<<<<<<<< + * read_integer(&int64, fp) + * int64 += col_args[j] + int64_prev[j] + */ + __pyx_t_2 = (((__pyx_cur_scope->__pyx_v_col_types[__pyx_v_j]) == __pyx_v_4s3ql_9deltadump__INTEGER) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":512 + * for j in range(col_count): + * if col_types[j] == _INTEGER: + * read_integer(&int64, fp) # <<<<<<<<<<<<<< + * int64 += col_args[j] + int64_prev[j] + * int64_prev[j] = int64 + */ + __pyx_t_30 = __pyx_f_4s3ql_9deltadump_read_integer((&__pyx_v_int64), __pyx_cur_scope->__pyx_v_fp); if (unlikely(__pyx_t_30 == -1)) __PYX_ERR(0, 512, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":513 + * if col_types[j] == _INTEGER: + * read_integer(&int64, fp) + * int64 += col_args[j] + int64_prev[j] # <<<<<<<<<<<<<< + * int64_prev[j] = int64 + * SQLITE_CHECK_RC(sqlite3_bind_int64(stmt, j + 1, int64), + */ + __pyx_v_int64 = (__pyx_v_int64 + ((__pyx_cur_scope->__pyx_v_col_args[__pyx_v_j]) + (__pyx_cur_scope->__pyx_v_int64_prev[__pyx_v_j]))); + + /* "s3ql/deltadump.pyx":514 + * read_integer(&int64, fp) + * int64 += col_args[j] + int64_prev[j] + * int64_prev[j] = int64 # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_bind_int64(stmt, j + 1, int64), + * SQLITE_OK, sqlite3_db) + */ + (__pyx_cur_scope->__pyx_v_int64_prev[__pyx_v_j]) = __pyx_v_int64; + + /* "s3ql/deltadump.pyx":515 + * int64 += col_args[j] + int64_prev[j] + * int64_prev[j] = int64 + * SQLITE_CHECK_RC(sqlite3_bind_int64(stmt, j + 1, int64), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db) + * + */ + __pyx_t_30 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_bind_int64(__pyx_cur_scope->__pyx_v_stmt, (__pyx_v_j + 1), __pyx_v_int64), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_30 == -1)) __PYX_ERR(0, 515, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":511 + * for i in range(row_count): + * for j in range(col_count): + * if col_types[j] == _INTEGER: # <<<<<<<<<<<<<< + * read_integer(&int64, fp) + * int64 += col_args[j] + int64_prev[j] + */ + } + + /* "s3ql/deltadump.pyx":518 + * SQLITE_OK, sqlite3_db) + * + * if col_types[j] == _TIME: # <<<<<<<<<<<<<< + * read_integer(&int64, fp) + * int64 += col_args[j] + int64_prev[j] + */ + __pyx_t_2 = (((__pyx_cur_scope->__pyx_v_col_types[__pyx_v_j]) == __pyx_v_4s3ql_9deltadump__TIME) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":519 + * + * if col_types[j] == _TIME: + * read_integer(&int64, fp) # <<<<<<<<<<<<<< + * int64 += col_args[j] + int64_prev[j] + * int64_prev[j] = int64 + */ + __pyx_t_30 = __pyx_f_4s3ql_9deltadump_read_integer((&__pyx_v_int64), __pyx_cur_scope->__pyx_v_fp); if (unlikely(__pyx_t_30 == -1)) __PYX_ERR(0, 519, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":520 + * if col_types[j] == _TIME: + * read_integer(&int64, fp) + * int64 += col_args[j] + int64_prev[j] # <<<<<<<<<<<<<< + * int64_prev[j] = int64 + * # Cast is safe, we know that the integer was converted from + */ + __pyx_v_int64 = (__pyx_v_int64 + ((__pyx_cur_scope->__pyx_v_col_args[__pyx_v_j]) + (__pyx_cur_scope->__pyx_v_int64_prev[__pyx_v_j]))); + + /* "s3ql/deltadump.pyx":521 + * read_integer(&int64, fp) + * int64 += col_args[j] + int64_prev[j] + * int64_prev[j] = int64 # <<<<<<<<<<<<<< + * # Cast is safe, we know that the integer was converted from + * # double at dump time. + */ + (__pyx_cur_scope->__pyx_v_int64_prev[__pyx_v_j]) = __pyx_v_int64; + + /* "s3ql/deltadump.pyx":524 + * # Cast is safe, we know that the integer was converted from + * # double at dump time. + * SQLITE_CHECK_RC(sqlite3_bind_double(stmt, j + 1, int64 / time_scale), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db) + * + */ + if (unlikely(__pyx_v_4s3ql_9deltadump_time_scale == 0)) { + PyErr_SetString(PyExc_ZeroDivisionError, "float division"); + __PYX_ERR(0, 524, __pyx_L8_error) + } + + /* "s3ql/deltadump.pyx":525 + * # double at dump time. + * SQLITE_CHECK_RC(sqlite3_bind_double(stmt, j + 1, int64 / time_scale), + * SQLITE_OK, sqlite3_db) # <<<<<<<<<<<<<< + * + * elif col_types[j] == _BLOB: + */ + __pyx_t_30 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_bind_double(__pyx_cur_scope->__pyx_v_stmt, (__pyx_v_j + 1), (((double)__pyx_v_int64) / __pyx_v_4s3ql_9deltadump_time_scale)), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_30 == -1)) __PYX_ERR(0, 524, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":518 + * SQLITE_OK, sqlite3_db) + * + * if col_types[j] == _TIME: # <<<<<<<<<<<<<< + * read_integer(&int64, fp) + * int64 += col_args[j] + int64_prev[j] + */ + goto __pyx_L39; + } + + /* "s3ql/deltadump.pyx":527 + * SQLITE_OK, sqlite3_db) + * + * elif col_types[j] == _BLOB: # <<<<<<<<<<<<<< + * if col_args[j] == 0: + * read_integer(&int64, fp) + */ + __pyx_t_2 = (((__pyx_cur_scope->__pyx_v_col_types[__pyx_v_j]) == __pyx_v_4s3ql_9deltadump__BLOB) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":528 + * + * elif col_types[j] == _BLOB: + * if col_args[j] == 0: # <<<<<<<<<<<<<< + * read_integer(&int64, fp) + * tmp = int64_prev[j] + int64 + */ + __pyx_t_2 = (((__pyx_cur_scope->__pyx_v_col_args[__pyx_v_j]) == 0) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":529 + * elif col_types[j] == _BLOB: + * if col_args[j] == 0: + * read_integer(&int64, fp) # <<<<<<<<<<<<<< + * tmp = int64_prev[j] + int64 + * if tmp < 0 or tmp > INT_MAX: + */ + __pyx_t_30 = __pyx_f_4s3ql_9deltadump_read_integer((&__pyx_v_int64), __pyx_cur_scope->__pyx_v_fp); if (unlikely(__pyx_t_30 == -1)) __PYX_ERR(0, 529, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":530 + * if col_args[j] == 0: + * read_integer(&int64, fp) + * tmp = int64_prev[j] + int64 # <<<<<<<<<<<<<< + * if tmp < 0 or tmp > INT_MAX: + * raise RuntimeError('Corrupted input') + */ + __pyx_v_tmp = ((__pyx_cur_scope->__pyx_v_int64_prev[__pyx_v_j]) + __pyx_v_int64); + + /* "s3ql/deltadump.pyx":531 + * read_integer(&int64, fp) + * tmp = int64_prev[j] + int64 + * if tmp < 0 or tmp > INT_MAX: # <<<<<<<<<<<<<< + * raise RuntimeError('Corrupted input') + * len_ = tmp + */ + __pyx_t_31 = ((__pyx_v_tmp < 0) != 0); + if (!__pyx_t_31) { + } else { + __pyx_t_2 = __pyx_t_31; + goto __pyx_L42_bool_binop_done; + } + __pyx_t_31 = ((__pyx_v_tmp > INT_MAX) != 0); + __pyx_t_2 = __pyx_t_31; + __pyx_L42_bool_binop_done:; + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":532 + * tmp = int64_prev[j] + int64 + * if tmp < 0 or tmp > INT_MAX: + * raise RuntimeError('Corrupted input') # <<<<<<<<<<<<<< + * len_ = tmp + * int64_prev[j] = tmp + */ + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_tuple__10, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 532, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(0, 532, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":531 + * read_integer(&int64, fp) + * tmp = int64_prev[j] + int64 + * if tmp < 0 or tmp > INT_MAX: # <<<<<<<<<<<<<< + * raise RuntimeError('Corrupted input') + * len_ = tmp + */ + } + + /* "s3ql/deltadump.pyx":533 + * if tmp < 0 or tmp > INT_MAX: + * raise RuntimeError('Corrupted input') + * len_ = tmp # <<<<<<<<<<<<<< + * int64_prev[j] = tmp + * else: + */ + __pyx_v_len_ = ((int)__pyx_v_tmp); + + /* "s3ql/deltadump.pyx":534 + * raise RuntimeError('Corrupted input') + * len_ = tmp + * int64_prev[j] = tmp # <<<<<<<<<<<<<< + * else: + * len_ = col_args[j] + */ + (__pyx_cur_scope->__pyx_v_int64_prev[__pyx_v_j]) = __pyx_v_tmp; + + /* "s3ql/deltadump.pyx":528 + * + * elif col_types[j] == _BLOB: + * if col_args[j] == 0: # <<<<<<<<<<<<<< + * read_integer(&int64, fp) + * tmp = int64_prev[j] + int64 + */ + goto __pyx_L40; + } + + /* "s3ql/deltadump.pyx":536 + * int64_prev[j] = tmp + * else: + * len_ = col_args[j] # <<<<<<<<<<<<<< + * + * if len_ > MAX_BLOB_SIZE: + */ + /*else*/ { + __pyx_v_len_ = (__pyx_cur_scope->__pyx_v_col_args[__pyx_v_j]); + } + __pyx_L40:; + + /* "s3ql/deltadump.pyx":538 + * len_ = col_args[j] + * + * if len_ > MAX_BLOB_SIZE: # <<<<<<<<<<<<<< + * raise RuntimeError('BLOB too large to read (%d vs %d)', len_, MAX_BLOB_SIZE) + * + */ + __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_len_); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 538, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_14 = __Pyx_GetModuleGlobalName(__pyx_n_s_MAX_BLOB_SIZE); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 538, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_4 = PyObject_RichCompare(__pyx_t_1, __pyx_t_14, Py_GT); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 538, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 538, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":539 + * + * if len_ > MAX_BLOB_SIZE: + * raise RuntimeError('BLOB too large to read (%d vs %d)', len_, MAX_BLOB_SIZE) # <<<<<<<<<<<<<< + * + * if len_ > 0: + */ + __pyx_t_4 = __Pyx_PyInt_From_int(__pyx_v_len_); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 539, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_14 = __Pyx_GetModuleGlobalName(__pyx_n_s_MAX_BLOB_SIZE); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 539, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_1 = PyTuple_New(3); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 539, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_kp_u_BLOB_too_large_to_read_d_vs_d); + __Pyx_GIVEREF(__pyx_kp_u_BLOB_too_large_to_read_d_vs_d); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_kp_u_BLOB_too_large_to_read_d_vs_d); + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_14); + PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_t_14); + __pyx_t_4 = 0; + __pyx_t_14 = 0; + __pyx_t_14 = __Pyx_PyObject_Call(__pyx_builtin_RuntimeError, __pyx_t_1, NULL); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 539, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_Raise(__pyx_t_14, 0, 0, 0); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __PYX_ERR(0, 539, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":538 + * len_ = col_args[j] + * + * if len_ > MAX_BLOB_SIZE: # <<<<<<<<<<<<<< + * raise RuntimeError('BLOB too large to read (%d vs %d)', len_, MAX_BLOB_SIZE) + * + */ + } + + /* "s3ql/deltadump.pyx":541 + * raise RuntimeError('BLOB too large to read (%d vs %d)', len_, MAX_BLOB_SIZE) + * + * if len_ > 0: # <<<<<<<<<<<<<< + * fread(buf, len_, fp) + * + */ + __pyx_t_2 = ((__pyx_v_len_ > 0) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":542 + * + * if len_ > 0: + * fread(buf, len_, fp) # <<<<<<<<<<<<<< + * + * SQLITE_CHECK_RC(sqlite3_bind_blob(stmt, j + 1, buf, len_, SQLITE_TRANSIENT), + */ + __pyx_t_30 = __pyx_f_4s3ql_9deltadump_fread(__pyx_cur_scope->__pyx_v_buf, ((unsigned int)__pyx_v_len_), __pyx_cur_scope->__pyx_v_fp); if (unlikely(__pyx_t_30 == -1)) __PYX_ERR(0, 542, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":541 + * raise RuntimeError('BLOB too large to read (%d vs %d)', len_, MAX_BLOB_SIZE) + * + * if len_ > 0: # <<<<<<<<<<<<<< + * fread(buf, len_, fp) + * + */ + } + + /* "s3ql/deltadump.pyx":544 + * fread(buf, len_, fp) + * + * SQLITE_CHECK_RC(sqlite3_bind_blob(stmt, j + 1, buf, len_, SQLITE_TRANSIENT), # <<<<<<<<<<<<<< + * SQLITE_OK, sqlite3_db) + * + */ + __pyx_t_30 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_bind_blob(__pyx_cur_scope->__pyx_v_stmt, (__pyx_v_j + 1), __pyx_cur_scope->__pyx_v_buf, __pyx_v_len_, SQLITE_TRANSIENT), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_30 == -1)) __PYX_ERR(0, 544, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":527 + * SQLITE_OK, sqlite3_db) + * + * elif col_types[j] == _BLOB: # <<<<<<<<<<<<<< + * if col_args[j] == 0: + * read_integer(&int64, fp) + */ + } + __pyx_L39:; + } + + /* "s3ql/deltadump.pyx":547 + * SQLITE_OK, sqlite3_db) + * + * SQLITE_CHECK_RC(sqlite3_step(stmt), SQLITE_DONE, sqlite3_db) # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_reset(stmt), SQLITE_OK, sqlite3_db) + * + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_step(__pyx_cur_scope->__pyx_v_stmt), SQLITE_DONE, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 547, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":548 + * + * SQLITE_CHECK_RC(sqlite3_step(stmt), SQLITE_DONE, sqlite3_db) + * SQLITE_CHECK_RC(sqlite3_reset(stmt), SQLITE_OK, sqlite3_db) # <<<<<<<<<<<<<< + * + * # Commit every once in a while + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_reset(__pyx_cur_scope->__pyx_v_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 548, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":551 + * + * # Commit every once in a while + * if i % trx_rows == 0: # <<<<<<<<<<<<<< + * # This isn't 100% ok -- if we have an exception in step(begin_stmt), + * # we the cleanup handler will execute the commit statement again + */ + __pyx_t_14 = __Pyx_PyInt_From_int(__pyx_v_i); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 551, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __pyx_t_1 = PyNumber_Remainder(__pyx_t_14, __pyx_v_trx_rows); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 551, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __pyx_t_14 = __Pyx_PyInt_EqObjC(__pyx_t_1, __pyx_int_0, 0, 0); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 551, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_14); if (unlikely(__pyx_t_2 < 0)) __PYX_ERR(0, 551, __pyx_L8_error) + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":555 + * # we the cleanup handler will execute the commit statement again + * # without an active transaction. + * SQLITE_CHECK_RC(sqlite3_step(commit_stmt), SQLITE_DONE, sqlite3_db) # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_step(begin_stmt), SQLITE_DONE, sqlite3_db) + * SQLITE_CHECK_RC(sqlite3_reset(commit_stmt), SQLITE_OK, sqlite3_db) + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_step(__pyx_cur_scope->__pyx_v_commit_stmt), SQLITE_DONE, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 555, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":556 + * # without an active transaction. + * SQLITE_CHECK_RC(sqlite3_step(commit_stmt), SQLITE_DONE, sqlite3_db) + * SQLITE_CHECK_RC(sqlite3_step(begin_stmt), SQLITE_DONE, sqlite3_db) # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_reset(commit_stmt), SQLITE_OK, sqlite3_db) + * SQLITE_CHECK_RC(sqlite3_reset(begin_stmt), SQLITE_OK, sqlite3_db) + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_step(__pyx_cur_scope->__pyx_v_begin_stmt), SQLITE_DONE, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 556, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":557 + * SQLITE_CHECK_RC(sqlite3_step(commit_stmt), SQLITE_DONE, sqlite3_db) + * SQLITE_CHECK_RC(sqlite3_step(begin_stmt), SQLITE_DONE, sqlite3_db) + * SQLITE_CHECK_RC(sqlite3_reset(commit_stmt), SQLITE_OK, sqlite3_db) # <<<<<<<<<<<<<< + * SQLITE_CHECK_RC(sqlite3_reset(begin_stmt), SQLITE_OK, sqlite3_db) + * + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_reset(__pyx_cur_scope->__pyx_v_commit_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 557, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":558 + * SQLITE_CHECK_RC(sqlite3_step(begin_stmt), SQLITE_DONE, sqlite3_db) + * SQLITE_CHECK_RC(sqlite3_reset(commit_stmt), SQLITE_OK, sqlite3_db) + * SQLITE_CHECK_RC(sqlite3_reset(begin_stmt), SQLITE_OK, sqlite3_db) # <<<<<<<<<<<<<< + * + * cdef inline int write_integer(int64_t int64, FILE * fp) except -1: + */ + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_reset(__pyx_cur_scope->__pyx_v_begin_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 558, __pyx_L8_error) + + /* "s3ql/deltadump.pyx":551 + * + * # Commit every once in a while + * if i % trx_rows == 0: # <<<<<<<<<<<<<< + * # This isn't 100% ok -- if we have an exception in step(begin_stmt), + * # we the cleanup handler will execute the commit statement again + */ + } + } + + /* "s3ql/deltadump.pyx":434 + * raise ValueError("Can't access in-memory databases") + * + * with ExitStack() as cm: # <<<<<<<<<<<<<< + * # Get SQLite connection + * log.debug('Opening connection to %s', db.file) + */ + } + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + goto __pyx_L15_try_end; + __pyx_L8_error:; + __Pyx_PyThreadState_assign + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_14); __pyx_t_14 = 0; + /*except:*/ { + __Pyx_AddTraceback("s3ql.deltadump.load_table", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_14, &__pyx_t_1, &__pyx_t_4) < 0) __PYX_ERR(0, 434, __pyx_L10_except_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = PyTuple_Pack(3, __pyx_t_14, __pyx_t_1, __pyx_t_4); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 434, __pyx_L10_except_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_23 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_6, NULL); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_23)) __PYX_ERR(0, 434, __pyx_L10_except_error) + __Pyx_GOTREF(__pyx_t_23); + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_t_23); + __Pyx_DECREF(__pyx_t_23); __pyx_t_23 = 0; + if (__pyx_t_2 < 0) __PYX_ERR(0, 434, __pyx_L10_except_error) + __pyx_t_31 = ((!(__pyx_t_2 != 0)) != 0); + if (__pyx_t_31) { + __Pyx_GIVEREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_ErrRestoreWithState(__pyx_t_14, __pyx_t_1, __pyx_t_4); + __pyx_t_14 = 0; __pyx_t_1 = 0; __pyx_t_4 = 0; + __PYX_ERR(0, 434, __pyx_L10_except_error) + } + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + goto __pyx_L9_exception_handled; + } + __pyx_L10_except_error:; + __Pyx_PyThreadState_assign + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_XGIVEREF(__pyx_t_9); + __Pyx_ExceptionReset(__pyx_t_7, __pyx_t_8, __pyx_t_9); + goto __pyx_L1_error; + __pyx_L9_exception_handled:; + __Pyx_PyThreadState_assign + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_XGIVEREF(__pyx_t_9); + __Pyx_ExceptionReset(__pyx_t_7, __pyx_t_8, __pyx_t_9); + __pyx_L15_try_end:; + } + } + /*finally:*/ { + /*normal exit:*/{ + if (__pyx_t_5) { + __pyx_t_9 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_tuple__11, NULL); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 434, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + } + goto __pyx_L7; + } + __pyx_L7:; + } + goto __pyx_L50; + __pyx_L4_error:; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + goto __pyx_L1_error; + __pyx_L50:; + } + + /* "s3ql/deltadump.pyx":401 + * fwrite(buf, len_, fp) + * + * def load_table(table, columns, db, fh, trx_rows=5000): # <<<<<<<<<<<<<< + * '''Load *columns* of *table* from *fh* + * + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_XDECREF(__pyx_t_14); + __Pyx_AddTraceback("s3ql.deltadump.load_table", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_cm); + __Pyx_XDECREF(__pyx_v_dbfile_b); + __Pyx_XDECREF(__pyx_v_pragma); + __Pyx_XDECREF(__pyx_v_val); + __Pyx_XDECREF(__pyx_v_cmd); + __Pyx_XDECREF(__pyx_v_col_names); + __Pyx_XDECREF(__pyx_v_query); + __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":560 + * SQLITE_CHECK_RC(sqlite3_reset(begin_stmt), SQLITE_OK, sqlite3_db) + * + * cdef inline int write_integer(int64_t int64, FILE * fp) except -1: # <<<<<<<<<<<<<< + * '''Write *int64* into *fp*, using as little space as possible + * + */ + +static CYTHON_INLINE int __pyx_f_4s3ql_9deltadump_write_integer(int64_t __pyx_v_int64, FILE *__pyx_v_fp) { + unsigned int __pyx_v_int8; + uint8_t __pyx_v_int8_real; + size_t __pyx_v_len_; + uint64_t __pyx_v_uint64; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + uint64_t __pyx_t_3; + int __pyx_t_4; + int __pyx_t_5; + __Pyx_RefNannySetupContext("write_integer", 0); + + /* "s3ql/deltadump.pyx":576 + * cdef uint64_t uint64 + * + * if int64 < 0: # <<<<<<<<<<<<<< + * uint64 = -int64 + * int8 = 0x80 # Highest bit set + */ + __pyx_t_1 = ((__pyx_v_int64 < 0) != 0); + if (__pyx_t_1) { + + /* "s3ql/deltadump.pyx":577 + * + * if int64 < 0: + * uint64 = -int64 # <<<<<<<<<<<<<< + * int8 = 0x80 # Highest bit set + * else: + */ + __pyx_v_uint64 = ((uint64_t)(-__pyx_v_int64)); + + /* "s3ql/deltadump.pyx":578 + * if int64 < 0: + * uint64 = -int64 + * int8 = 0x80 # Highest bit set # <<<<<<<<<<<<<< + * else: + * uint64 = int64 + */ + __pyx_v_int8 = 0x80; + + /* "s3ql/deltadump.pyx":576 + * cdef uint64_t uint64 + * + * if int64 < 0: # <<<<<<<<<<<<<< + * uint64 = -int64 + * int8 = 0x80 # Highest bit set + */ + goto __pyx_L3; + } + + /* "s3ql/deltadump.pyx":580 + * int8 = 0x80 # Highest bit set + * else: + * uint64 = int64 # <<<<<<<<<<<<<< + * int8 = 0 + * + */ + /*else*/ { + __pyx_v_uint64 = ((uint64_t)__pyx_v_int64); + + /* "s3ql/deltadump.pyx":581 + * else: + * uint64 = int64 + * int8 = 0 # <<<<<<<<<<<<<< + * + * if uint64 < 0x80 and uint64 not in (INT8, INT16, INT32, INT64): + */ + __pyx_v_int8 = 0; + } + __pyx_L3:; + + /* "s3ql/deltadump.pyx":583 + * int8 = 0 + * + * if uint64 < 0x80 and uint64 not in (INT8, INT16, INT32, INT64): # <<<<<<<<<<<<<< + * len_ = 0 + * int8 += uint64 + */ + __pyx_t_2 = ((__pyx_v_uint64 < 0x80) != 0); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L5_bool_binop_done; + } + __pyx_t_3 = __pyx_v_uint64; + __pyx_t_4 = ((__pyx_t_3 != __pyx_v_4s3ql_9deltadump_INT8) != 0); + if (__pyx_t_4) { + } else { + __pyx_t_2 = __pyx_t_4; + goto __pyx_L7_bool_binop_done; + } + __pyx_t_4 = ((__pyx_t_3 != __pyx_v_4s3ql_9deltadump_INT16) != 0); + if (__pyx_t_4) { + } else { + __pyx_t_2 = __pyx_t_4; + goto __pyx_L7_bool_binop_done; + } + __pyx_t_4 = ((__pyx_t_3 != __pyx_v_4s3ql_9deltadump_INT32) != 0); + if (__pyx_t_4) { + } else { + __pyx_t_2 = __pyx_t_4; + goto __pyx_L7_bool_binop_done; + } + __pyx_t_4 = ((__pyx_t_3 != __pyx_v_4s3ql_9deltadump_INT64) != 0); + __pyx_t_2 = __pyx_t_4; + __pyx_L7_bool_binop_done:; + __pyx_t_4 = (__pyx_t_2 != 0); + __pyx_t_1 = __pyx_t_4; + __pyx_L5_bool_binop_done:; + if (__pyx_t_1) { + + /* "s3ql/deltadump.pyx":584 + * + * if uint64 < 0x80 and uint64 not in (INT8, INT16, INT32, INT64): + * len_ = 0 # <<<<<<<<<<<<<< + * int8 += uint64 + * elif uint64 < UINT8_MAX: + */ + __pyx_v_len_ = 0; + + /* "s3ql/deltadump.pyx":585 + * if uint64 < 0x80 and uint64 not in (INT8, INT16, INT32, INT64): + * len_ = 0 + * int8 += uint64 # <<<<<<<<<<<<<< + * elif uint64 < UINT8_MAX: + * len_ = 1 + */ + __pyx_v_int8 = (__pyx_v_int8 + ((uint8_t)__pyx_v_uint64)); + + /* "s3ql/deltadump.pyx":583 + * int8 = 0 + * + * if uint64 < 0x80 and uint64 not in (INT8, INT16, INT32, INT64): # <<<<<<<<<<<<<< + * len_ = 0 + * int8 += uint64 + */ + goto __pyx_L4; + } + + /* "s3ql/deltadump.pyx":586 + * len_ = 0 + * int8 += uint64 + * elif uint64 < UINT8_MAX: # <<<<<<<<<<<<<< + * len_ = 1 + * int8 += INT8 + */ + __pyx_t_1 = ((__pyx_v_uint64 < UINT8_MAX) != 0); + if (__pyx_t_1) { + + /* "s3ql/deltadump.pyx":587 + * int8 += uint64 + * elif uint64 < UINT8_MAX: + * len_ = 1 # <<<<<<<<<<<<<< + * int8 += INT8 + * elif uint64 < UINT16_MAX: + */ + __pyx_v_len_ = 1; + + /* "s3ql/deltadump.pyx":588 + * elif uint64 < UINT8_MAX: + * len_ = 1 + * int8 += INT8 # <<<<<<<<<<<<<< + * elif uint64 < UINT16_MAX: + * len_ = 2 + */ + __pyx_v_int8 = (__pyx_v_int8 + __pyx_v_4s3ql_9deltadump_INT8); + + /* "s3ql/deltadump.pyx":586 + * len_ = 0 + * int8 += uint64 + * elif uint64 < UINT8_MAX: # <<<<<<<<<<<<<< + * len_ = 1 + * int8 += INT8 + */ + goto __pyx_L4; + } + + /* "s3ql/deltadump.pyx":589 + * len_ = 1 + * int8 += INT8 + * elif uint64 < UINT16_MAX: # <<<<<<<<<<<<<< + * len_ = 2 + * int8 += INT16 + */ + __pyx_t_1 = ((__pyx_v_uint64 < UINT16_MAX) != 0); + if (__pyx_t_1) { + + /* "s3ql/deltadump.pyx":590 + * int8 += INT8 + * elif uint64 < UINT16_MAX: + * len_ = 2 # <<<<<<<<<<<<<< + * int8 += INT16 + * elif uint64 < UINT32_MAX: + */ + __pyx_v_len_ = 2; + + /* "s3ql/deltadump.pyx":591 + * elif uint64 < UINT16_MAX: + * len_ = 2 + * int8 += INT16 # <<<<<<<<<<<<<< + * elif uint64 < UINT32_MAX: + * len_ = 4 + */ + __pyx_v_int8 = (__pyx_v_int8 + __pyx_v_4s3ql_9deltadump_INT16); + + /* "s3ql/deltadump.pyx":589 + * len_ = 1 + * int8 += INT8 + * elif uint64 < UINT16_MAX: # <<<<<<<<<<<<<< + * len_ = 2 + * int8 += INT16 + */ + goto __pyx_L4; + } + + /* "s3ql/deltadump.pyx":592 + * len_ = 2 + * int8 += INT16 + * elif uint64 < UINT32_MAX: # <<<<<<<<<<<<<< + * len_ = 4 + * int8 += INT32 + */ + __pyx_t_1 = ((__pyx_v_uint64 < UINT32_MAX) != 0); + if (__pyx_t_1) { + + /* "s3ql/deltadump.pyx":593 + * int8 += INT16 + * elif uint64 < UINT32_MAX: + * len_ = 4 # <<<<<<<<<<<<<< + * int8 += INT32 + * else: + */ + __pyx_v_len_ = 4; + + /* "s3ql/deltadump.pyx":594 + * elif uint64 < UINT32_MAX: + * len_ = 4 + * int8 += INT32 # <<<<<<<<<<<<<< + * else: + * len_ = 8 + */ + __pyx_v_int8 = (__pyx_v_int8 + __pyx_v_4s3ql_9deltadump_INT32); + + /* "s3ql/deltadump.pyx":592 + * len_ = 2 + * int8 += INT16 + * elif uint64 < UINT32_MAX: # <<<<<<<<<<<<<< + * len_ = 4 + * int8 += INT32 + */ + goto __pyx_L4; + } + + /* "s3ql/deltadump.pyx":596 + * int8 += INT32 + * else: + * len_ = 8 # <<<<<<<<<<<<<< + * int8 += INT64 + * + */ + /*else*/ { + __pyx_v_len_ = 8; + + /* "s3ql/deltadump.pyx":597 + * else: + * len_ = 8 + * int8 += INT64 # <<<<<<<<<<<<<< + * + * # Cast + */ + __pyx_v_int8 = (__pyx_v_int8 + __pyx_v_4s3ql_9deltadump_INT64); + } + __pyx_L4:; + + /* "s3ql/deltadump.pyx":600 + * + * # Cast + * int8_real = int8 # <<<<<<<<<<<<<< + * fwrite(&int8_real, 1, fp) + * if len_ != 0: + */ + __pyx_v_int8_real = ((uint8_t)__pyx_v_int8); + + /* "s3ql/deltadump.pyx":601 + * # Cast + * int8_real = int8 + * fwrite(&int8_real, 1, fp) # <<<<<<<<<<<<<< + * if len_ != 0: + * uint64 = htole64(uint64) + */ + __pyx_t_5 = __pyx_f_4s3ql_9deltadump_fwrite((&__pyx_v_int8_real), 1, __pyx_v_fp); if (unlikely(__pyx_t_5 == -1)) __PYX_ERR(0, 601, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":602 + * int8_real = int8 + * fwrite(&int8_real, 1, fp) + * if len_ != 0: # <<<<<<<<<<<<<< + * uint64 = htole64(uint64) + * fwrite(&uint64, len_, fp) + */ + __pyx_t_1 = ((__pyx_v_len_ != 0) != 0); + if (__pyx_t_1) { + + /* "s3ql/deltadump.pyx":603 + * fwrite(&int8_real, 1, fp) + * if len_ != 0: + * uint64 = htole64(uint64) # <<<<<<<<<<<<<< + * fwrite(&uint64, len_, fp) + * + */ + __pyx_v_uint64 = htole64(__pyx_v_uint64); + + /* "s3ql/deltadump.pyx":604 + * if len_ != 0: + * uint64 = htole64(uint64) + * fwrite(&uint64, len_, fp) # <<<<<<<<<<<<<< + * + * # len <= 8, safe to cast + */ + __pyx_t_5 = __pyx_f_4s3ql_9deltadump_fwrite((&__pyx_v_uint64), __pyx_v_len_, __pyx_v_fp); if (unlikely(__pyx_t_5 == -1)) __PYX_ERR(0, 604, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":602 + * int8_real = int8 + * fwrite(&int8_real, 1, fp) + * if len_ != 0: # <<<<<<<<<<<<<< + * uint64 = htole64(uint64) + * fwrite(&uint64, len_, fp) + */ + } + + /* "s3ql/deltadump.pyx":607 + * + * # len <= 8, safe to cast + * return len_ + 1 # <<<<<<<<<<<<<< + * + * cdef inline int read_integer(int64_t * out, FILE * fp) except -1: + */ + __pyx_r = (((int)__pyx_v_len_) + 1); + goto __pyx_L0; + + /* "s3ql/deltadump.pyx":560 + * SQLITE_CHECK_RC(sqlite3_reset(begin_stmt), SQLITE_OK, sqlite3_db) + * + * cdef inline int write_integer(int64_t int64, FILE * fp) except -1: # <<<<<<<<<<<<<< + * '''Write *int64* into *fp*, using as little space as possible + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("s3ql.deltadump.write_integer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "s3ql/deltadump.pyx":609 + * return len_ + 1 + * + * cdef inline int read_integer(int64_t * out, FILE * fp) except -1: # <<<<<<<<<<<<<< + * '''Read integer written using `write_integer` from *fp* + * + */ + +static CYTHON_INLINE int __pyx_f_4s3ql_9deltadump_read_integer(int64_t *__pyx_v_out, FILE *__pyx_v_fp) { + uint8_t __pyx_v_int8; + size_t __pyx_v_len_; + uint64_t __pyx_v_uint64; + char __pyx_v_negative; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + __Pyx_RefNannySetupContext("read_integer", 0); + + /* "s3ql/deltadump.pyx":620 + * cdef char negative + * + * fread(&int8, 1, fp) # <<<<<<<<<<<<<< + * + * if int8 & 0x80 != 0: + */ + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_fread((&__pyx_v_int8), 1, __pyx_v_fp); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 620, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":622 + * fread(&int8, 1, fp) + * + * if int8 & 0x80 != 0: # <<<<<<<<<<<<<< + * negative = 1 + * # Need to cast again due to integer promotion + */ + __pyx_t_2 = (((__pyx_v_int8 & 0x80) != 0) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":623 + * + * if int8 & 0x80 != 0: + * negative = 1 # <<<<<<<<<<<<<< + * # Need to cast again due to integer promotion + * int8 = (int8 & (~ 0x80)) + */ + __pyx_v_negative = 1; + + /* "s3ql/deltadump.pyx":625 + * negative = 1 + * # Need to cast again due to integer promotion + * int8 = (int8 & (~ 0x80)) # <<<<<<<<<<<<<< + * else: + * negative = 0 + */ + __pyx_v_int8 = ((uint8_t)(__pyx_v_int8 & (~0x80))); + + /* "s3ql/deltadump.pyx":622 + * fread(&int8, 1, fp) + * + * if int8 & 0x80 != 0: # <<<<<<<<<<<<<< + * negative = 1 + * # Need to cast again due to integer promotion + */ + goto __pyx_L3; + } + + /* "s3ql/deltadump.pyx":627 + * int8 = (int8 & (~ 0x80)) + * else: + * negative = 0 # <<<<<<<<<<<<<< + * + * if int8 == INT8: + */ + /*else*/ { + __pyx_v_negative = 0; + } + __pyx_L3:; + + /* "s3ql/deltadump.pyx":629 + * negative = 0 + * + * if int8 == INT8: # <<<<<<<<<<<<<< + * len_ = 1 + * elif int8 == INT16: + */ + __pyx_t_2 = ((__pyx_v_int8 == __pyx_v_4s3ql_9deltadump_INT8) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":630 + * + * if int8 == INT8: + * len_ = 1 # <<<<<<<<<<<<<< + * elif int8 == INT16: + * len_ = 2 + */ + __pyx_v_len_ = 1; + + /* "s3ql/deltadump.pyx":629 + * negative = 0 + * + * if int8 == INT8: # <<<<<<<<<<<<<< + * len_ = 1 + * elif int8 == INT16: + */ + goto __pyx_L4; + } + + /* "s3ql/deltadump.pyx":631 + * if int8 == INT8: + * len_ = 1 + * elif int8 == INT16: # <<<<<<<<<<<<<< + * len_ = 2 + * elif int8 == INT32: + */ + __pyx_t_2 = ((__pyx_v_int8 == __pyx_v_4s3ql_9deltadump_INT16) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":632 + * len_ = 1 + * elif int8 == INT16: + * len_ = 2 # <<<<<<<<<<<<<< + * elif int8 == INT32: + * len_ = 4 + */ + __pyx_v_len_ = 2; + + /* "s3ql/deltadump.pyx":631 + * if int8 == INT8: + * len_ = 1 + * elif int8 == INT16: # <<<<<<<<<<<<<< + * len_ = 2 + * elif int8 == INT32: + */ + goto __pyx_L4; + } + + /* "s3ql/deltadump.pyx":633 + * elif int8 == INT16: + * len_ = 2 + * elif int8 == INT32: # <<<<<<<<<<<<<< + * len_ = 4 + * elif int8 == INT64: + */ + __pyx_t_2 = ((__pyx_v_int8 == __pyx_v_4s3ql_9deltadump_INT32) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":634 + * len_ = 2 + * elif int8 == INT32: + * len_ = 4 # <<<<<<<<<<<<<< + * elif int8 == INT64: + * len_ = 8 + */ + __pyx_v_len_ = 4; + + /* "s3ql/deltadump.pyx":633 + * elif int8 == INT16: + * len_ = 2 + * elif int8 == INT32: # <<<<<<<<<<<<<< + * len_ = 4 + * elif int8 == INT64: + */ + goto __pyx_L4; + } + + /* "s3ql/deltadump.pyx":635 + * elif int8 == INT32: + * len_ = 4 + * elif int8 == INT64: # <<<<<<<<<<<<<< + * len_ = 8 + * else: + */ + __pyx_t_2 = ((__pyx_v_int8 == __pyx_v_4s3ql_9deltadump_INT64) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":636 + * len_ = 4 + * elif int8 == INT64: + * len_ = 8 # <<<<<<<<<<<<<< + * else: + * len_ = 0 + */ + __pyx_v_len_ = 8; + + /* "s3ql/deltadump.pyx":635 + * elif int8 == INT32: + * len_ = 4 + * elif int8 == INT64: # <<<<<<<<<<<<<< + * len_ = 8 + * else: + */ + goto __pyx_L4; + } + + /* "s3ql/deltadump.pyx":638 + * len_ = 8 + * else: + * len_ = 0 # <<<<<<<<<<<<<< + * uint64 = int8 + * + */ + /*else*/ { + __pyx_v_len_ = 0; + + /* "s3ql/deltadump.pyx":639 + * else: + * len_ = 0 + * uint64 = int8 # <<<<<<<<<<<<<< + * + * if len_ != 0: + */ + __pyx_v_uint64 = __pyx_v_int8; + } + __pyx_L4:; + + /* "s3ql/deltadump.pyx":641 + * uint64 = int8 + * + * if len_ != 0: # <<<<<<<<<<<<<< + * uint64 = 0 + * fread(&uint64, len_, fp) + */ + __pyx_t_2 = ((__pyx_v_len_ != 0) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":642 + * + * if len_ != 0: + * uint64 = 0 # <<<<<<<<<<<<<< + * fread(&uint64, len_, fp) + * uint64 = le64toh(uint64) + */ + __pyx_v_uint64 = 0; + + /* "s3ql/deltadump.pyx":643 + * if len_ != 0: + * uint64 = 0 + * fread(&uint64, len_, fp) # <<<<<<<<<<<<<< + * uint64 = le64toh(uint64) + * + */ + __pyx_t_1 = __pyx_f_4s3ql_9deltadump_fread((&__pyx_v_uint64), __pyx_v_len_, __pyx_v_fp); if (unlikely(__pyx_t_1 == -1)) __PYX_ERR(0, 643, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":644 + * uint64 = 0 + * fread(&uint64, len_, fp) + * uint64 = le64toh(uint64) # <<<<<<<<<<<<<< + * + * if negative == 1: + */ + __pyx_v_uint64 = le64toh(__pyx_v_uint64); + + /* "s3ql/deltadump.pyx":641 + * uint64 = int8 + * + * if len_ != 0: # <<<<<<<<<<<<<< + * uint64 = 0 + * fread(&uint64, len_, fp) + */ + } + + /* "s3ql/deltadump.pyx":646 + * uint64 = le64toh(uint64) + * + * if negative == 1: # <<<<<<<<<<<<<< + * out[0] = - < int64_t > uint64 + * else: + */ + __pyx_t_2 = ((__pyx_v_negative == 1) != 0); + if (__pyx_t_2) { + + /* "s3ql/deltadump.pyx":647 + * + * if negative == 1: + * out[0] = - < int64_t > uint64 # <<<<<<<<<<<<<< + * else: + * out[0] = < int64_t > uint64 + */ + (__pyx_v_out[0]) = (-((int64_t)__pyx_v_uint64)); + + /* "s3ql/deltadump.pyx":646 + * uint64 = le64toh(uint64) + * + * if negative == 1: # <<<<<<<<<<<<<< + * out[0] = - < int64_t > uint64 + * else: + */ + goto __pyx_L6; + } + + /* "s3ql/deltadump.pyx":649 + * out[0] = - < int64_t > uint64 + * else: + * out[0] = < int64_t > uint64 # <<<<<<<<<<<<<< + * + * # len <= 8, safe to cast + */ + /*else*/ { + (__pyx_v_out[0]) = ((int64_t)__pyx_v_uint64); + } + __pyx_L6:; + + /* "s3ql/deltadump.pyx":652 + * + * # len <= 8, safe to cast + * return len_ + 1 # <<<<<<<<<<<<<< + */ + __pyx_r = (((int)__pyx_v_len_) + 1); + goto __pyx_L0; + + /* "s3ql/deltadump.pyx":609 + * return len_ + 1 + * + * cdef inline int read_integer(int64_t * out, FILE * fp) except -1: # <<<<<<<<<<<<<< + * '''Read integer written using `write_integer` from *fp* + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("s3ql.deltadump.read_integer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *__pyx_freelist_4s3ql_9deltadump___pyx_scope_struct__dump_table[8]; +static int __pyx_freecount_4s3ql_9deltadump___pyx_scope_struct__dump_table = 0; + +static PyObject *__pyx_tp_new_4s3ql_9deltadump___pyx_scope_struct__dump_table(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_4s3ql_9deltadump___pyx_scope_struct__dump_table > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table)))) { + o = (PyObject*)__pyx_freelist_4s3ql_9deltadump___pyx_scope_struct__dump_table[--__pyx_freecount_4s3ql_9deltadump___pyx_scope_struct__dump_table]; + memset(o, 0, sizeof(struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table)); + (void) PyObject_INIT(o, t); + } else { + o = (*t->tp_alloc)(t, 0); + if (unlikely(!o)) return 0; + } + return o; +} + +static void __pyx_tp_dealloc_4s3ql_9deltadump___pyx_scope_struct__dump_table(PyObject *o) { + if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_4s3ql_9deltadump___pyx_scope_struct__dump_table < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table)))) { + __pyx_freelist_4s3ql_9deltadump___pyx_scope_struct__dump_table[__pyx_freecount_4s3ql_9deltadump___pyx_scope_struct__dump_table++] = ((struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *)o); + } else { + (*Py_TYPE(o)->tp_free)(o); + } +} + +static PyTypeObject __pyx_type_4s3ql_9deltadump___pyx_scope_struct__dump_table = { + PyVarObject_HEAD_INIT(0, 0) + "s3ql.deltadump.__pyx_scope_struct__dump_table", /*tp_name*/ + sizeof(struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_4s3ql_9deltadump___pyx_scope_struct__dump_table, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_4s3ql_9deltadump___pyx_scope_struct__dump_table, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *__pyx_freelist_4s3ql_9deltadump___pyx_scope_struct_1_load_table[8]; +static int __pyx_freecount_4s3ql_9deltadump___pyx_scope_struct_1_load_table = 0; + +static PyObject *__pyx_tp_new_4s3ql_9deltadump___pyx_scope_struct_1_load_table(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_4s3ql_9deltadump___pyx_scope_struct_1_load_table > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table)))) { + o = (PyObject*)__pyx_freelist_4s3ql_9deltadump___pyx_scope_struct_1_load_table[--__pyx_freecount_4s3ql_9deltadump___pyx_scope_struct_1_load_table]; + memset(o, 0, sizeof(struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table)); + (void) PyObject_INIT(o, t); + } else { + o = (*t->tp_alloc)(t, 0); + if (unlikely(!o)) return 0; + } + return o; +} + +static void __pyx_tp_dealloc_4s3ql_9deltadump___pyx_scope_struct_1_load_table(PyObject *o) { + if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_4s3ql_9deltadump___pyx_scope_struct_1_load_table < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table)))) { + __pyx_freelist_4s3ql_9deltadump___pyx_scope_struct_1_load_table[__pyx_freecount_4s3ql_9deltadump___pyx_scope_struct_1_load_table++] = ((struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *)o); + } else { + (*Py_TYPE(o)->tp_free)(o); + } +} + +static PyTypeObject __pyx_type_4s3ql_9deltadump___pyx_scope_struct_1_load_table = { + PyVarObject_HEAD_INIT(0, 0) + "s3ql.deltadump.__pyx_scope_struct_1_load_table", /*tp_name*/ + sizeof(struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_4s3ql_9deltadump___pyx_scope_struct_1_load_table, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_4s3ql_9deltadump___pyx_scope_struct_1_load_table, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef __pyx_moduledef = { + #if PY_VERSION_HEX < 0x03020000 + { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, + #else + PyModuleDef_HEAD_INIT, + #endif + "deltadump", + __pyx_k_deltadump_pyx_this_file_is_part, /* m_doc */ + -1, /* m_size */ + __pyx_methods /* m_methods */, + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ +}; +#endif + +static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_kp_b_BEGIN_TRANSACTION, __pyx_k_BEGIN_TRANSACTION, sizeof(__pyx_k_BEGIN_TRANSACTION), 0, 0, 0, 0}, + {&__pyx_n_s_BLOB, __pyx_k_BLOB, sizeof(__pyx_k_BLOB), 0, 0, 1, 1}, + {&__pyx_kp_u_BLOB_too_large_to_read_d_vs_d, __pyx_k_BLOB_too_large_to_read_d_vs_d, sizeof(__pyx_k_BLOB_too_large_to_read_d_vs_d), 0, 1, 0, 0}, + {&__pyx_kp_b_COMMIT_TRANSACTION, __pyx_k_COMMIT_TRANSACTION, sizeof(__pyx_k_COMMIT_TRANSACTION), 0, 0, 0, 0}, + {&__pyx_kp_u_Can_not_dump_BLOB_of_size_d_max, __pyx_k_Can_not_dump_BLOB_of_size_d_max, sizeof(__pyx_k_Can_not_dump_BLOB_of_size_d_max), 0, 1, 0, 0}, + {&__pyx_kp_u_Can_t_access_in_memory_databases, __pyx_k_Can_t_access_in_memory_databases, sizeof(__pyx_k_Can_t_access_in_memory_databases), 0, 1, 0, 0}, + {&__pyx_kp_u_Can_t_dump_NULL_values, __pyx_k_Can_t_dump_NULL_values, sizeof(__pyx_k_Can_t_dump_NULL_values), 0, 1, 0, 0}, + {&__pyx_kp_u_Corrupted_input, __pyx_k_Corrupted_input, sizeof(__pyx_k_Corrupted_input), 0, 1, 0, 0}, + {&__pyx_n_s_ExitStack, __pyx_k_ExitStack, sizeof(__pyx_k_ExitStack), 0, 0, 1, 1}, + {&__pyx_kp_u_INSERT_INTO_s_s_VALUES_s, __pyx_k_INSERT_INTO_s_s_VALUES_s, sizeof(__pyx_k_INSERT_INTO_s_s_VALUES_s), 0, 1, 0, 0}, + {&__pyx_n_s_INTEGER, __pyx_k_INTEGER, sizeof(__pyx_k_INTEGER), 0, 0, 1, 1}, + {&__pyx_n_s_IOError, __pyx_k_IOError, sizeof(__pyx_k_IOError), 0, 0, 1, 1}, + {&__pyx_kp_u_Invalid_type_for_column_d, __pyx_k_Invalid_type_for_column_d, sizeof(__pyx_k_Invalid_type_for_column_d), 0, 1, 0, 0}, + {&__pyx_kp_u_Length_d_d_in_column_d, __pyx_k_Length_d_d_in_column_d, sizeof(__pyx_k_Length_d_d_in_column_d), 0, 1, 0, 0}, + {&__pyx_n_s_MAX_BLOB_SIZE, __pyx_k_MAX_BLOB_SIZE, sizeof(__pyx_k_MAX_BLOB_SIZE), 0, 0, 1, 1}, + {&__pyx_n_s_OSError, __pyx_k_OSError, sizeof(__pyx_k_OSError), 0, 0, 1, 1}, + {&__pyx_kp_u_Opening_connection_to_s, __pyx_k_Opening_connection_to_s, sizeof(__pyx_k_Opening_connection_to_s), 0, 1, 0, 0}, + {&__pyx_kp_u_PRAGMA_s, __pyx_k_PRAGMA_s, sizeof(__pyx_k_PRAGMA_s), 0, 1, 0, 0}, + {&__pyx_kp_u_PRAGMA_s_s, __pyx_k_PRAGMA_s_s, sizeof(__pyx_k_PRAGMA_s_s), 0, 1, 0, 0}, + {&__pyx_n_s_RuntimeError, __pyx_k_RuntimeError, sizeof(__pyx_k_RuntimeError), 0, 0, 1, 1}, + {&__pyx_kp_u_SELECT_COUNT_rowid_FROM_s, __pyx_k_SELECT_COUNT_rowid_FROM_s, sizeof(__pyx_k_SELECT_COUNT_rowid_FROM_s), 0, 1, 0, 0}, + {&__pyx_kp_u_SELECT_s_FROM_s_ORDER_BY_s, __pyx_k_SELECT_s_FROM_s_ORDER_BY_s, sizeof(__pyx_k_SELECT_s_FROM_s_ORDER_BY_s), 0, 1, 0, 0}, + {&__pyx_kp_u_SQLite_code_used_by_APSW_was_com, __pyx_k_SQLite_code_used_by_APSW_was_com, sizeof(__pyx_k_SQLite_code_used_by_APSW_was_com), 0, 1, 0, 0}, + {&__pyx_kp_u_SQLite_version_mismatch_between, __pyx_k_SQLite_version_mismatch_between, sizeof(__pyx_k_SQLite_version_mismatch_between), 0, 1, 0, 0}, + {&__pyx_n_s_TIME, __pyx_k_TIME, sizeof(__pyx_k_TIME), 0, 0, 1, 1}, + {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, + {&__pyx_kp_u__4, __pyx_k__4, sizeof(__pyx_k__4), 0, 1, 0, 0}, + {&__pyx_kp_u__9, __pyx_k__9, sizeof(__pyx_k__9), 0, 1, 0, 0}, + {&__pyx_n_s_apsw, __pyx_k_apsw, sizeof(__pyx_k_apsw), 0, 0, 1, 1}, + {&__pyx_kp_u_apsw_sqlite_compile_options_s_s3, __pyx_k_apsw_sqlite_compile_options_s_s3, sizeof(__pyx_k_apsw_sqlite_compile_options_s_s3), 0, 1, 0, 0}, + {&__pyx_n_s_apsw_sqlite_options, __pyx_k_apsw_sqlite_options, sizeof(__pyx_k_apsw_sqlite_options), 0, 0, 1, 1}, + {&__pyx_n_s_apsw_sqlite_version, __pyx_k_apsw_sqlite_version, sizeof(__pyx_k_apsw_sqlite_version), 0, 0, 1, 1}, + {&__pyx_kp_u_apsw_sqlite_version_s_s3ql_sqlit, __pyx_k_apsw_sqlite_version_s_s3ql_sqlit, sizeof(__pyx_k_apsw_sqlite_version_s_s3ql_sqlit), 0, 1, 0, 0}, + {&__pyx_n_s_begin_stmt, __pyx_k_begin_stmt, sizeof(__pyx_k_begin_stmt), 0, 0, 1, 1}, + {&__pyx_n_s_buf, __pyx_k_buf, sizeof(__pyx_k_buf), 0, 0, 1, 1}, + {&__pyx_n_s_callback, __pyx_k_callback, sizeof(__pyx_k_callback), 0, 0, 1, 1}, + {&__pyx_n_s_check_sqlite, __pyx_k_check_sqlite, sizeof(__pyx_k_check_sqlite), 0, 0, 1, 1}, + {&__pyx_n_s_cm, __pyx_k_cm, sizeof(__pyx_k_cm), 0, 0, 1, 1}, + {&__pyx_n_s_cmd, __pyx_k_cmd, sizeof(__pyx_k_cmd), 0, 0, 1, 1}, + {&__pyx_n_s_col_args, __pyx_k_col_args, sizeof(__pyx_k_col_args), 0, 0, 1, 1}, + {&__pyx_n_s_col_count, __pyx_k_col_count, sizeof(__pyx_k_col_count), 0, 0, 1, 1}, + {&__pyx_n_s_col_names, __pyx_k_col_names, sizeof(__pyx_k_col_names), 0, 0, 1, 1}, + {&__pyx_n_s_col_types, __pyx_k_col_types, sizeof(__pyx_k_col_types), 0, 0, 1, 1}, + {&__pyx_n_s_columns, __pyx_k_columns, sizeof(__pyx_k_columns), 0, 0, 1, 1}, + {&__pyx_n_s_commit_stmt, __pyx_k_commit_stmt, sizeof(__pyx_k_commit_stmt), 0, 0, 1, 1}, + {&__pyx_n_s_compile_options, __pyx_k_compile_options, sizeof(__pyx_k_compile_options), 0, 0, 1, 1}, + {&__pyx_n_s_contextlib, __pyx_k_contextlib, sizeof(__pyx_k_contextlib), 0, 0, 1, 1}, + {&__pyx_n_s_count, __pyx_k_count, sizeof(__pyx_k_count), 0, 0, 1, 1}, + {&__pyx_n_s_db, __pyx_k_db, sizeof(__pyx_k_db), 0, 0, 1, 1}, + {&__pyx_n_s_dbfile_b, __pyx_k_dbfile_b, sizeof(__pyx_k_dbfile_b), 0, 0, 1, 1}, + {&__pyx_n_s_debug, __pyx_k_debug, sizeof(__pyx_k_debug), 0, 0, 1, 1}, + {&__pyx_n_s_dump_table, __pyx_k_dump_table, sizeof(__pyx_k_dump_table), 0, 0, 1, 1}, + {&__pyx_n_s_dump_table_locals_lambda, __pyx_k_dump_table_locals_lambda, sizeof(__pyx_k_dump_table_locals_lambda), 0, 0, 1, 1}, + {&__pyx_kp_u_dump_table_s_writing_d_rows, __pyx_k_dump_table_s_writing_d_rows, sizeof(__pyx_k_dump_table_s_writing_d_rows), 0, 1, 0, 0}, + {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, + {&__pyx_n_s_enter, __pyx_k_enter, sizeof(__pyx_k_enter), 0, 0, 1, 1}, + {&__pyx_n_s_exceptionfor, __pyx_k_exceptionfor, sizeof(__pyx_k_exceptionfor), 0, 0, 1, 1}, + {&__pyx_n_s_exit, __pyx_k_exit, sizeof(__pyx_k_exit), 0, 0, 1, 1}, + {&__pyx_n_s_fh, __pyx_k_fh, sizeof(__pyx_k_fh), 0, 0, 1, 1}, + {&__pyx_n_s_file, __pyx_k_file, sizeof(__pyx_k_file), 0, 0, 1, 1}, + {&__pyx_n_s_fileno, __pyx_k_fileno, sizeof(__pyx_k_fileno), 0, 0, 1, 1}, + {&__pyx_n_u_foreign_keys, __pyx_k_foreign_keys, sizeof(__pyx_k_foreign_keys), 0, 1, 0, 1}, + {&__pyx_n_s_fp, __pyx_k_fp, sizeof(__pyx_k_fp), 0, 0, 1, 1}, + {&__pyx_n_s_getLogger, __pyx_k_getLogger, sizeof(__pyx_k_getLogger), 0, 0, 1, 1}, + {&__pyx_n_s_get_val, __pyx_k_get_val, sizeof(__pyx_k_get_val), 0, 0, 1, 1}, + {&__pyx_n_s_getfilesystemencoding, __pyx_k_getfilesystemencoding, sizeof(__pyx_k_getfilesystemencoding), 0, 0, 1, 1}, + {&__pyx_kp_s_home_nikratio_in_progress_s3ql, __pyx_k_home_nikratio_in_progress_s3ql, sizeof(__pyx_k_home_nikratio_in_progress_s3ql), 0, 0, 1, 0}, + {&__pyx_n_s_i, __pyx_k_i, sizeof(__pyx_k_i), 0, 0, 1, 1}, + {&__pyx_n_s_idx, __pyx_k_idx, sizeof(__pyx_k_idx), 0, 0, 1, 1}, + {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, + {&__pyx_n_s_int64, __pyx_k_int64, sizeof(__pyx_k_int64), 0, 0, 1, 1}, + {&__pyx_n_s_int64_prev, __pyx_k_int64_prev, sizeof(__pyx_k_int64_prev), 0, 0, 1, 1}, + {&__pyx_n_s_itertools, __pyx_k_itertools, sizeof(__pyx_k_itertools), 0, 0, 1, 1}, + {&__pyx_n_s_j, __pyx_k_j, sizeof(__pyx_k_j), 0, 0, 1, 1}, + {&__pyx_n_s_len, __pyx_k_len, sizeof(__pyx_k_len), 0, 0, 1, 1}, + {&__pyx_n_s_load_table, __pyx_k_load_table, sizeof(__pyx_k_load_table), 0, 0, 1, 1}, + {&__pyx_n_s_load_table_locals_lambda, __pyx_k_load_table_locals_lambda, sizeof(__pyx_k_load_table_locals_lambda), 0, 0, 1, 1}, + {&__pyx_kp_u_load_table_s_reading_d_rows, __pyx_k_load_table_s_reading_d_rows, sizeof(__pyx_k_load_table_s_reading_d_rows), 0, 1, 0, 0}, + {&__pyx_n_s_log, __pyx_k_log, sizeof(__pyx_k_log), 0, 0, 1, 1}, + {&__pyx_n_s_logging, __pyx_k_logging, sizeof(__pyx_k_logging), 0, 0, 1, 1}, + {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, + {&__pyx_kp_u_memory, __pyx_k_memory, sizeof(__pyx_k_memory), 0, 1, 0, 0}, + {&__pyx_n_s_name, __pyx_k_name, sizeof(__pyx_k_name), 0, 0, 1, 1}, + {&__pyx_n_s_order, __pyx_k_order, sizeof(__pyx_k_order), 0, 0, 1, 1}, + {&__pyx_n_s_os, __pyx_k_os, sizeof(__pyx_k_os), 0, 0, 1, 1}, + {&__pyx_n_s_pragma, __pyx_k_pragma, sizeof(__pyx_k_pragma), 0, 0, 1, 1}, + {&__pyx_n_s_query, __pyx_k_query, sizeof(__pyx_k_query), 0, 0, 1, 1}, + {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, + {&__pyx_n_s_rc, __pyx_k_rc, sizeof(__pyx_k_rc), 0, 0, 1, 1}, + {&__pyx_n_s_row_count, __pyx_k_row_count, sizeof(__pyx_k_row_count), 0, 0, 1, 1}, + {&__pyx_n_s_s3ql_deltadump, __pyx_k_s3ql_deltadump, sizeof(__pyx_k_s3ql_deltadump), 0, 0, 1, 1}, + {&__pyx_n_s_s3ql_sqlite_options, __pyx_k_s3ql_sqlite_options, sizeof(__pyx_k_s3ql_sqlite_options), 0, 0, 1, 1}, + {&__pyx_n_s_s3ql_sqlite_version, __pyx_k_s3ql_sqlite_version, sizeof(__pyx_k_s3ql_sqlite_version), 0, 0, 1, 1}, + {&__pyx_n_s_sqlite3_db, __pyx_k_sqlite3_db, sizeof(__pyx_k_sqlite3_db), 0, 0, 1, 1}, + {&__pyx_n_s_sqlitelibversion, __pyx_k_sqlitelibversion, sizeof(__pyx_k_sqlitelibversion), 0, 0, 1, 1}, + {&__pyx_n_s_stmt, __pyx_k_stmt, sizeof(__pyx_k_stmt), 0, 0, 1, 1}, + {&__pyx_n_u_surrogateescape, __pyx_k_surrogateescape, sizeof(__pyx_k_surrogateescape), 0, 1, 0, 1}, + {&__pyx_n_u_synchronous, __pyx_k_synchronous, sizeof(__pyx_k_synchronous), 0, 1, 0, 1}, + {&__pyx_n_s_sys, __pyx_k_sys, sizeof(__pyx_k_sys), 0, 0, 1, 1}, + {&__pyx_n_s_table, __pyx_k_table, sizeof(__pyx_k_table), 0, 0, 1, 1}, + {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, + {&__pyx_n_s_tmp, __pyx_k_tmp, sizeof(__pyx_k_tmp), 0, 0, 1, 1}, + {&__pyx_n_s_trx_rows, __pyx_k_trx_rows, sizeof(__pyx_k_trx_rows), 0, 0, 1, 1}, + {&__pyx_n_s_val, __pyx_k_val, sizeof(__pyx_k_val), 0, 0, 1, 1}, + {0, 0, 0, 0, 0, 0, 0} +}; +static int __Pyx_InitCachedBuiltins(void) { + __pyx_builtin_OSError = __Pyx_GetBuiltinName(__pyx_n_s_OSError); if (!__pyx_builtin_OSError) __PYX_ERR(0, 136, __pyx_L1_error) + __pyx_builtin_IOError = __Pyx_GetBuiltinName(__pyx_n_s_IOError); if (!__pyx_builtin_IOError) __PYX_ERR(0, 116, __pyx_L1_error) + __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(0, 206, __pyx_L1_error) + __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 208, __pyx_L1_error) + __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_n_s_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(0, 258, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} + +static int __Pyx_InitCachedConstants(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "s3ql/deltadump.pyx":263 + * apsw_sqlite_options = set(apsw.compile_options) + * s3ql_sqlite_options = set() + * for idx in itertools.count(0): # <<<<<<<<<<<<<< + * buf = sqlite3_compileoption_get(idx) + * if buf is NULL: + */ + __pyx_tuple__2 = PyTuple_Pack(1, __pyx_int_0); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(0, 263, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__2); + __Pyx_GIVEREF(__pyx_tuple__2); + + /* "s3ql/deltadump.pyx":319 + * + * if db.file == ':memory:': + * raise ValueError("Can't access in-memory databases") # <<<<<<<<<<<<<< + * + * with ExitStack() as cm: + */ + __pyx_tuple__3 = PyTuple_Pack(1, __pyx_kp_u_Can_t_access_in_memory_databases); if (unlikely(!__pyx_tuple__3)) __PYX_ERR(0, 319, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__3); + __Pyx_GIVEREF(__pyx_tuple__3); + + /* "s3ql/deltadump.pyx":368 + * for i in range(col_count): + * if sqlite3_column_type(stmt, i) is SQLITE_NULL: + * raise ValueError("Can't dump NULL values") # <<<<<<<<<<<<<< + * + * if col_types[i] == _INTEGER: + */ + __pyx_tuple__5 = PyTuple_Pack(1, __pyx_kp_u_Can_t_dump_NULL_values); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(0, 368, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__5); + __Pyx_GIVEREF(__pyx_tuple__5); + + /* "s3ql/deltadump.pyx":321 + * raise ValueError("Can't access in-memory databases") + * + * with ExitStack() as cm: # <<<<<<<<<<<<<< + * # Get SQLite connection + * log.debug('Opening connection to %s', db.file) + */ + __pyx_tuple__6 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(0, 321, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__6); + __Pyx_GIVEREF(__pyx_tuple__6); + + /* "s3ql/deltadump.pyx":432 + * + * if db.file == ':memory:': + * raise ValueError("Can't access in-memory databases") # <<<<<<<<<<<<<< + * + * with ExitStack() as cm: + */ + __pyx_tuple__7 = PyTuple_Pack(1, __pyx_kp_u_Can_t_access_in_memory_databases); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(0, 432, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__7); + __Pyx_GIVEREF(__pyx_tuple__7); + + /* "s3ql/deltadump.pyx":447 + * + * # Copy settings + * for pragma in ('synchronous', 'foreign_keys'): # <<<<<<<<<<<<<< + * val = db.get_val('PRAGMA %s' % pragma) + * cmd = ('PRAGMA %s = %s' % (pragma, val)).encode('utf-8') + */ + __pyx_tuple__8 = PyTuple_Pack(2, __pyx_n_u_synchronous, __pyx_n_u_foreign_keys); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(0, 447, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__8); + __Pyx_GIVEREF(__pyx_tuple__8); + + /* "s3ql/deltadump.pyx":532 + * tmp = int64_prev[j] + int64 + * if tmp < 0 or tmp > INT_MAX: + * raise RuntimeError('Corrupted input') # <<<<<<<<<<<<<< + * len_ = tmp + * int64_prev[j] = tmp + */ + __pyx_tuple__10 = PyTuple_Pack(1, __pyx_kp_u_Corrupted_input); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(0, 532, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__10); + __Pyx_GIVEREF(__pyx_tuple__10); + + /* "s3ql/deltadump.pyx":434 + * raise ValueError("Can't access in-memory databases") + * + * with ExitStack() as cm: # <<<<<<<<<<<<<< + * # Get SQLite connection + * log.debug('Opening connection to %s', db.file) + */ + __pyx_tuple__11 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__11)) __PYX_ERR(0, 434, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__11); + __Pyx_GIVEREF(__pyx_tuple__11); + + /* "s3ql/deltadump.pyx":237 + * return fp + * + * def check_sqlite(): # <<<<<<<<<<<<<< + * '''Check if deltadump and apsw module use compatible SQLite code. + * + */ + __pyx_tuple__12 = PyTuple_Pack(6, __pyx_n_s_buf, __pyx_n_s_apsw_sqlite_version, __pyx_n_s_s3ql_sqlite_version, __pyx_n_s_apsw_sqlite_options, __pyx_n_s_s3ql_sqlite_options, __pyx_n_s_idx); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(0, 237, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__12); + __Pyx_GIVEREF(__pyx_tuple__12); + __pyx_codeobj__13 = (PyObject*)__Pyx_PyCode_New(0, 0, 6, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__12, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_home_nikratio_in_progress_s3ql, __pyx_n_s_check_sqlite, 237, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__13)) __PYX_ERR(0, 237, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":280 + * s3ql_sqlite_options - apsw_sqlite_options)) + * + * def dump_table(table, order, columns, db, fh): # <<<<<<<<<<<<<< + * '''Dump *columns* of *table* into *fh* + * + */ + __pyx_tuple__14 = PyTuple_Pack(23, __pyx_n_s_table, __pyx_n_s_order, __pyx_n_s_columns, __pyx_n_s_db, __pyx_n_s_fh, __pyx_n_s_sqlite3_db, __pyx_n_s_stmt, __pyx_n_s_col_types, __pyx_n_s_col_args, __pyx_n_s_col_count, __pyx_n_s_rc, __pyx_n_s_i, __pyx_n_s_len, __pyx_n_s_int64_prev, __pyx_n_s_int64, __pyx_n_s_tmp, __pyx_n_s_fp, __pyx_n_s_buf, __pyx_n_s_row_count, __pyx_n_s_cm, __pyx_n_s_dbfile_b, __pyx_n_s_col_names, __pyx_n_s_query); if (unlikely(!__pyx_tuple__14)) __PYX_ERR(0, 280, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__14); + __Pyx_GIVEREF(__pyx_tuple__14); + __pyx_codeobj__15 = (PyObject*)__Pyx_PyCode_New(5, 0, 23, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_home_nikratio_in_progress_s3ql, __pyx_n_s_dump_table, 280, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__15)) __PYX_ERR(0, 280, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":401 + * fwrite(buf, len_, fp) + * + * def load_table(table, columns, db, fh, trx_rows=5000): # <<<<<<<<<<<<<< + * '''Load *columns* of *table* from *fh* + * + */ + __pyx_tuple__16 = PyTuple_Pack(29, __pyx_n_s_table, __pyx_n_s_columns, __pyx_n_s_db, __pyx_n_s_fh, __pyx_n_s_trx_rows, __pyx_n_s_sqlite3_db, __pyx_n_s_stmt, __pyx_n_s_begin_stmt, __pyx_n_s_commit_stmt, __pyx_n_s_col_types, __pyx_n_s_col_args, __pyx_n_s_col_count, __pyx_n_s_rc, __pyx_n_s_len, __pyx_n_s_i, __pyx_n_s_j, __pyx_n_s_int64_prev, __pyx_n_s_fp, __pyx_n_s_buf, __pyx_n_s_row_count, __pyx_n_s_int64, __pyx_n_s_tmp, __pyx_n_s_cm, __pyx_n_s_dbfile_b, __pyx_n_s_pragma, __pyx_n_s_val, __pyx_n_s_cmd, __pyx_n_s_col_names, __pyx_n_s_query); if (unlikely(!__pyx_tuple__16)) __PYX_ERR(0, 401, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__16); + __Pyx_GIVEREF(__pyx_tuple__16); + __pyx_codeobj__17 = (PyObject*)__Pyx_PyCode_New(5, 0, 29, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__16, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_home_nikratio_in_progress_s3ql, __pyx_n_s_load_table, 401, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__17)) __PYX_ERR(0, 401, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_InitGlobals(void) { + if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + __pyx_int_0 = PyInt_FromLong(0); if (unlikely(!__pyx_int_0)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_int_4096 = PyInt_FromLong(4096); if (unlikely(!__pyx_int_4096)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_int_5000 = PyInt_FromLong(5000); if (unlikely(!__pyx_int_5000)) __PYX_ERR(0, 1, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} + +#if PY_MAJOR_VERSION < 3 +PyMODINIT_FUNC initdeltadump(void); /*proto*/ +PyMODINIT_FUNC initdeltadump(void) +#else +PyMODINIT_FUNC PyInit_deltadump(void); /*proto*/ +PyMODINIT_FUNC PyInit_deltadump(void) +#endif +{ + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + __Pyx_RefNannyDeclarations + #if CYTHON_REFNANNY + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); + if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); + } + #endif + __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit_deltadump(void)", 0); + if (__Pyx_check_binary_version() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + #ifdef __Pyx_CyFunction_USED + if (__pyx_CyFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Coroutine_USED + if (__pyx_Coroutine_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_StopAsyncIteration_USED + if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + /*--- Library function declarations ---*/ + /*--- Threads initialization code ---*/ + #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS + #ifdef WITH_THREAD /* Python build with threading support? */ + PyEval_InitThreads(); + #endif + #endif + /*--- Module creation code ---*/ + #if PY_MAJOR_VERSION < 3 + __pyx_m = Py_InitModule4("deltadump", __pyx_methods, __pyx_k_deltadump_pyx_this_file_is_part, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); + #else + __pyx_m = PyModule_Create(&__pyx_moduledef); + #endif + if (unlikely(!__pyx_m)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_d); + __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + #if CYTHON_COMPILING_IN_PYPY + Py_INCREF(__pyx_b); + #endif + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitGlobals() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) + if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + if (__pyx_module_is_main_s3ql__deltadump) { + if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + } + #if PY_MAJOR_VERSION >= 3 + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "s3ql.deltadump")) { + if (unlikely(PyDict_SetItemString(modules, "s3ql.deltadump", __pyx_m) < 0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + #endif + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global init code ---*/ + /*--- Variable export code ---*/ + /*--- Function export code ---*/ + /*--- Type init code ---*/ + if (PyType_Ready(&__pyx_type_4s3ql_9deltadump___pyx_scope_struct__dump_table) < 0) __PYX_ERR(0, 280, __pyx_L1_error) + __pyx_type_4s3ql_9deltadump___pyx_scope_struct__dump_table.tp_print = 0; + __pyx_ptype_4s3ql_9deltadump___pyx_scope_struct__dump_table = &__pyx_type_4s3ql_9deltadump___pyx_scope_struct__dump_table; + if (PyType_Ready(&__pyx_type_4s3ql_9deltadump___pyx_scope_struct_1_load_table) < 0) __PYX_ERR(0, 401, __pyx_L1_error) + __pyx_type_4s3ql_9deltadump___pyx_scope_struct_1_load_table.tp_print = 0; + __pyx_ptype_4s3ql_9deltadump___pyx_scope_struct_1_load_table = &__pyx_type_4s3ql_9deltadump___pyx_scope_struct_1_load_table; + /*--- Type import code ---*/ + __pyx_ptype_7cpython_4type_type = __Pyx_ImportType(__Pyx_BUILTIN_MODULE_NAME, "type", + #if CYTHON_COMPILING_IN_PYPY + sizeof(PyTypeObject), + #else + sizeof(PyHeapTypeObject), + #endif + 0); if (unlikely(!__pyx_ptype_7cpython_4type_type)) __PYX_ERR(1, 9, __pyx_L1_error) + /*--- Variable import code ---*/ + /*--- Function import code ---*/ + /*--- Execution code ---*/ + #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) + if (__Pyx_patch_abc() < 0) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + + /* "s3ql/deltadump.pyx":79 + * SQLITE_OPEN_READONLY + * + * from contextlib import ExitStack # <<<<<<<<<<<<<< + * import apsw + * import os + */ + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 79, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_n_s_ExitStack); + __Pyx_GIVEREF(__pyx_n_s_ExitStack); + PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_ExitStack); + __pyx_t_2 = __Pyx_Import(__pyx_n_s_contextlib, __pyx_t_1, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 79, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_ExitStack); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 79, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_ExitStack, __pyx_t_1) < 0) __PYX_ERR(0, 79, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "s3ql/deltadump.pyx":80 + * + * from contextlib import ExitStack + * import apsw # <<<<<<<<<<<<<< + * import os + * from .logging import logging # Ensure use of custom logger class + */ + __pyx_t_2 = __Pyx_Import(__pyx_n_s_apsw, 0, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 80, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_apsw, __pyx_t_2) < 0) __PYX_ERR(0, 80, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "s3ql/deltadump.pyx":81 + * from contextlib import ExitStack + * import apsw + * import os # <<<<<<<<<<<<<< + * from .logging import logging # Ensure use of custom logger class + * import itertools + */ + __pyx_t_2 = __Pyx_Import(__pyx_n_s_os, 0, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 81, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_os, __pyx_t_2) < 0) __PYX_ERR(0, 81, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "s3ql/deltadump.pyx":82 + * import apsw + * import os + * from .logging import logging # Ensure use of custom logger class # <<<<<<<<<<<<<< + * import itertools + * import sys + */ + __pyx_t_2 = PyList_New(1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 82, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_n_s_logging); + __Pyx_GIVEREF(__pyx_n_s_logging); + PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_logging); + __pyx_t_1 = __Pyx_Import(__pyx_n_s_logging, __pyx_t_2, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 82, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_logging); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 82, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_logging, __pyx_t_2) < 0) __PYX_ERR(0, 82, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":83 + * import os + * from .logging import logging # Ensure use of custom logger class + * import itertools # <<<<<<<<<<<<<< + * import sys + * + */ + __pyx_t_1 = __Pyx_Import(__pyx_n_s_itertools, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 83, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_itertools, __pyx_t_1) < 0) __PYX_ERR(0, 83, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":84 + * from .logging import logging # Ensure use of custom logger class + * import itertools + * import sys # <<<<<<<<<<<<<< + * + * log = logging.getLogger(__name__) + */ + __pyx_t_1 = __Pyx_Import(__pyx_n_s_sys, 0, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 84, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_sys, __pyx_t_1) < 0) __PYX_ERR(0, 84, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":86 + * import sys + * + * log = logging.getLogger(__name__) # <<<<<<<<<<<<<< + * + * # Column types + */ + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_logging); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_getLogger); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_name); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = NULL; + if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_4)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + } + } + if (!__pyx_t_4) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else { + __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4); __pyx_t_4 = NULL; + __Pyx_GIVEREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_t_2); + __pyx_t_2 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (PyDict_SetItem(__pyx_d, __pyx_n_s_log, __pyx_t_1) < 0) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":89 + * + * # Column types + * cdef int _INTEGER = 1 # <<<<<<<<<<<<<< + * cdef int _BLOB = 2 + * cdef int _TIME = 3 + */ + __pyx_v_4s3ql_9deltadump__INTEGER = 1; + + /* "s3ql/deltadump.pyx":90 + * # Column types + * cdef int _INTEGER = 1 + * cdef int _BLOB = 2 # <<<<<<<<<<<<<< + * cdef int _TIME = 3 + * + */ + __pyx_v_4s3ql_9deltadump__BLOB = 2; + + /* "s3ql/deltadump.pyx":91 + * cdef int _INTEGER = 1 + * cdef int _BLOB = 2 + * cdef int _TIME = 3 # <<<<<<<<<<<<<< + * + * # Make column types available as Python objects + */ + __pyx_v_4s3ql_9deltadump__TIME = 3; + + /* "s3ql/deltadump.pyx":94 + * + * # Make column types available as Python objects + * INTEGER = _INTEGER # <<<<<<<<<<<<<< + * BLOB = _BLOB + * TIME = _TIME + */ + __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_4s3ql_9deltadump__INTEGER); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 94, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_INTEGER, __pyx_t_1) < 0) __PYX_ERR(0, 94, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":95 + * # Make column types available as Python objects + * INTEGER = _INTEGER + * BLOB = _BLOB # <<<<<<<<<<<<<< + * TIME = _TIME + * + */ + __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_4s3ql_9deltadump__BLOB); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 95, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_BLOB, __pyx_t_1) < 0) __PYX_ERR(0, 95, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":96 + * INTEGER = _INTEGER + * BLOB = _BLOB + * TIME = _TIME # <<<<<<<<<<<<<< + * + * # Integer length codes + */ + __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_4s3ql_9deltadump__TIME); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 96, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_TIME, __pyx_t_1) < 0) __PYX_ERR(0, 96, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":99 + * + * # Integer length codes + * cdef uint8_t INT8 = 127 # <<<<<<<<<<<<<< + * cdef uint8_t INT16 = 126 + * cdef uint8_t INT32 = 125 + */ + __pyx_v_4s3ql_9deltadump_INT8 = 0x7F; + + /* "s3ql/deltadump.pyx":100 + * # Integer length codes + * cdef uint8_t INT8 = 127 + * cdef uint8_t INT16 = 126 # <<<<<<<<<<<<<< + * cdef uint8_t INT32 = 125 + * cdef uint8_t INT64 = 124 + */ + __pyx_v_4s3ql_9deltadump_INT16 = 0x7E; + + /* "s3ql/deltadump.pyx":101 + * cdef uint8_t INT8 = 127 + * cdef uint8_t INT16 = 126 + * cdef uint8_t INT32 = 125 # <<<<<<<<<<<<<< + * cdef uint8_t INT64 = 124 + * + */ + __pyx_v_4s3ql_9deltadump_INT32 = 0x7D; + + /* "s3ql/deltadump.pyx":102 + * cdef uint8_t INT16 = 126 + * cdef uint8_t INT32 = 125 + * cdef uint8_t INT64 = 124 # <<<<<<<<<<<<<< + * + * # Maximum size of BLOBs + */ + __pyx_v_4s3ql_9deltadump_INT64 = 0x7C; + + /* "s3ql/deltadump.pyx":105 + * + * # Maximum size of BLOBs + * MAX_BLOB_SIZE = 4096 # <<<<<<<<<<<<<< + * + * # Scale factor from time floats to integers. 1e9 would give nanosecond + */ + if (PyDict_SetItem(__pyx_d, __pyx_n_s_MAX_BLOB_SIZE, __pyx_int_4096) < 0) __PYX_ERR(0, 105, __pyx_L1_error) + + /* "s3ql/deltadump.pyx":110 + * # resolution but introduces rounding errors, so we use 1 << 30 (which is + * # ~1.074e9, i.e. we get a little more precision than nanoseconds). + * cdef double time_scale = 1 << 30 # <<<<<<<<<<<<<< + * + * cdef inline int fwrite(const_void * buf, size_t len_, FILE * fp) except -1: + */ + __pyx_v_4s3ql_9deltadump_time_scale = 1073741824.0; + + /* "s3ql/deltadump.pyx":136 + * return None + * + * cdef int raise_from_errno(err_class=OSError) except -1: # <<<<<<<<<<<<<< + * '''Raise OSError for current errno value''' + * + */ + __Pyx_INCREF(__pyx_builtin_OSError); + __pyx_k_ = __pyx_builtin_OSError; + __Pyx_GIVEREF(__pyx_builtin_OSError); + + /* "s3ql/deltadump.pyx":237 + * return fp + * + * def check_sqlite(): # <<<<<<<<<<<<<< + * '''Check if deltadump and apsw module use compatible SQLite code. + * + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_1check_sqlite, NULL, __pyx_n_s_s3ql_deltadump); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 237, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_check_sqlite, __pyx_t_1) < 0) __PYX_ERR(0, 237, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":280 + * s3ql_sqlite_options - apsw_sqlite_options)) + * + * def dump_table(table, order, columns, db, fh): # <<<<<<<<<<<<<< + * '''Dump *columns* of *table* into *fh* + * + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_3dump_table, NULL, __pyx_n_s_s3ql_deltadump); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 280, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_dump_table, __pyx_t_1) < 0) __PYX_ERR(0, 280, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":401 + * fwrite(buf, len_, fp) + * + * def load_table(table, columns, db, fh, trx_rows=5000): # <<<<<<<<<<<<<< + * '''Load *columns* of *table* from *fh* + * + */ + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_5load_table, NULL, __pyx_n_s_s3ql_deltadump); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 401, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_load_table, __pyx_t_1) < 0) __PYX_ERR(0, 401, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "s3ql/deltadump.pyx":1 + * ''' # <<<<<<<<<<<<<< + * deltadump.pyx - this file is part of S3QL (http://s3ql.googlecode.com) + * + */ + __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + if (__pyx_m) { + if (__pyx_d) { + __Pyx_AddTraceback("init s3ql.deltadump", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + Py_DECREF(__pyx_m); __pyx_m = 0; + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init s3ql.deltadump"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if PY_MAJOR_VERSION < 3 + return; + #else + return __pyx_m; + #endif +} + +/* --- Runtime support code --- */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule((char *)modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); + if (unlikely(!result)) { + PyErr_Format(PyExc_NameError, +#if PY_MAJOR_VERSION >= 3 + "name '%U' is not defined", name); +#else + "name '%.200s' is not defined", PyString_AS_STRING(name)); +#endif + } + return result; +} + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = func->ob_type->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +} +#endif + +#if PY_MAJOR_VERSION < 3 +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, + CYTHON_UNUSED PyObject *cause) { + __Pyx_PyThreadState_declare + Py_XINCREF(type); + if (!value || value == Py_None) + value = NULL; + else + Py_INCREF(value); + if (!tb || tb == Py_None) + tb = NULL; + else { + Py_INCREF(tb); + if (!PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto raise_error; + } + } + if (PyType_Check(type)) { +#if CYTHON_COMPILING_IN_PYPY + if (!value) { + Py_INCREF(Py_None); + value = Py_None; + } +#endif + PyErr_NormalizeException(&type, &value, &tb); + } else { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto raise_error; + } + value = type; + type = (PyObject*) Py_TYPE(type); + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } + } + __Pyx_PyThreadState_assign + __Pyx_ErrRestore(type, value, tb); + return; +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(tb); + return; +} +#else +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } +#if PY_VERSION_HEX >= 0x03030000 + if (cause) { +#else + if (cause && cause != Py_None) { +#endif + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if CYTHON_COMPILING_IN_PYPY + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#else + PyThreadState *tstate = PyThreadState_GET(); + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} +#endif + +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { + PyObject *result; +#if CYTHON_COMPILING_IN_CPYTHON + result = PyDict_GetItem(__pyx_d, name); + if (likely(result)) { + Py_INCREF(result); + } else { +#else + result = PyObject_GetItem(__pyx_d, name); + if (!result) { + PyErr_Clear(); +#endif + result = __Pyx_GetBuiltinName(name); + } + return result; +} + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = PyCFunction_GET_FUNCTION(func); + self = PyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +#if CYTHON_COMPILING_IN_CPYTHON +static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_New(1); + if (unlikely(!args)) return NULL; + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 0, arg); + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { +#ifdef __Pyx_CyFunction_USED + if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { +#else + if (likely(PyCFunction_Check(func))) { +#endif + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, arg); + } + } + return __Pyx__PyObject_CallOneArg(func, arg); +} +#else +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_Pack(1, arg); + if (unlikely(!args)) return NULL; + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +#endif + +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { + PyObject *r; + if (!j) return NULL; + r = PyObject_GetItem(o, j); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_COMPILING_IN_CPYTHON + if (wraparound & unlikely(i < 0)) i += PyList_GET_SIZE(o); + if ((!boundscheck) || likely((0 <= i) & (i < PyList_GET_SIZE(o)))) { + PyObject *r = PyList_GET_ITEM(o, i); + Py_INCREF(r); + return r; + } + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +#else + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_COMPILING_IN_CPYTHON + if (wraparound & unlikely(i < 0)) i += PyTuple_GET_SIZE(o); + if ((!boundscheck) || likely((0 <= i) & (i < PyTuple_GET_SIZE(o)))) { + PyObject *r = PyTuple_GET_ITEM(o, i); + Py_INCREF(r); + return r; + } + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +#else + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_COMPILING_IN_CPYTHON + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); + if ((!boundscheck) || (likely((n >= 0) & (n < PyList_GET_SIZE(o))))) { + PyObject *r = PyList_GET_ITEM(o, n); + Py_INCREF(r); + return r; + } + } + else if (PyTuple_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); + if ((!boundscheck) || likely((n >= 0) & (n < PyTuple_GET_SIZE(o)))) { + PyObject *r = PyTuple_GET_ITEM(o, n); + Py_INCREF(r); + return r; + } + } else { + PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; + if (likely(m && m->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { + Py_ssize_t l = m->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + PyErr_Clear(); + } + } + return m->sq_item(o, i); + } + } +#else + if (is_list || PySequence_Check(o)) { + return PySequence_GetItem(o, i); + } +#endif + return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); +} + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { +#ifdef __Pyx_CyFunction_USED + if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { +#else + if (likely(PyCFunction_Check(func))) { +#endif + if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { + return __Pyx_PyObject_CallMethO(func, NULL); + } + } + return __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL); +} +#endif + +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION >= 3 + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); + #else + "%s() got multiple values for keyword argument '%s'", func_name, + PyString_AsString(kw_name)); + #endif +} + +static int __Pyx_ParseOptionalKeywords( + PyObject *kwds, + PyObject **argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject *key = 0, *value = 0; + Py_ssize_t pos = 0; + PyObject*** name; + PyObject*** first_kw_arg = argnames + num_pos_args; + while (PyDict_Next(kwds, &pos, &key, &value)) { + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + values[name-argnames] = value; + continue; + } + name = first_kw_arg; + #if PY_MAJOR_VERSION < 3 + if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { + while (*name) { + if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) + && _PyString_Eq(**name, key)) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + if ((**argname == key) || ( + (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) + && _PyString_Eq(**argname, key))) { + goto arg_passed_twice; + } + argname++; + } + } + } else + #endif + if (likely(PyUnicode_Check(key))) { + while (*name) { + int cmp = (**name == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**name, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + int cmp = (**argname == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**argname, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + argname++; + } + } + } else + goto invalid_keyword_type; + if (kwds2) { + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else { + goto invalid_keyword; + } + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%.200s() got an unexpected keyword argument '%.200s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif +bad: + return -1; +} + +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY + return PyObject_RichCompareBool(s1, s2, equals); +#else + if (s1 == s2) { + return (equals == Py_EQ); + } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) { + const char *ps1, *ps2; + Py_ssize_t length = PyBytes_GET_SIZE(s1); + if (length != PyBytes_GET_SIZE(s2)) + return (equals == Py_NE); + ps1 = PyBytes_AS_STRING(s1); + ps2 = PyBytes_AS_STRING(s2); + if (ps1[0] != ps2[0]) { + return (equals == Py_NE); + } else if (length == 1) { + return (equals == Py_EQ); + } else { + int result = memcmp(ps1, ps2, (size_t)length); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) { + return (equals == Py_NE); + } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) { + return (equals == Py_NE); + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +#endif +} + +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY + return PyObject_RichCompareBool(s1, s2, equals); +#else +#if PY_MAJOR_VERSION < 3 + PyObject* owned_ref = NULL; +#endif + int s1_is_unicode, s2_is_unicode; + if (s1 == s2) { + goto return_eq; + } + s1_is_unicode = PyUnicode_CheckExact(s1); + s2_is_unicode = PyUnicode_CheckExact(s2); +#if PY_MAJOR_VERSION < 3 + if ((s1_is_unicode & (!s2_is_unicode)) && PyString_CheckExact(s2)) { + owned_ref = PyUnicode_FromObject(s2); + if (unlikely(!owned_ref)) + return -1; + s2 = owned_ref; + s2_is_unicode = 1; + } else if ((s2_is_unicode & (!s1_is_unicode)) && PyString_CheckExact(s1)) { + owned_ref = PyUnicode_FromObject(s1); + if (unlikely(!owned_ref)) + return -1; + s1 = owned_ref; + s1_is_unicode = 1; + } else if (((!s2_is_unicode) & (!s1_is_unicode))) { + return __Pyx_PyBytes_Equals(s1, s2, equals); + } +#endif + if (s1_is_unicode & s2_is_unicode) { + Py_ssize_t length; + int kind; + void *data1, *data2; + if (unlikely(__Pyx_PyUnicode_READY(s1) < 0) || unlikely(__Pyx_PyUnicode_READY(s2) < 0)) + return -1; + length = __Pyx_PyUnicode_GET_LENGTH(s1); + if (length != __Pyx_PyUnicode_GET_LENGTH(s2)) { + goto return_ne; + } + kind = __Pyx_PyUnicode_KIND(s1); + if (kind != __Pyx_PyUnicode_KIND(s2)) { + goto return_ne; + } + data1 = __Pyx_PyUnicode_DATA(s1); + data2 = __Pyx_PyUnicode_DATA(s2); + if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) { + goto return_ne; + } else if (length == 1) { + goto return_eq; + } else { + int result = memcmp(data1, data2, (size_t)(length * kind)); + #if PY_MAJOR_VERSION < 3 + Py_XDECREF(owned_ref); + #endif + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & s2_is_unicode) { + goto return_ne; + } else if ((s2 == Py_None) & s1_is_unicode) { + goto return_ne; + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +return_eq: + #if PY_MAJOR_VERSION < 3 + Py_XDECREF(owned_ref); + #endif + return (equals == Py_EQ); +return_ne: + #if PY_MAJOR_VERSION < 3 + Py_XDECREF(owned_ref); + #endif + return (equals == Py_NE); +#endif +} + +static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { + PyObject* fake_module; + PyTypeObject* cached_type = NULL; + fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); + if (!fake_module) return NULL; + Py_INCREF(fake_module); + cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); + if (cached_type) { + if (!PyType_Check((PyObject*)cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", + type->tp_name); + goto bad; + } + if (cached_type->tp_basicsize != type->tp_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + type->tp_name); + goto bad; + } + } else { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + if (PyType_Ready(type) < 0) goto bad; + if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) + goto bad; + Py_INCREF(type); + cached_type = type; + } +done: + Py_DECREF(fake_module); + return cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +static PyObject * +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure) +{ + if (unlikely(op->func_doc == NULL)) { + if (op->func.m_ml->ml_doc) { +#if PY_MAJOR_VERSION >= 3 + op->func_doc = PyUnicode_FromString(op->func.m_ml->ml_doc); +#else + op->func_doc = PyString_FromString(op->func.m_ml->ml_doc); +#endif + if (unlikely(op->func_doc == NULL)) + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } + } + Py_INCREF(op->func_doc); + return op->func_doc; +} +static int +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value) +{ + PyObject *tmp = op->func_doc; + if (value == NULL) { + value = Py_None; + } + Py_INCREF(value); + op->func_doc = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_name == NULL)) { +#if PY_MAJOR_VERSION >= 3 + op->func_name = PyUnicode_InternFromString(op->func.m_ml->ml_name); +#else + op->func_name = PyString_InternFromString(op->func.m_ml->ml_name); +#endif + if (unlikely(op->func_name == NULL)) + return NULL; + } + Py_INCREF(op->func_name); + return op->func_name; +} +static int +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value) +{ + PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + if (unlikely(value == NULL || !PyUnicode_Check(value))) { +#else + if (unlikely(value == NULL || !PyString_Check(value))) { +#endif + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + tmp = op->func_name; + Py_INCREF(value); + op->func_name = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op) +{ + Py_INCREF(op->func_qualname); + return op->func_qualname; +} +static int +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value) +{ + PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + if (unlikely(value == NULL || !PyUnicode_Check(value))) { +#else + if (unlikely(value == NULL || !PyString_Check(value))) { +#endif + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + tmp = op->func_qualname; + Py_INCREF(value); + op->func_qualname = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_self(__pyx_CyFunctionObject *m, CYTHON_UNUSED void *closure) +{ + PyObject *self; + self = m->func_closure; + if (self == NULL) + self = Py_None; + Py_INCREF(self); + return self; +} +static PyObject * +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_dict == NULL)) { + op->func_dict = PyDict_New(); + if (unlikely(op->func_dict == NULL)) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; +} +static int +__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value) +{ + PyObject *tmp; + if (unlikely(value == NULL)) { + PyErr_SetString(PyExc_TypeError, + "function's dictionary may not be deleted"); + return -1; + } + if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "setting function's dictionary to a non-dict"); + return -1; + } + tmp = op->func_dict; + Py_INCREF(value); + op->func_dict = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op) +{ + Py_INCREF(op->func_globals); + return op->func_globals; +} +static PyObject * +__Pyx_CyFunction_get_closure(CYTHON_UNUSED __pyx_CyFunctionObject *op) +{ + Py_INCREF(Py_None); + return Py_None; +} +static PyObject * +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op) +{ + PyObject* result = (op->func_code) ? op->func_code : Py_None; + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { + int result = 0; + PyObject *res = op->defaults_getter((PyObject *) op); + if (unlikely(!res)) + return -1; + #if CYTHON_COMPILING_IN_CPYTHON + op->defaults_tuple = PyTuple_GET_ITEM(res, 0); + Py_INCREF(op->defaults_tuple); + op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); + Py_INCREF(op->defaults_kwdict); + #else + op->defaults_tuple = PySequence_ITEM(res, 0); + if (unlikely(!op->defaults_tuple)) result = -1; + else { + op->defaults_kwdict = PySequence_ITEM(res, 1); + if (unlikely(!op->defaults_kwdict)) result = -1; + } + #endif + Py_DECREF(res); + return result; +} +static int +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value) { + PyObject* tmp; + if (!value) { + value = Py_None; + } else if (value != Py_None && !PyTuple_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "__defaults__ must be set to a tuple object"); + return -1; + } + Py_INCREF(value); + tmp = op->defaults_tuple; + op->defaults_tuple = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_tuple; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + result = op->defaults_tuple; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value) { + PyObject* tmp; + if (!value) { + value = Py_None; + } else if (value != Py_None && !PyDict_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults__ must be set to a dict object"); + return -1; + } + Py_INCREF(value); + tmp = op->defaults_kwdict; + op->defaults_kwdict = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_kwdict; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + result = op->defaults_kwdict; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value) { + PyObject* tmp; + if (!value || value == Py_None) { + value = NULL; + } else if (!PyDict_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "__annotations__ must be set to a dict object"); + return -1; + } + Py_XINCREF(value); + tmp = op->func_annotations; + op->func_annotations = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op) { + PyObject* result = op->func_annotations; + if (unlikely(!result)) { + result = PyDict_New(); + if (unlikely(!result)) return NULL; + op->func_annotations = result; + } + Py_INCREF(result); + return result; +} +static PyGetSetDef __pyx_CyFunction_getsets[] = { + {(char *) "func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {(char *) "__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {(char *) "func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {(char *) "__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {(char *) "__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, + {(char *) "__self__", (getter)__Pyx_CyFunction_get_self, 0, 0, 0}, + {(char *) "func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {(char *) "__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {(char *) "func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {(char *) "__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {(char *) "func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {(char *) "__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {(char *) "func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {(char *) "__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {(char *) "func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {(char *) "__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {(char *) "__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, + {(char *) "__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {0, 0, 0, 0, 0} +}; +static PyMemberDef __pyx_CyFunction_members[] = { + {(char *) "__module__", T_OBJECT, offsetof(__pyx_CyFunctionObject, func.m_module), PY_WRITE_RESTRICTED, 0}, + {0, 0, 0, 0, 0} +}; +static PyObject * +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, CYTHON_UNUSED PyObject *args) +{ +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_FromString(m->func.m_ml->ml_name); +#else + return PyString_FromString(m->func.m_ml->ml_name); +#endif +} +static PyMethodDef __pyx_CyFunction_methods[] = { + {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} +}; +#if PY_VERSION_HEX < 0x030500A0 +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) +#else +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func.m_weakreflist) +#endif +static PyObject *__Pyx_CyFunction_New(PyTypeObject *type, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + __pyx_CyFunctionObject *op = PyObject_GC_New(__pyx_CyFunctionObject, type); + if (op == NULL) + return NULL; + op->flags = flags; + __Pyx_CyFunction_weakreflist(op) = NULL; + op->func.m_ml = ml; + op->func.m_self = (PyObject *) op; + Py_XINCREF(closure); + op->func_closure = closure; + Py_XINCREF(module); + op->func.m_module = module; + op->func_dict = NULL; + op->func_name = NULL; + Py_INCREF(qualname); + op->func_qualname = qualname; + op->func_doc = NULL; + op->func_classobj = NULL; + op->func_globals = globals; + Py_INCREF(op->func_globals); + Py_XINCREF(code); + op->func_code = code; + op->defaults_pyobjects = 0; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_kwdict = NULL; + op->defaults_getter = NULL; + op->func_annotations = NULL; + PyObject_GC_Track(op); + return (PyObject *) op; +} +static int +__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) +{ + Py_CLEAR(m->func_closure); + Py_CLEAR(m->func.m_module); + Py_CLEAR(m->func_dict); + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_qualname); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_globals); + Py_CLEAR(m->func_code); + Py_CLEAR(m->func_classobj); + Py_CLEAR(m->defaults_tuple); + Py_CLEAR(m->defaults_kwdict); + Py_CLEAR(m->func_annotations); + if (m->defaults) { + PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); + int i; + for (i = 0; i < m->defaults_pyobjects; i++) + Py_XDECREF(pydefaults[i]); + PyObject_Free(m->defaults); + m->defaults = NULL; + } + return 0; +} +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + if (__Pyx_CyFunction_weakreflist(m) != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + PyObject_GC_Del(m); +} +static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) +{ + Py_VISIT(m->func_closure); + Py_VISIT(m->func.m_module); + Py_VISIT(m->func_dict); + Py_VISIT(m->func_name); + Py_VISIT(m->func_qualname); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_globals); + Py_VISIT(m->func_code); + Py_VISIT(m->func_classobj); + Py_VISIT(m->defaults_tuple); + Py_VISIT(m->defaults_kwdict); + if (m->defaults) { + PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); + int i; + for (i = 0; i < m->defaults_pyobjects; i++) + Py_VISIT(pydefaults[i]); + } + return 0; +} +static PyObject *__Pyx_CyFunction_descr_get(PyObject *func, PyObject *obj, PyObject *type) +{ + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + if (m->flags & __Pyx_CYFUNCTION_STATICMETHOD) { + Py_INCREF(func); + return func; + } + if (m->flags & __Pyx_CYFUNCTION_CLASSMETHOD) { + if (type == NULL) + type = (PyObject *)(Py_TYPE(obj)); + return __Pyx_PyMethod_New(func, type, (PyObject *)(Py_TYPE(type))); + } + if (obj == Py_None) + obj = NULL; + return __Pyx_PyMethod_New(func, obj, type); +} +static PyObject* +__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) +{ +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_FromFormat("", + op->func_qualname, (void *)op); +#else + return PyString_FromFormat("", + PyString_AsString(op->func_qualname), (void *)op); +#endif +} +#if CYTHON_COMPILING_IN_PYPY +static PyObject * __Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyCFunctionObject* f = (PyCFunctionObject*)func; + PyCFunction meth = f->m_ml->ml_meth; + PyObject *self = f->m_self; + Py_ssize_t size; + switch (f->m_ml->ml_flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { + case METH_VARARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) + return (*meth)(self, arg); + break; + case METH_VARARGS | METH_KEYWORDS: + return (*(PyCFunctionWithKeywords)meth)(self, arg, kw); + case METH_NOARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { + size = PyTuple_GET_SIZE(arg); + if (likely(size == 0)) + return (*meth)(self, NULL); + PyErr_Format(PyExc_TypeError, + "%.200s() takes no arguments (%" CYTHON_FORMAT_SSIZE_T "d given)", + f->m_ml->ml_name, size); + return NULL; + } + break; + case METH_O: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { + size = PyTuple_GET_SIZE(arg); + if (likely(size == 1)) { + PyObject *result, *arg0 = PySequence_ITEM(arg, 0); + if (unlikely(!arg0)) return NULL; + result = (*meth)(self, arg0); + Py_DECREF(arg0); + return result; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes exactly one argument (%" CYTHON_FORMAT_SSIZE_T "d given)", + f->m_ml->ml_name, size); + return NULL; + } + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags in " + "__Pyx_CyFunction_Call. METH_OLDARGS is no " + "longer supported!"); + return NULL; + } + PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", + f->m_ml->ml_name); + return NULL; +} +#else +static PyObject * __Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + return PyCFunction_Call(func, arg, kw); +} +#endif +static PyTypeObject __pyx_CyFunctionType_type = { + PyVarObject_HEAD_INIT(0, 0) + "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, + (destructor) __Pyx_CyFunction_dealloc, + 0, + 0, + 0, +#if PY_MAJOR_VERSION < 3 + 0, +#else + 0, +#endif + (reprfunc) __Pyx_CyFunction_repr, + 0, + 0, + 0, + 0, + __Pyx_CyFunction_Call, + 0, + 0, + 0, + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + 0, + (traverseproc) __Pyx_CyFunction_traverse, + (inquiry) __Pyx_CyFunction_clear, + 0, +#if PY_VERSION_HEX < 0x030500A0 + offsetof(__pyx_CyFunctionObject, func_weakreflist), +#else + offsetof(PyCFunctionObject, m_weakreflist), +#endif + 0, + 0, + __pyx_CyFunction_methods, + __pyx_CyFunction_members, + __pyx_CyFunction_getsets, + 0, + 0, + __Pyx_CyFunction_descr_get, + 0, + offsetof(__pyx_CyFunctionObject, func_dict), + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, +#if PY_VERSION_HEX >= 0x030400a1 + 0, +#endif +}; +static int __pyx_CyFunction_init(void) { +#if !CYTHON_COMPILING_IN_PYPY + __pyx_CyFunctionType_type.tp_call = PyCFunction_Call; +#endif + __pyx_CyFunctionType = __Pyx_FetchCommonType(&__pyx_CyFunctionType_type); + if (__pyx_CyFunctionType == NULL) { + return -1; + } + return 0; +} +static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *func, size_t size, int pyobjects) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyObject_Malloc(size); + if (!m->defaults) + return PyErr_NoMemory(); + memset(m->defaults, 0, size); + m->defaults_pyobjects = pyobjects; + return m->defaults; +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_kwdict = dict; + Py_INCREF(dict); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->func_annotations = dict; + Py_INCREF(dict); +} + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} +#endif + +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { +#endif + PyObject *local_type, *local_value, *local_tb; +#if CYTHON_COMPILING_IN_CPYTHON + PyObject *tmp_type, *tmp_value, *tmp_tb; + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + #if PY_MAJOR_VERSION >= 3 + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; + } + #endif + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_COMPILING_IN_CPYTHON + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +} + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = *type; + tstate->exc_value = *value; + tstate->exc_traceback = *tb; + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); + PyErr_SetExcInfo(*type, *value, *tb); + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#endif + +#if CYTHON_COMPILING_IN_CPYTHON +static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { + if (op1 == op2) { + Py_RETURN_TRUE; + } + #if PY_MAJOR_VERSION < 3 + if (likely(PyInt_CheckExact(op1))) { + const long b = intval; + long a = PyInt_AS_LONG(op1); + if (a == b) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + } + #endif + #if CYTHON_USE_PYLONG_INTERNALS && PY_MAJOR_VERSION >= 3 + if (likely(PyLong_CheckExact(op1))) { + const long b = intval; + long a; + const digit* digits = ((PyLongObject*)op1)->ob_digit; + const Py_ssize_t size = Py_SIZE(op1); + if (likely(__Pyx_sst_abs(size) <= 1)) { + a = likely(size) ? digits[0] : 0; + if (size == -1) a = -a; + } else { + switch (size) { + case -2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case 2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = (long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case -3: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + a = -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case 3: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + a = (long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case -4: + if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + a = -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case 4: + if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + a = (long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + #if PyLong_SHIFT < 30 && PyLong_SHIFT != 15 + default: return PyLong_Type.tp_richcompare(op1, op2, Py_EQ); + #else + default: Py_RETURN_FALSE; + #endif + } + } + if (a == b) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + } + #endif + if (PyFloat_CheckExact(op1)) { + const long b = intval; + double a = PyFloat_AS_DOUBLE(op1); + if ((double)a == (double)b) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + } + return PyObject_RichCompare(op1, op2, Py_EQ); +} +#endif + +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { + PyObject *empty_list = 0; + PyObject *module = 0; + PyObject *global_dict = 0; + PyObject *empty_dict = 0; + PyObject *list; + #if PY_VERSION_HEX < 0x03030000 + PyObject *py_import; + py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s_import); + if (!py_import) + goto bad; + #endif + if (from_list) + list = from_list; + else { + empty_list = PyList_New(0); + if (!empty_list) + goto bad; + list = empty_list; + } + global_dict = PyModule_GetDict(__pyx_m); + if (!global_dict) + goto bad; + empty_dict = PyDict_New(); + if (!empty_dict) + goto bad; + { + #if PY_MAJOR_VERSION >= 3 + if (level == -1) { + if (strchr(__Pyx_MODULE_NAME, '.')) { + #if PY_VERSION_HEX < 0x03030000 + PyObject *py_level = PyInt_FromLong(1); + if (!py_level) + goto bad; + module = PyObject_CallFunctionObjArgs(py_import, + name, global_dict, empty_dict, list, py_level, NULL); + Py_DECREF(py_level); + #else + module = PyImport_ImportModuleLevelObject( + name, global_dict, empty_dict, list, 1); + #endif + if (!module) { + if (!PyErr_ExceptionMatches(PyExc_ImportError)) + goto bad; + PyErr_Clear(); + } + } + level = 0; + } + #endif + if (!module) { + #if PY_VERSION_HEX < 0x03030000 + PyObject *py_level = PyInt_FromLong(level); + if (!py_level) + goto bad; + module = PyObject_CallFunctionObjArgs(py_import, + name, global_dict, empty_dict, list, py_level, NULL); + Py_DECREF(py_level); + #else + module = PyImport_ImportModuleLevelObject( + name, global_dict, empty_dict, list, level); + #endif + } + } +bad: + #if PY_VERSION_HEX < 0x03030000 + Py_XDECREF(py_import); + #endif + Py_XDECREF(empty_list); + Py_XDECREF(empty_dict); + return module; +} + +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { + PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); + if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Format(PyExc_ImportError, + #if PY_MAJOR_VERSION < 3 + "cannot import name %.230s", PyString_AS_STRING(name)); + #else + "cannot import name %S", name); + #endif + } + return value; +} + +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static PyCodeObject *__pyx_find_code_object(int code_line) { + PyCodeObject* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { + return NULL; + } + code_object = __pyx_code_cache.entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = 64; + __pyx_code_cache.count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { + PyCodeObject* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_DECREF(tmp); + return; + } + if (__pyx_code_cache.count == __pyx_code_cache.max_count) { + int new_max = __pyx_code_cache.max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = new_max; + } + for (i=__pyx_code_cache.count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + __pyx_code_cache.count++; + Py_INCREF(code_object); +} + +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyObject *py_srcfile = 0; + PyObject *py_funcname = 0; + #if PY_MAJOR_VERSION < 3 + py_srcfile = PyString_FromString(filename); + #else + py_srcfile = PyUnicode_FromString(filename); + #endif + if (!py_srcfile) goto bad; + if (c_line) { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #else + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #endif + } + else { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromString(funcname); + #else + py_funcname = PyUnicode_FromString(funcname); + #endif + } + if (!py_funcname) goto bad; + py_code = __Pyx_PyCode_New( + 0, + 0, + 0, + 0, + 0, + __pyx_empty_bytes, /*PyObject *code,*/ + __pyx_empty_tuple, /*PyObject *consts,*/ + __pyx_empty_tuple, /*PyObject *names,*/ + __pyx_empty_tuple, /*PyObject *varnames,*/ + __pyx_empty_tuple, /*PyObject *freevars,*/ + __pyx_empty_tuple, /*PyObject *cellvars,*/ + py_srcfile, /*PyObject *filename,*/ + py_funcname, /*PyObject *name,*/ + py_line, + __pyx_empty_bytes /*PyObject *lnotab*/ + ); + Py_DECREF(py_srcfile); + Py_DECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_srcfile); + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + py_code = __pyx_find_code_object(c_line ? c_line : py_line); + if (!py_code) { + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) goto bad; + __pyx_insert_code_object(c_line ? c_line : py_line, py_code); + } + py_frame = PyFrame_New( + PyThreadState_GET(), /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + py_frame->f_lineno = py_line; + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} + +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { + const int neg_one = (int) -1, const_zero = (int) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + } + } else { + if (sizeof(int) <= sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(int), + little, !is_unsigned); + } +} + +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); + } +} + +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int64_t(int64_t value) { + const int64_t neg_one = (int64_t) -1, const_zero = (int64_t) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(int64_t) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(int64_t) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); + } else if (sizeof(int64_t) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); + } + } else { + if (sizeof(int64_t) <= sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(int64_t) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(int64_t), + little, !is_unsigned); + } +} + +static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { + const int neg_one = (int) -1, const_zero = (int) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(int) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (int) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(int) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) -digits[0]) + case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) + case -2: + if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } +#endif + if (sizeof(int) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + int val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (int) -1; + } + } else { + int val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (int) -1; + val = __Pyx_PyInt_As_int(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +static CYTHON_INLINE int64_t __Pyx_PyInt_As_int64_t(PyObject *x) { + const int64_t neg_one = (int64_t) -1, const_zero = (int64_t) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(int64_t) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(int64_t, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (int64_t) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int64_t) 0; + case 1: __PYX_VERIFY_RETURN_INT(int64_t, digit, digits[0]) + case 2: + if (8 * sizeof(int64_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int64_t) >= 2 * PyLong_SHIFT) { + return (int64_t) (((((int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(int64_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int64_t) >= 3 * PyLong_SHIFT) { + return (int64_t) (((((((int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(int64_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int64_t) >= 4 * PyLong_SHIFT) { + return (int64_t) (((((((((int64_t)digits[3]) << PyLong_SHIFT) | (int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int64_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(int64_t) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(int64_t, unsigned long, PyLong_AsUnsignedLong(x)) + } else if (sizeof(int64_t) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int64_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int64_t) 0; + case -1: __PYX_VERIFY_RETURN_INT(int64_t, sdigit, (sdigit) -digits[0]) + case 1: __PYX_VERIFY_RETURN_INT(int64_t, digit, +digits[0]) + case -2: + if (8 * sizeof(int64_t) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int64_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int64_t) - 1 > 2 * PyLong_SHIFT) { + return (int64_t) (((int64_t)-1)*(((((int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(int64_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int64_t) - 1 > 2 * PyLong_SHIFT) { + return (int64_t) ((((((int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(int64_t) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int64_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int64_t) - 1 > 3 * PyLong_SHIFT) { + return (int64_t) (((int64_t)-1)*(((((((int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(int64_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int64_t) - 1 > 3 * PyLong_SHIFT) { + return (int64_t) ((((((((int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(int64_t) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int64_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int64_t) - 1 > 4 * PyLong_SHIFT) { + return (int64_t) (((int64_t)-1)*(((((((((int64_t)digits[3]) << PyLong_SHIFT) | (int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(int64_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int64_t) - 1 > 4 * PyLong_SHIFT) { + return (int64_t) ((((((((((int64_t)digits[3]) << PyLong_SHIFT) | (int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + } +#endif + if (sizeof(int64_t) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(int64_t, long, PyLong_AsLong(x)) + } else if (sizeof(int64_t) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int64_t, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + int64_t val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (int64_t) -1; + } + } else { + int64_t val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (int64_t) -1; + val = __Pyx_PyInt_As_int64_t(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int64_t"); + return (int64_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int64_t"); + return (int64_t) -1; +} + +static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { + const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(size_t) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(size_t, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (size_t) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, digits[0]) + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 2 * PyLong_SHIFT) { + return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 3 * PyLong_SHIFT) { + return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 4 * PyLong_SHIFT) { + return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (size_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(size_t) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) + } else if (sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case -1: __PYX_VERIFY_RETURN_INT(size_t, sdigit, (sdigit) -digits[0]) + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, +digits[0]) + case -2: + if (8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + } +#endif + if (sizeof(size_t) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) + } else if (sizeof(size_t) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + size_t val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (size_t) -1; + } + } else { + size_t val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (size_t) -1; + val = __Pyx_PyInt_As_size_t(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to size_t"); + return (size_t) -1; +} + +static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(long) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (long) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case 1: __PYX_VERIFY_RETURN_INT(long, digit, digits[0]) + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 2 * PyLong_SHIFT) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 3 * PyLong_SHIFT) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 4 * PyLong_SHIFT) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(long) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) -digits[0]) + case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) + case -2: + if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } +#endif + if (sizeof(long) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + long val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (long) -1; + } + } else { + long val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (long) -1; + val = __Pyx_PyInt_As_long(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +static int __Pyx_check_binary_version(void) { + char ctversion[4], rtversion[4]; + PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); + PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); + if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compiletime version %s of module '%.100s' " + "does not match runtime version %s", + ctversion, __Pyx_MODULE_NAME, rtversion); + return PyErr_WarnEx(NULL, message, 1); + } + return 0; +} + +#ifndef __PYX_HAVE_RT_ImportModule +#define __PYX_HAVE_RT_ImportModule +static PyObject *__Pyx_ImportModule(const char *name) { + PyObject *py_name = 0; + PyObject *py_module = 0; + py_name = __Pyx_PyIdentifier_FromString(name); + if (!py_name) + goto bad; + py_module = PyImport_Import(py_name); + Py_DECREF(py_name); + return py_module; +bad: + Py_XDECREF(py_name); + return 0; +} +#endif + +#ifndef __PYX_HAVE_RT_ImportType +#define __PYX_HAVE_RT_ImportType +static PyTypeObject *__Pyx_ImportType(const char *module_name, const char *class_name, + size_t size, int strict) +{ + PyObject *py_module = 0; + PyObject *result = 0; + PyObject *py_name = 0; + char warning[200]; + Py_ssize_t basicsize; +#ifdef Py_LIMITED_API + PyObject *py_basicsize; +#endif + py_module = __Pyx_ImportModule(module_name); + if (!py_module) + goto bad; + py_name = __Pyx_PyIdentifier_FromString(class_name); + if (!py_name) + goto bad; + result = PyObject_GetAttr(py_module, py_name); + Py_DECREF(py_name); + py_name = 0; + Py_DECREF(py_module); + py_module = 0; + if (!result) + goto bad; + if (!PyType_Check(result)) { + PyErr_Format(PyExc_TypeError, + "%.200s.%.200s is not a type object", + module_name, class_name); + goto bad; + } +#ifndef Py_LIMITED_API + basicsize = ((PyTypeObject *)result)->tp_basicsize; +#else + py_basicsize = PyObject_GetAttrString(result, "__basicsize__"); + if (!py_basicsize) + goto bad; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = 0; + if (basicsize == (Py_ssize_t)-1 && PyErr_Occurred()) + goto bad; +#endif + if (!strict && (size_t)basicsize > size) { + PyOS_snprintf(warning, sizeof(warning), + "%s.%s size changed, may indicate binary incompatibility. Expected %zd, got %zd", + module_name, class_name, basicsize, size); + if (PyErr_WarnEx(NULL, warning, 0) < 0) goto bad; + } + else if ((size_t)basicsize != size) { + PyErr_Format(PyExc_ValueError, + "%.200s.%.200s has the wrong size, try recompiling. Expected %zd, got %zd", + module_name, class_name, basicsize, size); + goto bad; + } + return (PyTypeObject *)result; +bad: + Py_XDECREF(py_module); + Py_XDECREF(result); + return NULL; +} +#endif + +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { + while (t->p) { + #if PY_MAJOR_VERSION < 3 + if (t->is_unicode) { + *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); + } else if (t->intern) { + *t->p = PyString_InternFromString(t->s); + } else { + *t->p = PyString_FromStringAndSize(t->s, t->n - 1); + } + #else + if (t->is_unicode | t->is_str) { + if (t->intern) { + *t->p = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); + } else { + *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + #endif + if (!*t->p) + return -1; + ++t; + } + return 0; +} + +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); +} +static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if CYTHON_COMPILING_IN_CPYTHON && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) + if ( +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + __Pyx_sys_getdefaultencoding_not_ascii && +#endif + PyUnicode_Check(o)) { +#if PY_VERSION_HEX < 0x03030000 + char* defenc_c; + PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); + if (!defenc) return NULL; + defenc_c = PyBytes_AS_STRING(defenc); +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + { + char* end = defenc_c + PyBytes_GET_SIZE(defenc); + char* c; + for (c = defenc_c; c < end; c++) { + if ((unsigned char) (*c) >= 128) { + PyUnicode_AsASCIIString(o); + return NULL; + } + } + } +#endif + *length = PyBytes_GET_SIZE(defenc); + return defenc_c; +#else + if (__Pyx_PyUnicode_READY(o) == -1) return NULL; +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (PyUnicode_IS_ASCII(o)) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif + } else +#endif +#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) + if (PyByteArray_Check(o)) { + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); + } else +#endif + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { + PyNumberMethods *m; + const char *name = NULL; + PyObject *res = NULL; +#if PY_MAJOR_VERSION < 3 + if (PyInt_Check(x) || PyLong_Check(x)) +#else + if (PyLong_Check(x)) +#endif + return __Pyx_NewRef(x); + m = Py_TYPE(x)->tp_as_number; +#if PY_MAJOR_VERSION < 3 + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Int(x); + } + else if (m && m->nb_long) { + name = "long"; + res = PyNumber_Long(x); + } +#else + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Long(x); + } +#endif + if (res) { +#if PY_MAJOR_VERSION < 3 + if (!PyInt_Check(res) && !PyLong_Check(res)) { +#else + if (!PyLong_Check(res)) { +#endif + PyErr_Format(PyExc_TypeError, + "__%.4s__ returned non-%.4s (type %.200s)", + name, name, Py_TYPE(res)->tp_name); + Py_DECREF(res); + return NULL; + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_CheckExact(b))) { + if (sizeof(Py_ssize_t) >= sizeof(long)) + return PyInt_AS_LONG(b); + else + return PyInt_AsSsize_t(x); + } +#endif + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)b)->ob_digit; + const Py_ssize_t size = Py_SIZE(b); + if (likely(__Pyx_sst_abs(size) <= 1)) { + ival = likely(size) ? digits[0] : 0; + if (size == -1) ival = -ival; + return ival; + } else { + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyInt_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { + return PyInt_FromSize_t(ival); +} + + +#endif /* Py_PYTHON_H */ diff --git a/src/s3ql/deltadump.pyx b/src/s3ql/deltadump.pyx new file mode 100644 index 0000000..587a53b --- /dev/null +++ b/src/s3ql/deltadump.pyx @@ -0,0 +1,652 @@ +''' +deltadump.pyx - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright © 2008 Nikolaus Rath + +This program can be distributed under the terms of the GNU GPLv3. +''' + +# Analysis of Cython code not really working yet +#@PydevCodeAnalysisIgnore + +from cpython.long cimport PyLong_AsVoidPtr +from cpython.exc cimport PyErr_NoMemory +from libc.stdio cimport (FILE, const_char, const_void, fclose as fclose_c, + fwrite as fwrite_c, fread as fread_c, ftell) +from libc.string cimport strerror +from libc.errno cimport errno +from libc.stdlib cimport calloc as calloc_c, free as free_c +from libc.stdint cimport (int64_t, uint8_t, uint16_t, uint32_t, uint64_t) +from posix.unistd cimport dup, lseek, SEEK_SET + +# These import are not (yet?) in the Cython provided cpython module +cdef extern from *: + object PyUnicode_FromString(const_char *u) + +cdef extern from 'stdint.h' nogil: + int UINT8_MAX + int UINT16_MAX + int UINT32_MAX + int INT_MAX + +cdef extern from 'stdio.h' nogil: + FILE * fdopen(int fd, const_char * mode) + int fflush(FILE * stream) + int fileno(FILE * stream) + +cdef extern from 'endian_indep.h' nogil: + uint64_t htole64(uint64_t host_64bits) + uint64_t le64toh(uint64_t little_endian_64bits) + +cdef extern from 'sqlite3.h' nogil: + ctypedef int sqlite3 + ctypedef int sqlite3_stmt + ctypedef int64_t sqlite3_int64 + + const_char *sqlite3_errmsg(sqlite3*) + int sqlite3_prepare_v2(sqlite3 * db, + char * zSql, + int nByte, + sqlite3_stmt ** ppStmt, + char ** pzTail) + int sqlite3_step(sqlite3_stmt *) + sqlite3_int64 sqlite3_column_int64(sqlite3_stmt * , int iCol) + const_void * sqlite3_column_blob(sqlite3_stmt * , int iCol) + int sqlite3_column_bytes(sqlite3_stmt * , int iCol) + int sqlite3_bind_blob(sqlite3_stmt * , int iCol, const_void * , int n, void(*)(void *)) + int sqlite3_bind_int64(sqlite3_stmt * , int iCol, sqlite3_int64) + int sqlite3_reset(sqlite3_stmt * pStmt) + int sqlite3_finalize(sqlite3_stmt * pStmt) + int sqlite3_column_type(sqlite3_stmt * , int iCol) + double sqlite3_column_double(sqlite3_stmt * , int iCol) + int sqlite3_bind_double(sqlite3_stmt * , int, double) + const_char *sqlite3_compileoption_get(int N) + const_char *sqlite3_libversion() + int sqlite3_close(sqlite3*) + int sqlite3_open_v2(const_char *filename, sqlite3 **ppDb, + int flags, const_char *zVfs) + int sqlite3_extended_result_codes(sqlite3*, int onoff) + void SQLITE_TRANSIENT(void *) + + enum: + SQLITE_OK + SQLITE_DONE + SQLITE_ROW + SQLITE_NULL + SQLITE_OPEN_READWRITE + SQLITE_OPEN_READONLY + +from contextlib import ExitStack +import apsw +import os +from .logging import logging # Ensure use of custom logger class +import itertools +import sys + +log = logging.getLogger(__name__) + +# Column types +cdef int _INTEGER = 1 +cdef int _BLOB = 2 +cdef int _TIME = 3 + +# Make column types available as Python objects +INTEGER = _INTEGER +BLOB = _BLOB +TIME = _TIME + +# Integer length codes +cdef uint8_t INT8 = 127 +cdef uint8_t INT16 = 126 +cdef uint8_t INT32 = 125 +cdef uint8_t INT64 = 124 + +# Maximum size of BLOBs +MAX_BLOB_SIZE = 4096 + +# Scale factor from time floats to integers. 1e9 would give nanosecond +# resolution but introduces rounding errors, so we use 1 << 30 (which is +# ~1.074e9, i.e. we get a little more precision than nanoseconds). +cdef double time_scale = 1 << 30 + +cdef inline int fwrite(const_void * buf, size_t len_, FILE * fp) except -1: + '''Call libc's fwrite() and raise exception on failure''' + + if fwrite_c(buf, len_, 1, fp) != 1: + raise_from_errno(IOError) + return 0 + +cdef inline int fread(void * buf, size_t len_, FILE * fp) except -1: + '''Call libc's fread() and raise exception on failure''' + + if fread_c(buf, len_, 1, fp) != 1: + raise_from_errno(IOError) + return 0 + +cdef free(void * ptr): + '''Call libc.free() + + This is a Python wrapper, so that we can call free in e.g. + a lambda expression. + ''' + + free_c(ptr) + return None + +cdef int raise_from_errno(err_class=OSError) except -1: + '''Raise OSError for current errno value''' + + raise err_class(errno, PyUnicode_FromString(strerror(errno))) + +cdef int fclose(FILE * fp) except -1: + '''Call libc.fclose() and raise exception on failure''' + + cdef ssize_t off + + # Explicitly flush data that needs to be written. This is + # important, so that we can safely reposition the fd position + # below (which is necessary in case there is cached input data) + if fflush(fp) != 0: + raise_from_errno() + + # Reposition FD to position of FILE*, otherwise next read from FD will miss + # data currently in stream buffer. It seems that call to fflush() achieves + # the same thing, but this does not seem to be documented so we don't rely + # on it. + off = ftell(fp) + if off == -1: + raise_from_errno() + + if lseek(fileno(fp), off, SEEK_SET) != off: + raise_from_errno() + + if fclose_c(fp) != 0: + raise_from_errno() + + return 0 + +cdef void * calloc(size_t cnt, size_t size) except NULL: + '''Call libc.calloc and raise exception on failure''' + + cdef void * ptr + + ptr = calloc_c(cnt, size) + + if ptr is NULL: + PyErr_NoMemory() + + return ptr + + +cdef int SQLITE_CHECK_RC(int rc, int success, sqlite3* db) except -1: + '''Raise correct exception if *rc* != *success*''' + + if rc != success: + exc = apsw.exceptionfor(rc) + raise type(exc)(PyUnicode_FromString(sqlite3_errmsg(db))) + + return 0 + + +cdef int prep_columns(columns, int** col_types_p, int** col_args_p) except -1: + '''Allocate col_types and col_args, return number of columns + + Both arrays are allocated dynamically, caller has to ensure + that they're freed again. + ''' + cdef size_t col_count + cdef int *col_types + cdef int *col_args + + col_count = len(columns) # guaranteed positive + col_types = < int *> calloc(col_count, sizeof(int)) + col_args = < int *> calloc(col_count, sizeof(int)) + + # Initialize col_args and col_types + for i in range(col_count): + if columns[i][1] not in (BLOB, INTEGER, TIME): + raise ValueError("Invalid type for column %d" % i) + col_types[i] = columns[i][1] + + if len(columns[i]) == 3: + col_args[i] = columns[i][2] + else: + col_args[i] = 0 + + col_types_p[0] = col_types + col_args_p[0] = col_args + + # We can safely assume that this fits into an int + return col_count + +cdef FILE* dup_to_fp(fh, const_char* mode) except NULL: + '''Duplicate fd from *fh* and open as FILE*''' + + cdef int fd + + fd = dup(fh.fileno()) + if fd == -1: + raise_from_errno() + + fp = fdopen(fd, mode) + if fp == NULL: + raise_from_errno() + + return fp + +def check_sqlite(): + '''Check if deltadump and apsw module use compatible SQLite code. + + This functions look at versions and compile options of the SQLite + code used by the *apsw* module and the *deltadump* module. If they + do not match exactly, a `RuntimeError` is raised. + + Only if both modules use the same SQLite version compiled with the + same options can the database object be shared between *apsw* and + *deltadump*. + ''' + + cdef const_char *buf + + apsw_sqlite_version = apsw.sqlitelibversion() + s3ql_sqlite_version = PyUnicode_FromString(sqlite3_libversion()) + log.debug('apsw sqlite version: %s, ' + 's3ql sqlite version: %s', + apsw_sqlite_version, + s3ql_sqlite_version) + if apsw_sqlite_version != s3ql_sqlite_version: + raise RuntimeError('SQLite version mismatch between APSW and S3QL ' + '(%s vs %s)' % (apsw_sqlite_version, s3ql_sqlite_version)) + + apsw_sqlite_options = set(apsw.compile_options) + s3ql_sqlite_options = set() + for idx in itertools.count(0): + buf = sqlite3_compileoption_get(idx) + if buf is NULL: + break + s3ql_sqlite_options.add(PyUnicode_FromString(buf)) + + log.debug('apsw sqlite compile options: %s, ' + 's3ql sqlite compile options: %s', + apsw_sqlite_options, + s3ql_sqlite_options) + if apsw_sqlite_options != s3ql_sqlite_options: + raise RuntimeError('SQLite code used by APSW was compiled with different ' + 'options than SQLite code available to S3QL! ' + 'Differing settings: + %s, - %s' % + (apsw_sqlite_options - s3ql_sqlite_options, + s3ql_sqlite_options - apsw_sqlite_options)) + +def dump_table(table, order, columns, db, fh): + '''Dump *columns* of *table* into *fh* + + *order* specifies the order in which the rows are written and must be a + string that can be inserted after the "ORDER BY" clause in an SQL SELECT + statement. + + *db* is an `s3ql.Connection` instance for the database. + + *columns* must a list of 3-tuples, one for each column that should be + stored. The first element of the tuple must contain the column name and the + second element the type of data stored in the column (`INTEGER`, `TIME` + or `BLOB`). Times will be converted to nanosecond integers. + + For integers and seconds, the third tuple element specifies the expected + change of the values between rows. For blobs it can be either zero + (indicating variable length columns) or an integer specifying the length of + the column values in bytes. + + This function will open a separate connection to the database, so + the *db* connection should not be in EXCLUSIVE locking mode. + (Using a separate connection avoids the requirement on the *apsw* + and *deltadump* modules be linked against against binary + compatible SQLite libraries). + ''' + + cdef sqlite3 *sqlite3_db + cdef sqlite3_stmt *stmt + cdef int *col_types + cdef int *col_args + cdef int col_count, rc, i + cdef size_t len_ + cdef int64_t *int64_prev + cdef int64_t int64, tmp + cdef FILE *fp + cdef const_void *buf + cdef int64_t row_count + + if db.file == ':memory:': + raise ValueError("Can't access in-memory databases") + + with ExitStack() as cm: + # Get SQLite connection + log.debug('Opening connection to %s', db.file) + dbfile_b = db.file.encode(sys.getfilesystemencoding(), 'surrogateescape') + SQLITE_CHECK_RC(sqlite3_open_v2(dbfile_b, &sqlite3_db, + SQLITE_OPEN_READONLY, NULL), + SQLITE_OK, sqlite3_db) + cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), + SQLITE_OK, sqlite3_db)) + SQLITE_CHECK_RC(sqlite3_extended_result_codes(sqlite3_db, 1), + SQLITE_OK, sqlite3_db) + + # Get FILE* for buffered reading from *fh* + fp = dup_to_fp(fh, b'wb') + cm.callback(lambda: fclose(fp)) + + # Allocate col_args and col_types + col_count = prep_columns(columns, &col_types, &col_args) + cm.callback(lambda: free(col_args)) + cm.callback(lambda: free(col_types)) + + # Allocate int64_prev + int64_prev = calloc( len(columns), sizeof(int64_t)) + cm.callback(lambda: free(int64_prev)) + + # Prepare statement + col_names = [ x[0] for x in columns ] + query = ("SELECT %s FROM %s ORDER BY %s " % + (', '.join(col_names), table, order)).encode('utf-8') + SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + SQLITE_OK, sqlite3_db) + cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(stmt), + SQLITE_OK, sqlite3_db)) + + row_count = db.get_val("SELECT COUNT(rowid) FROM %s" % table) + log.debug('dump_table(%s): writing %d rows', table, row_count) + write_integer(row_count, fp) + + # Iterate through rows + while True: + rc = sqlite3_step(stmt) + if rc == SQLITE_DONE: + break + SQLITE_CHECK_RC(rc, SQLITE_ROW, sqlite3_db) + + for i in range(col_count): + if sqlite3_column_type(stmt, i) is SQLITE_NULL: + raise ValueError("Can't dump NULL values") + + if col_types[i] == _INTEGER: + int64 = sqlite3_column_int64(stmt, i) + tmp = int64 + int64 -= int64_prev[i] + col_args[i] + int64_prev[i] = tmp + write_integer(int64, fp) + + elif col_types[i] == _TIME: + int64 = (sqlite3_column_double(stmt, i) * time_scale) + tmp = int64 + int64 -= int64_prev[i] + col_args[i] + int64_prev[i] = tmp + write_integer(int64, fp) + + elif col_types[i] == _BLOB: + buf = sqlite3_column_blob(stmt, i) + rc = sqlite3_column_bytes(stmt, i) + if rc > MAX_BLOB_SIZE: + raise ValueError('Can not dump BLOB of size %d (max: %d)', + rc, MAX_BLOB_SIZE) + # Safe to cast now + len_ = rc + if col_args[i] == 0: + write_integer(rc - int64_prev[i], fp) + int64_prev[i] = rc + elif rc != col_args[i]: + raise ValueError("Length %d != %d in column %d" % (len_, col_args[i], i)) + + if len_ != 0: + fwrite(buf, len_, fp) + +def load_table(table, columns, db, fh, trx_rows=5000): + '''Load *columns* of *table* from *fh* + + *db* is an `s3ql.Connection` instance for the database. + + *columns* must be the same list of 3-tuples that was passed to + `dump_table` when creating the dump stored in *fh*. + + This function will open a separate connection to the database, so + the *db* connection should not be in EXCLUSIVE locking mode. + (Using a separate connection avoids the requirement on the *apsw* + and *deltadump* modules be linked against against binary + compatible SQLite libraries). + + When writing into the table, a new transaction will be started + every *trx_rows* rows. + ''' + + cdef sqlite3 *sqlite3_db + cdef sqlite3_stmt *stmt + cdef sqlite3_stmt *begin_stmt + cdef sqlite3_stmt *commit_stmt + cdef int *col_types + cdef int *col_args + cdef int col_count, rc, len_, i, j + cdef int64_t *int64_prev + cdef FILE *fp + cdef void *buf + cdef int64_t row_count, int64, tmp + + if db.file == ':memory:': + raise ValueError("Can't access in-memory databases") + + with ExitStack() as cm: + # Get SQLite connection + log.debug('Opening connection to %s', db.file) + dbfile_b = db.file.encode(sys.getfilesystemencoding(), 'surrogateescape') + SQLITE_CHECK_RC(sqlite3_open_v2(dbfile_b, &sqlite3_db, + SQLITE_OPEN_READWRITE, NULL), + SQLITE_OK, sqlite3_db) + cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), + SQLITE_OK, sqlite3_db)) + SQLITE_CHECK_RC(sqlite3_extended_result_codes(sqlite3_db, 1), + SQLITE_OK, sqlite3_db) + + # Copy settings + for pragma in ('synchronous', 'foreign_keys'): + val = db.get_val('PRAGMA %s' % pragma) + cmd = ('PRAGMA %s = %s' % (pragma, val)).encode('utf-8') + SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, cmd, -1, &stmt, NULL), + SQLITE_OK, sqlite3_db) + try: + rc = sqlite3_step(stmt) + if rc == SQLITE_ROW: + rc = sqlite3_step(stmt) + SQLITE_CHECK_RC(rc, SQLITE_DONE, sqlite3_db) + finally: + SQLITE_CHECK_RC(sqlite3_finalize(stmt), SQLITE_OK, sqlite3_db) + + # Get FILE* for buffered reading from *fh* + fp = dup_to_fp(fh, b'rb') + cm.callback(lambda: fclose(fp)) + + # Allocate col_args and col_types + col_count = prep_columns(columns, &col_types, &col_args) + cm.callback(lambda: free(col_args)) + cm.callback(lambda: free(col_types)) + + # Allocate int64_prev + int64_prev = calloc( len(columns), sizeof(int64_t)) + cm.callback(lambda: free(int64_prev)) + + # Prepare INSERT statement + col_names = [ x[0] for x in columns ] + query = ("INSERT INTO %s (%s) VALUES(%s)" + % (table, ', '.join(col_names), + ', '.join('?' * col_count))).encode('utf-8') + SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), + SQLITE_OK, sqlite3_db) + cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(stmt), + SQLITE_OK, sqlite3_db)) + + # Prepare BEGIN statement + query = b'BEGIN TRANSACTION' + SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &begin_stmt, NULL), + SQLITE_OK, sqlite3_db) + cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(begin_stmt), + SQLITE_OK, sqlite3_db)) + + # Prepare COMMIT statement + query = b'COMMIT TRANSACTION' + SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &commit_stmt, NULL), + SQLITE_OK, sqlite3_db) + cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(commit_stmt), + SQLITE_OK, sqlite3_db)) + + buf = calloc(MAX_BLOB_SIZE, 1) + cm.callback(lambda: free(buf)) + read_integer(&row_count, fp) + log.debug('load_table(%s): reading %d rows', table, row_count) + + # Start transaction + SQLITE_CHECK_RC(sqlite3_step(begin_stmt), SQLITE_DONE, sqlite3_db) + cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_step(commit_stmt), + SQLITE_DONE, sqlite3_db)) + SQLITE_CHECK_RC(sqlite3_reset(begin_stmt), SQLITE_OK, sqlite3_db) + + # Iterate through rows + for i in range(row_count): + for j in range(col_count): + if col_types[j] == _INTEGER: + read_integer(&int64, fp) + int64 += col_args[j] + int64_prev[j] + int64_prev[j] = int64 + SQLITE_CHECK_RC(sqlite3_bind_int64(stmt, j + 1, int64), + SQLITE_OK, sqlite3_db) + + if col_types[j] == _TIME: + read_integer(&int64, fp) + int64 += col_args[j] + int64_prev[j] + int64_prev[j] = int64 + # Cast is safe, we know that the integer was converted from + # double at dump time. + SQLITE_CHECK_RC(sqlite3_bind_double(stmt, j + 1, int64 / time_scale), + SQLITE_OK, sqlite3_db) + + elif col_types[j] == _BLOB: + if col_args[j] == 0: + read_integer(&int64, fp) + tmp = int64_prev[j] + int64 + if tmp < 0 or tmp > INT_MAX: + raise RuntimeError('Corrupted input') + len_ = tmp + int64_prev[j] = tmp + else: + len_ = col_args[j] + + if len_ > MAX_BLOB_SIZE: + raise RuntimeError('BLOB too large to read (%d vs %d)', len_, MAX_BLOB_SIZE) + + if len_ > 0: + fread(buf, len_, fp) + + SQLITE_CHECK_RC(sqlite3_bind_blob(stmt, j + 1, buf, len_, SQLITE_TRANSIENT), + SQLITE_OK, sqlite3_db) + + SQLITE_CHECK_RC(sqlite3_step(stmt), SQLITE_DONE, sqlite3_db) + SQLITE_CHECK_RC(sqlite3_reset(stmt), SQLITE_OK, sqlite3_db) + + # Commit every once in a while + if i % trx_rows == 0: + # This isn't 100% ok -- if we have an exception in step(begin_stmt), + # we the cleanup handler will execute the commit statement again + # without an active transaction. + SQLITE_CHECK_RC(sqlite3_step(commit_stmt), SQLITE_DONE, sqlite3_db) + SQLITE_CHECK_RC(sqlite3_step(begin_stmt), SQLITE_DONE, sqlite3_db) + SQLITE_CHECK_RC(sqlite3_reset(commit_stmt), SQLITE_OK, sqlite3_db) + SQLITE_CHECK_RC(sqlite3_reset(begin_stmt), SQLITE_OK, sqlite3_db) + +cdef inline int write_integer(int64_t int64, FILE * fp) except -1: + '''Write *int64* into *fp*, using as little space as possible + + Return the number of bytes written, or -1 on error. + ''' + + # This is meant to be a `uint8_t`. However, due to integer promotion + # any expression always has at least type `int`. So we would need an + # explicit cast for every assignment to this value. It's easier to + # declare it as `int` instead, and just cast once at the end. See also + # https://stackoverflow.com/questions/32574514/ + cdef unsigned int8 + cdef uint8_t int8_real + cdef size_t len_ + cdef uint64_t uint64 + + if int64 < 0: + uint64 = -int64 + int8 = 0x80 # Highest bit set + else: + uint64 = int64 + int8 = 0 + + if uint64 < 0x80 and uint64 not in (INT8, INT16, INT32, INT64): + len_ = 0 + int8 += uint64 + elif uint64 < UINT8_MAX: + len_ = 1 + int8 += INT8 + elif uint64 < UINT16_MAX: + len_ = 2 + int8 += INT16 + elif uint64 < UINT32_MAX: + len_ = 4 + int8 += INT32 + else: + len_ = 8 + int8 += INT64 + + # Cast + int8_real = int8 + fwrite(&int8_real, 1, fp) + if len_ != 0: + uint64 = htole64(uint64) + fwrite(&uint64, len_, fp) + + # len <= 8, safe to cast + return len_ + 1 + +cdef inline int read_integer(int64_t * out, FILE * fp) except -1: + '''Read integer written using `write_integer` from *fp* + + Return the number of bytes read, or -1 on error. + ''' + + cdef uint8_t int8 + cdef size_t len_ + cdef uint64_t uint64 + cdef char negative + + fread(&int8, 1, fp) + + if int8 & 0x80 != 0: + negative = 1 + # Need to cast again due to integer promotion + int8 = (int8 & (~ 0x80)) + else: + negative = 0 + + if int8 == INT8: + len_ = 1 + elif int8 == INT16: + len_ = 2 + elif int8 == INT32: + len_ = 4 + elif int8 == INT64: + len_ = 8 + else: + len_ = 0 + uint64 = int8 + + if len_ != 0: + uint64 = 0 + fread(&uint64, len_, fp) + uint64 = le64toh(uint64) + + if negative == 1: + out[0] = - < int64_t > uint64 + else: + out[0] = < int64_t > uint64 + + # len <= 8, safe to cast + return len_ + 1 diff --git a/src/s3ql/endian_indep.h b/src/s3ql/endian_indep.h new file mode 100644 index 0000000..91283fc --- /dev/null +++ b/src/s3ql/endian_indep.h @@ -0,0 +1,37 @@ +/* +endian_indep.h - this file is part of S3QL (http://s3ql.googlecode.com) + +Copyright (C) Nikolaus Rath + +This program can be distributed under the terms of the GNU GPLv3. +*/ + +#ifndef __ENDIAN_INDEP_H__ +#define __ENDIAN_INDEP_H__ + +#ifdef __APPLE__ +#include + +#define htobe16(x) OSSwapHostToBigInt16(x) +#define htole16(x) OSSwapHostToLittleInt16(x) +#define be16toh(x) OSSwapBigToHostInt16(x) +#define le16toh(x) OSSwapLittleToHostInt16(x) + +#define htobe32(x) OSSwapHostToBigInt32(x) +#define htole32(x) OSSwapHostToLittleInt32(x) +#define be32toh(x) OSSwapBigToHostInt32(x) +#define le32toh(x) OSSwapLittleToHostInt32(x) + +#define htobe64(x) OSSwapHostToBigInt64(x) +#define htole64(x) OSSwapHostToLittleInt64(x) +#define be64toh(x) OSSwapBigToHostInt64(x) +#define le64toh(x) OSSwapLittleToHostInt64(x) + +#elif __FreeBSD__ +#include + +#else +#include +#endif + +#endif /* __ENDIAN_INDEP_H__ */ diff --git a/src/s3ql/exit_stack.py b/src/s3ql/exit_stack.py new file mode 100644 index 0000000..228ca25 --- /dev/null +++ b/src/s3ql/exit_stack.py @@ -0,0 +1,70 @@ +''' +exit_stack.py - this file is part of S3QL. + +Copyright © 2013 Nikolaus Rath +Copyright © 2013 the Python Software Foundation. + +This module provides the ExitStack class. + +For Python versions 3.3.3 or newer, it simply exports the vanilla version from +contextlib. + +For Python 3.3.0 - 3.3.2, it provides a derived class that overwrites the +__exit__ method with the version from CPython revision 423736775f6b to fix +Python issue 19092 (cf. http://bugs.python.org/issue19092). + +This work can be distributed under the terms of the GNU GPLv3. +''' + +import sys +from contextlib import ExitStack as _ExitStack + +if sys.version_info < (3,3): + raise RuntimeError("Unsupported Python version: %s" % sys.version_info) + +if sys.version_info >= (3,3,3): + ExitStack = _ExitStack + +else: + class ExitStack(_ExitStack): + def __exit__(self, *exc_details): + received_exc = exc_details[0] is not None + + # We manipulate the exception state so it behaves as though + # we were actually nesting multiple with statements + frame_exc = sys.exc_info()[1] + def _fix_exception_context(new_exc, old_exc): + while 1: + exc_context = new_exc.__context__ + if exc_context in (None, frame_exc): + break + new_exc = exc_context + new_exc.__context__ = old_exc + + # Callbacks are invoked in LIFO order to match the behaviour of + # nested context managers + suppressed_exc = False + pending_raise = False + while self._exit_callbacks: + cb = self._exit_callbacks.pop() + try: + if cb(*exc_details): + suppressed_exc = True + pending_raise = False + exc_details = (None, None, None) + except: + new_exc_details = sys.exc_info() + # simulate the stack of exceptions by setting the context + _fix_exception_context(new_exc_details[1], exc_details[1]) + pending_raise = True + exc_details = new_exc_details + if pending_raise: + try: + # bare "raise exc_details[1]" replaces our carefully + # set-up context + fixed_ctx = exc_details[1].__context__ + raise exc_details[1] + except BaseException: + exc_details[1].__context__ = fixed_ctx + raise + return received_exc and suppressed_exc diff --git a/src/s3ql/fs.py b/src/s3ql/fs.py new file mode 100644 index 0000000..1fddd94 --- /dev/null +++ b/src/s3ql/fs.py @@ -0,0 +1,1212 @@ +''' +fs.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging +from . import deltadump, CTRL_NAME, CTRL_INODE +from .backends.common import NoSuchObject, CorruptedObjectError +from .common import get_path, parse_literal, time_ns +from .database import NoSuchRowError +from .inode_cache import OutOfInodesError +from io import BytesIO +from llfuse import FUSEError +import collections +import errno +import llfuse +import math +import os +import stat +import struct +import time + +# We work in bytes +CTRL_NAME = CTRL_NAME.encode('us-ascii') + +# standard logger for this module +log = logging.getLogger(__name__) + +# For long requests, we force a GIL release in the following interval +GIL_RELEASE_INTERVAL = 0.05 + +# ACL_ERRNO is the error code returned for requests that try +# to modify or access extendeda attributes associated with ACL. +# Since we currently don't know how to keep these in sync +# with permission bits, we cannot support ACLs despite having +# full support for extended attributes. +# +# What errno to use for ACL_ERRNO is a bit tricky. acl_set_fd(3) +# returns ENOTSUP if the file system does not support ACLs. However, +# this function uses the setxattr() syscall internally, and +# setxattr(3) states that ENOTSUP means that the file system does not +# support extended attributes at all. A test with btrfs mounted with +# -o noacl shows that the actual errno returned by setxattr() is +# EOPNOTSUPP. Also, some Python versions do not know about +# errno.ENOTSUPP and errno(3) says that on Linux, EOPNOTSUPP and ENOTSUP +# have the same value (despite this violating POSIX). +# +# All in all, the situation seems complicated, so we try to use +# EOPNOTSUPP with a fallback on ENOTSUP just in case. + +if not hasattr(errno, 'EOPNOTSUPP'): + ACL_ERRNO = errno.ENOTSUP +else: + ACL_ERRNO = errno.EOPNOTSUPP + +class Operations(llfuse.Operations): + """A full-featured file system for online data storage + + This class implements low-level FUSE operations and is meant to be passed to + llfuse.init(). + + The ``access`` method of this class always gives full access, independent of + file permissions. If the FUSE library is initialized with ``allow_other`` or + ``allow_root``, the ``default_permissions`` option should therefore always + be passed as well. + + + Attributes: + ----------- + + :cache: Holds information about cached blocks + :inode_cache: A cache for the attributes of the currently opened inodes. + :open_inodes: dict of currently opened inodes. This is used to not remove + the blocks of unlinked inodes that are still open. + :upload_event: If set, triggers a metadata upload + :failsafe: Set when backend problems are encountered. In that case, fs only + allows read access. + :broken_blocks: Caches information about corrupted blocks to avoid repeated (pointless) + attempts to retrieve them. This attribute is a dict (indexed by inodes) + of sets of block indices. Broken blocks are removed from the cache + when an inode is forgotten. + + Multithreading + -------------- + + All methods are reentrant and may release the global lock while they + are running. + + + Directory Entry Types + ---------------------- + + S3QL is quite agnostic when it comes to directory entry types. Every + directory entry can contain other entries *and* have a associated data, + size, link target and device number. However, S3QL makes some provisions for + users relying on unlink()/rmdir() to fail for a directory/file. For that, it + explicitly checks the st_mode attribute. + """ + + def __init__(self, block_cache, db, max_obj_size, inode_cache, + upload_event=None): + super().__init__() + + self.inodes = inode_cache + self.db = db + self.upload_event = upload_event + self.open_inodes = collections.defaultdict(lambda: 0) + self.max_obj_size = max_obj_size + self.cache = block_cache + self.failsafe = False + self.broken_blocks = collections.defaultdict(set) + + # Root inode is always open + self.open_inodes[llfuse.ROOT_INODE] += 1 + + def destroy(self): + self.forget(list(self.open_inodes.items())) + self.inodes.destroy() + + def lookup(self, id_p, name, ctx): + return self._lookup(id_p, name, ctx).entry_attributes() + + def _lookup(self, id_p, name, ctx): + log.debug('started with %d, %r', id_p, name) + + if name == CTRL_NAME: + inode = self.inodes[CTRL_INODE] + + # Make sure the control file is only writable by the user + # who mounted the file system (but don't mark inode as dirty) + object.__setattr__(inode, 'uid', os.getuid()) + object.__setattr__(inode, 'gid', os.getgid()) + + elif name == '.': + inode = self.inodes[id_p] + + elif name == '..': + id_ = self.db.get_val("SELECT parent_inode FROM contents WHERE inode=?", + (id_p,)) + inode = self.inodes[id_] + + else: + try: + id_ = self.db.get_val("SELECT inode FROM contents_v WHERE name=? AND parent_inode=?", + (name, id_p)) + except NoSuchRowError: + raise llfuse.FUSEError(errno.ENOENT) + inode = self.inodes[id_] + + self.open_inodes[inode.id] += 1 + return inode + + def getattr(self, id_, ctx): + log.debug('started with %d', id_) + if id_ == CTRL_INODE: + # Make sure the control file is only writable by the user + # who mounted the file system (but don't mark inode as dirty) + inode = self.inodes[CTRL_INODE] + object.__setattr__(inode, 'uid', os.getuid()) + object.__setattr__(inode, 'gid', os.getgid()) + return inode.entry_attributes() + + return self.inodes[id_].entry_attributes() + + def readlink(self, id_, ctx): + log.debug('started with %d', id_) + now_ns = time_ns() + inode = self.inodes[id_] + if inode.atime_ns < inode.ctime_ns or inode.atime_ns < inode.mtime_ns: + inode.atime_ns = now_ns + try: + return self.db.get_val("SELECT target FROM symlink_targets WHERE inode=?", (id_,)) + except NoSuchRowError: + log.warning('Inode does not have symlink target: %d', id_) + raise FUSEError(errno.EINVAL) + + def opendir(self, id_, ctx): + log.debug('started with %d', id_) + return id_ + + def readdir(self, id_, off): + log.debug('started with %d, %d', id_, off) + if off == 0: + off = -1 + + inode = self.inodes[id_] + if inode.atime_ns < inode.ctime_ns or inode.atime_ns < inode.mtime_ns: + inode.atime_ns = time_ns() + + # NFS treats offsets 1 and 2 special, so we have to exclude + # them. + with self.db.query("SELECT name_id, name, inode FROM contents_v " + 'WHERE parent_inode=? AND name_id > ? ORDER BY name_id', + (id_, off-3)) as res: + for (next_, name, cid_) in res: + yield (name, self.inodes[cid_].entry_attributes(), next_+3) + + def getxattr(self, id_, name, ctx): + log.debug('started with %d, %r', id_, name) + # Handle S3QL commands + if id_ == CTRL_INODE: + if name == b's3ql_pid?': + return ('%d' % os.getpid()).encode() + + elif name == b's3qlstat': + return self.extstat() + + raise llfuse.FUSEError(errno.EINVAL) + + # http://code.google.com/p/s3ql/issues/detail?id=385 + elif name in (b'system.posix_acl_access', + b'system.posix_acl_default'): + raise FUSEError(ACL_ERRNO) + + else: + try: + value = self.db.get_val('SELECT value FROM ext_attributes_v WHERE inode=? AND name=?', + (id_, name)) + except NoSuchRowError: + raise llfuse.FUSEError(llfuse.ENOATTR) + return value + + def listxattr(self, id_, ctx): + log.debug('started with %d', id_) + names = list() + with self.db.query('SELECT name FROM ext_attributes_v WHERE inode=?', (id_,)) as res: + for (name,) in res: + names.append(name) + return names + + def setxattr(self, id_, name, value, ctx): + log.debug('started with %d, %r, %r', id_, name, value) + + # Handle S3QL commands + if id_ == CTRL_INODE: + if name == b's3ql_flushcache!': + self.cache.clear() + + elif name == b'copy': + try: + tup = parse_literal(value, (int, int)) + except ValueError: + log.warning('Received malformed command via control inode') + raise FUSEError.EINVAL() + self.copy_tree(*tup) + + elif name == b'upload-meta': + if self.upload_event is not None: + self.upload_event.set() + else: + raise llfuse.FUSEError(errno.ENOTTY) + + elif name == b'lock': + try: + id_ = parse_literal(value, int) + except ValueError: + log.warning('Received malformed command via control inode') + raise FUSEError.EINVAL() + self.lock_tree(id_) + + elif name == b'rmtree': + try: + tup = parse_literal(value, (int, bytes)) + except ValueError: + log.warning('Received malformed command via control inode') + raise FUSEError.EINVAL() + self.remove_tree(*tup) + + elif name == b'logging': + try: + (lvl, modules)= parse_literal(value, (int, str)) + except (ValueError, KeyError): + log.warning('Received malformed command via control inode') + raise FUSEError.EINVAL() + update_logging(lvl, modules.split(',') if modules else None) + + elif name == b'cachesize': + try: + self.cache.cache.max_size = parse_literal(value, int) + except ValueError: + log.warning('Received malformed command via control inode') + raise FUSEError.EINVAL() + log.debug('updated cache size to %d bytes', self.cache.cache.max_size) + + else: + log.warning('Received unknown command via control inode') + raise llfuse.FUSEError(errno.EINVAL) + + # http://code.google.com/p/s3ql/issues/detail?id=385 + elif name in (b'system.posix_acl_access', + b'system.posix_acl_default'): + raise FUSEError(ACL_ERRNO) + + else: + if self.failsafe or self.inodes[id_].locked: + raise FUSEError(errno.EPERM) + + if len(value) > deltadump.MAX_BLOB_SIZE: + raise FUSEError(errno.EINVAL) + + self.db.execute('INSERT OR REPLACE INTO ext_attributes (inode, name_id, value) ' + 'VALUES(?, ?, ?)', (id_, self._add_name(name), value)) + self.inodes[id_].ctime_ns = time_ns() + + def removexattr(self, id_, name, ctx): + log.debug('started with %d, %r', id_, name) + + if self.failsafe or self.inodes[id_].locked: + raise FUSEError(errno.EPERM) + + try: + name_id = self._del_name(name) + except NoSuchRowError: + raise llfuse.FUSEError(llfuse.ENOATTR) + + changes = self.db.execute('DELETE FROM ext_attributes WHERE inode=? AND name_id=?', + (id_, name_id)) + if changes == 0: + raise llfuse.FUSEError(llfuse.ENOATTR) + + self.inodes[id_].ctime_ns = time_ns() + + def lock_tree(self, id0): + '''Lock directory tree''' + + if self.failsafe: + raise FUSEError(errno.EPERM) + + log.debug('started with %d', id0) + queue = [ id0 ] + self.inodes[id0].locked = True + processed = 0 # Number of steps since last GIL release + stamp = time_ns() # Time of last GIL release + gil_step = 250 # Approx. number of steps between GIL releases + while True: + id_p = queue.pop() + with self.db.query('SELECT inode FROM contents WHERE parent_inode=?', + (id_p,)) as res: + for (id_,) in res: + self.inodes[id_].locked = True + processed += 1 + + if self.db.has_val('SELECT 1 FROM contents WHERE parent_inode=?', (id_,)): + queue.append(id_) + + if not queue: + break + + if processed > gil_step: + dt = time.time() - stamp + gil_step = max(int(gil_step * GIL_RELEASE_INTERVAL / dt), 250) + log.debug('Adjusting gil_step to %d', gil_step) + processed = 0 + llfuse.lock.yield_(100) + log.debug('re-acquired lock') + stamp = time.time() + + log.debug('finished') + + def remove_tree(self, id_p0, name0): + '''Remove directory tree''' + + if self.failsafe: + raise FUSEError(errno.EPERM) + + log.debug('started with %d, %s', id_p0, name0) + + if self.inodes[id_p0].locked: + raise FUSEError(errno.EPERM) + + id0 = self._lookup(id_p0, name0, ctx=None).id + queue = [ id0 ] # Directories that we still need to delete + processed = 0 # Number of steps since last GIL release + stamp = time.time() # Time of last GIL release + gil_step = 250 # Approx. number of steps between GIL releases + while queue: # For every directory + found_subdirs = False # Does current directory have subdirectories? + id_p = queue.pop() + if id_p in self.open_inodes: + inval_entry = lambda x: llfuse.invalidate_entry(id_p, x) + else: + inval_entry = lambda x: None + + with self.db.query('SELECT name_id, inode FROM contents WHERE ' + 'parent_inode=?', (id_p,)) as res: + for (name_id, id_) in res: + + if self.db.has_val('SELECT 1 FROM contents WHERE parent_inode=?', (id_,)): + if not found_subdirs: + # When current directory has subdirectories, we must reinsert + # it into queue + found_subdirs = True + queue.append(id_p) + queue.append(id_) + + else: + name = self.db.get_val("SELECT name FROM names WHERE id=?", (name_id,)) + inval_entry(name) + self._remove(id_p, name, id_, force=True) + + processed += 1 + if processed > gil_step: + # Also reinsert current directory if we need to yield to other threads + if not found_subdirs: + queue.append(id_p) + break + + if processed > gil_step: + dt = time.time() - stamp + gil_step = max(int(gil_step * GIL_RELEASE_INTERVAL / dt), 250) + log.debug('Adjusting gil_step to %d and yielding', gil_step) + processed = 0 + llfuse.lock.yield_(100) + log.debug('re-acquired lock') + stamp = time.time() + + + if id_p0 in self.open_inodes: + log.debug('invalidate_entry(%d, %r)', id_p0, name0) + llfuse.invalidate_entry(id_p0, name0) + self._remove(id_p0, name0, id0, force=True) + + self.forget([(id0, 1)]) + log.debug('finished') + + + def copy_tree(self, src_id, target_id): + '''Efficiently copy directory tree''' + + if self.failsafe: + raise FUSEError(errno.EPERM) + + log.debug('started with %d, %d', src_id, target_id) + + # To avoid lookups and make code tidier + make_inode = self.inodes.create_inode + db = self.db + + # First we make sure that all blocks are in the database + self.cache.commit() + log.debug('committed cache') + + # Copy target attributes + # These come from setxattr, so they may have been deleted + # without being in open_inodes + try: + src_inode = self.inodes[src_id] + target_inode = self.inodes[target_id] + except KeyError: + raise FUSEError(errno.ENOENT) + for attr in ('atime_ns', 'ctime_ns', 'mtime_ns', 'mode', 'uid', 'gid'): + setattr(target_inode, attr, getattr(src_inode, attr)) + + # We first replicate into a dummy inode, so that we + # need to invalidate only once. + now_ns = time_ns() + tmp = make_inode(mtime_ns=now_ns, ctime_ns=now_ns, atime_ns=now_ns, + uid=0, gid=0, mode=0, refcount=0) + + queue = [ (src_id, tmp.id, 0) ] + id_cache = dict() + processed = 0 # Number of steps since last GIL release + stamp = time.time() # Time of last GIL release + gil_step = 250 # Approx. number of steps between GIL releases + while queue: + (src_id, target_id, off) = queue.pop() + log.debug('Processing directory (%d, %d, %d)', src_id, target_id, off) + with db.query('SELECT name_id, inode FROM contents WHERE parent_inode=? ' + 'AND name_id > ? ORDER BY name_id', (src_id, off)) as res: + for (name_id, id_) in res: + + if id_ not in id_cache: + inode = self.inodes[id_] + + try: + inode_new = make_inode(refcount=1, mode=inode.mode, size=inode.size, + uid=inode.uid, gid=inode.gid, + mtime_ns=inode.mtime_ns, atime_ns=inode.atime_ns, + ctime_ns=inode.ctime_ns, rdev=inode.rdev) + except OutOfInodesError: + log.warning('Could not find a free inode') + raise FUSEError(errno.ENOSPC) + + id_new = inode_new.id + + if inode.refcount != 1: + id_cache[id_] = id_new + + db.execute('INSERT INTO symlink_targets (inode, target) ' + 'SELECT ?, target FROM symlink_targets WHERE inode=?', + (id_new, id_)) + + db.execute('INSERT INTO ext_attributes (inode, name_id, value) ' + 'SELECT ?, name_id, value FROM ext_attributes WHERE inode=?', + (id_new, id_)) + db.execute('UPDATE names SET refcount = refcount + 1 WHERE ' + 'id IN (SELECT name_id FROM ext_attributes WHERE inode=?)', + (id_,)) + + processed += db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) ' + 'SELECT ?, blockno, block_id FROM inode_blocks ' + 'WHERE inode=?', (id_new, id_)) + db.execute('REPLACE INTO blocks (id, hash, refcount, size, obj_id) ' + 'SELECT id, hash, refcount+COUNT(id), size, obj_id ' + 'FROM inode_blocks JOIN blocks ON block_id = id ' + 'WHERE inode = ? GROUP BY id', (id_new,)) + + if db.has_val('SELECT 1 FROM contents WHERE parent_inode=?', (id_,)): + queue.append((id_, id_new, 0)) + else: + id_new = id_cache[id_] + self.inodes[id_new].refcount += 1 + + db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?, ?, ?)', + (name_id, id_new, target_id)) + db.execute('UPDATE names SET refcount=refcount+1 WHERE id=?', (name_id,)) + + processed += 1 + + if processed > gil_step: + log.debug('Requeueing (%d, %d, %d) to yield lock', + src_id, target_id, name_id) + queue.append((src_id, target_id, name_id)) + break + + if processed > gil_step: + dt = time.time() - stamp + gil_step = max(int(gil_step * GIL_RELEASE_INTERVAL / dt), 250) + log.debug('Adjusting gil_step to %d and yielding', gil_step) + processed = 0 + llfuse.lock.yield_(100) + log.debug('re-acquired lock') + stamp = time.time() + + # Make replication visible + self.db.execute('UPDATE contents SET parent_inode=? WHERE parent_inode=?', + (target_inode.id, tmp.id)) + del self.inodes[tmp.id] + llfuse.invalidate_inode(target_inode.id) + + log.debug('finished') + + def unlink(self, id_p, name, ctx): + log.debug('started with %d, %r', id_p, name) + if self.failsafe: + raise FUSEError(errno.EPERM) + + inode = self._lookup(id_p, name, ctx) + + if stat.S_ISDIR(inode.mode): + raise llfuse.FUSEError(errno.EISDIR) + + self._remove(id_p, name, inode.id) + + self.forget([(inode.id, 1)]) + + def rmdir(self, id_p, name, ctx): + log.debug('started with %d, %r', id_p, name) + if self.failsafe: + raise FUSEError(errno.EPERM) + + inode = self._lookup(id_p, name, ctx) + + if self.inodes[id_p].locked: + raise FUSEError(errno.EPERM) + + if not stat.S_ISDIR(inode.mode): + raise llfuse.FUSEError(errno.ENOTDIR) + + self._remove(id_p, name, inode.id) + + self.forget([(inode.id, 1)]) + + def _remove(self, id_p, name, id_, force=False): + '''Remove entry `name` with parent inode `id_p` + + `id_` must be the inode of `name`. If `force` is True, then + the `locked` attribute is ignored. + + This method releases the global lock. + ''' + + log.debug('started with %d, %r', id_p, name) + + now_ns = time_ns() + + # Check that there are no child entries + if self.db.has_val("SELECT 1 FROM contents WHERE parent_inode=?", (id_,)): + log.debug("Attempted to remove entry with children: %s", + get_path(id_p, self.db, name)) + raise llfuse.FUSEError(errno.ENOTEMPTY) + + if self.inodes[id_p].locked and not force: + raise FUSEError(errno.EPERM) + + name_id = self._del_name(name) + self.db.execute("DELETE FROM contents WHERE name_id=? AND parent_inode=?", + (name_id, id_p)) + + inode = self.inodes[id_] + inode.refcount -= 1 + inode.ctime_ns = now_ns + + inode_p = self.inodes[id_p] + inode_p.mtime_ns = now_ns + inode_p.ctime_ns = now_ns + + if inode.refcount == 0 and id_ not in self.open_inodes: + log.debug('removing from cache') + self.cache.remove(id_, 0, int(math.ceil(inode.size / self.max_obj_size))) + # Since the inode is not open, it's not possible that new blocks + # get created at this point and we can safely delete the inode + self.db.execute('UPDATE names SET refcount = refcount - 1 WHERE ' + 'id IN (SELECT name_id FROM ext_attributes WHERE inode=?)', + (id_,)) + self.db.execute('DELETE FROM names WHERE refcount=0 AND ' + 'id IN (SELECT name_id FROM ext_attributes WHERE inode=?)', + (id_,)) + self.db.execute('DELETE FROM ext_attributes WHERE inode=?', (id_,)) + self.db.execute('DELETE FROM symlink_targets WHERE inode=?', (id_,)) + del self.inodes[id_] + + log.debug('finished') + + def symlink(self, id_p, name, target, ctx): + log.debug('started with %d, %r, %r', id_p, name, target) + + if self.failsafe: + raise FUSEError(errno.EPERM) + + mode = (stat.S_IFLNK | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) + + # Unix semantics require the size of a symlink to be the length + # of its target. Therefore, we create symlink directory entries + # with this size. If the kernel ever learns to open and read + # symlinks directly, it will read the corresponding number of \0 + # bytes. + inode = self._create(id_p, name, mode, ctx, size=len(target)) + self.db.execute('INSERT INTO symlink_targets (inode, target) VALUES(?,?)', + (inode.id, target)) + self.open_inodes[inode.id] += 1 + return inode.entry_attributes() + + def rename(self, id_p_old, name_old, id_p_new, name_new, ctx): + log.debug('started with %d, %r, %d, %r', id_p_old, name_old, id_p_new, name_new) + if name_new == CTRL_NAME or name_old == CTRL_NAME: + log.warning('Attempted to rename s3ql control file (%s -> %s)', + get_path(id_p_old, self.db, name_old), + get_path(id_p_new, self.db, name_new)) + raise llfuse.FUSEError(errno.EACCES) + + + if (self.failsafe or self.inodes[id_p_old].locked + or self.inodes[id_p_new].locked): + raise FUSEError(errno.EPERM) + + inode_old = self._lookup(id_p_old, name_old, ctx) + + try: + inode_new = self._lookup(id_p_new, name_new, ctx) + except llfuse.FUSEError as exc: + if exc.errno != errno.ENOENT: + raise + else: + target_exists = False + else: + target_exists = True + + if target_exists: + self._replace(id_p_old, name_old, id_p_new, name_new, + inode_old.id, inode_new.id) + self.forget([(inode_old.id, 1), (inode_new.id, 1)]) + else: + self._rename(id_p_old, name_old, id_p_new, name_new) + self.forget([(inode_old.id, 1)]) + + def _add_name(self, name): + '''Get id for *name* and increase refcount + + Name is inserted in table if it does not yet exist. + ''' + + try: + name_id = self.db.get_val('SELECT id FROM names WHERE name=?', (name,)) + except NoSuchRowError: + name_id = self.db.rowid('INSERT INTO names (name, refcount) VALUES(?,?)', + (name, 1)) + else: + self.db.execute('UPDATE names SET refcount=refcount+1 WHERE id=?', (name_id,)) + return name_id + + def _del_name(self, name): + '''Decrease refcount for *name* + + Name is removed from table if refcount drops to zero. Returns the + (possibly former) id of the name. + ''' + + (name_id, refcount) = self.db.get_row('SELECT id, refcount FROM names WHERE name=?', (name,)) + + if refcount > 1: + self.db.execute('UPDATE names SET refcount=refcount-1 WHERE id=?', (name_id,)) + else: + self.db.execute('DELETE FROM names WHERE id=?', (name_id,)) + + return name_id + + def _rename(self, id_p_old, name_old, id_p_new, name_new): + now_ns = time_ns() + + name_id_new = self._add_name(name_new) + name_id_old = self._del_name(name_old) + + self.db.execute("UPDATE contents SET name_id=?, parent_inode=? WHERE name_id=? " + "AND parent_inode=?", (name_id_new, id_p_new, + name_id_old, id_p_old)) + + inode_p_old = self.inodes[id_p_old] + inode_p_new = self.inodes[id_p_new] + inode_p_old.mtime_ns = now_ns + inode_p_new.mtime_ns = now_ns + inode_p_old.ctime_ns = now_ns + inode_p_new.ctime_ns = now_ns + + def _replace(self, id_p_old, name_old, id_p_new, name_new, + id_old, id_new): + + now_ns = time_ns() + + if self.db.has_val("SELECT 1 FROM contents WHERE parent_inode=?", (id_new,)): + log.info("Attempted to overwrite entry with children: %s", + get_path(id_p_new, self.db, name_new)) + raise llfuse.FUSEError(errno.EINVAL) + + # Replace target + name_id_new = self.db.get_val('SELECT id FROM names WHERE name=?', (name_new,)) + self.db.execute("UPDATE contents SET inode=? WHERE name_id=? AND parent_inode=?", + (id_old, name_id_new, id_p_new)) + + # Delete old name + name_id_old = self._del_name(name_old) + self.db.execute('DELETE FROM contents WHERE name_id=? AND parent_inode=?', + (name_id_old, id_p_old)) + + inode_new = self.inodes[id_new] + inode_new.refcount -= 1 + inode_new.ctime_ns = now_ns + + inode_p_old = self.inodes[id_p_old] + inode_p_old.ctime_ns = now_ns + inode_p_old.mtime_ns = now_ns + + inode_p_new = self.inodes[id_p_new] + inode_p_new.ctime_ns = now_ns + inode_p_new.mtime_ns = now_ns + + if inode_new.refcount == 0 and id_new not in self.open_inodes: + self.cache.remove(id_new, 0, + int(math.ceil(inode_new.size / self.max_obj_size))) + # Since the inode is not open, it's not possible that new blocks + # get created at this point and we can safely delete the inode + self.db.execute('UPDATE names SET refcount = refcount - 1 WHERE ' + 'id IN (SELECT name_id FROM ext_attributes WHERE inode=?)', + (id_new,)) + self.db.execute('DELETE FROM names WHERE refcount=0') + self.db.execute('DELETE FROM ext_attributes WHERE inode=?', (id_new,)) + self.db.execute('DELETE FROM symlink_targets WHERE inode=?', (id_new,)) + del self.inodes[id_new] + + + def link(self, id_, new_id_p, new_name, ctx): + log.debug('started with %d, %d, %r', id_, new_id_p, new_name) + + if new_name == CTRL_NAME or id_ == CTRL_INODE: + log.warning('Attempted to create s3ql control file at %s', + get_path(new_id_p, self.db, new_name)) + raise llfuse.FUSEError(errno.EACCES) + + now_ns = time_ns() + inode_p = self.inodes[new_id_p] + + if inode_p.refcount == 0: + log.warning('Attempted to create entry %s with unlinked parent %d', + new_name, new_id_p) + raise FUSEError(errno.EINVAL) + + if self.failsafe or inode_p.locked: + raise FUSEError(errno.EPERM) + + inode_p.ctime_ns = now_ns + inode_p.mtime_ns = now_ns + + self.db.execute("INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)", + (self._add_name(new_name), id_, new_id_p)) + inode = self.inodes[id_] + inode.refcount += 1 + inode.ctime_ns = now_ns + + self.open_inodes[inode.id] += 1 + return inode.entry_attributes() + + def setattr(self, id_, attr, fields, fh, ctx): + """Handles FUSE setattr() requests""" + inode = self.inodes[id_] + if fh is not None: + assert fh == id_ + now_ns = time_ns() + + if self.failsafe or inode.locked: + raise FUSEError(errno.EPERM) + + if fields.update_mode: + inode.mode = attr.st_mode + + if fields.update_uid: + inode.uid = attr.st_uid + + if fields.update_gid: + inode.gid = attr.st_gid + + if fields.update_atime: + inode.atime_ns = attr.st_atime_ns + + if fields.update_mtime: + inode.mtime_ns = attr.st_mtime_ns + + inode.ctime_ns = now_ns + + # This needs to go last, because the call to cache.remove and cache.get + # will release the global lock and may thus evict the *inode* object + # from the cache. + if fields.update_size: + len_ = attr.st_size + + # Determine blocks to delete + last_block = len_ // self.max_obj_size + cutoff = len_ % self.max_obj_size + total_blocks = int(math.ceil(inode.size / self.max_obj_size)) + + # Adjust file size + inode.size = len_ + + # Delete blocks and truncate last one if required + if cutoff == 0: + self.cache.remove(id_, last_block, total_blocks) + else: + self.cache.remove(id_, last_block + 1, total_blocks) + try: + with self.cache.get(id_, last_block) as fh: + fh.truncate(cutoff) + except NoSuchObject as exc: + log.warning('Backend lost block %d of inode %d (id %s)!', + last_block, id_, exc.key) + raise + + except CorruptedObjectError as exc: + log.warning('Backend returned malformed data for block %d of inode %d (%s)', + last_block, id_, exc) + self.failsafe = True + self.broken_blocks[id_].add(last_block) + raise FUSEError(errno.EIO) + + + return inode.entry_attributes() + + def mknod(self, id_p, name, mode, rdev, ctx): + log.debug('started with %d, %r', id_p, name) + if self.failsafe: + raise FUSEError(errno.EPERM) + inode = self._create(id_p, name, mode, ctx, rdev=rdev) + self.open_inodes[inode.id] += 1 + return inode.entry_attributes() + + def mkdir(self, id_p, name, mode, ctx): + log.debug('started with %d, %r', id_p, name) + if self.failsafe: + raise FUSEError(errno.EPERM) + inode = self._create(id_p, name, mode, ctx) + self.open_inodes[inode.id] += 1 + return inode.entry_attributes() + + def extstat(self): + '''Return extended file system statistics''' + + log.debug('started') + + # Flush inode cache to get better estimate of total fs size + self.inodes.flush() + + entries = self.db.get_val("SELECT COUNT(rowid) FROM contents") + blocks = self.db.get_val("SELECT COUNT(id) FROM objects") + inodes = self.db.get_val("SELECT COUNT(id) FROM inodes") + fs_size = self.db.get_val('SELECT SUM(size) FROM inodes') or 0 + dedup_size = self.db.get_val('SELECT SUM(size) FROM blocks') or 0 + + # Objects that are currently being uploaded/compressed have size == -1 + compr_size = self.db.get_val('SELECT SUM(size) FROM objects ' + 'WHERE size > 0') or 0 + + return struct.pack('QQQQQQQQQQQQ', entries, blocks, inodes, fs_size, dedup_size, + compr_size, self.db.get_size(), *self.cache.get_usage()) + + def statfs(self, ctx): + log.debug('started') + + stat_ = llfuse.StatvfsData() + + # Get number of blocks & inodes + blocks = self.db.get_val("SELECT COUNT(id) FROM objects") + inodes = self.db.get_val("SELECT COUNT(id) FROM inodes") + size = self.db.get_val('SELECT SUM(size) FROM blocks') + + if size is None: + size = 0 + + # file system block size, i.e. the minimum amount of space that can + # be allocated. This doesn't make much sense for S3QL, so we just + # return the average size of stored blocks. + stat_.f_frsize = max(4096, size // blocks) if blocks != 0 else 4096 + + # This should actually be the "preferred block size for doing IO. However, `df` incorrectly + # interprets f_blocks, f_bfree and f_bavail in terms of f_bsize rather than f_frsize as it + # should (according to statvfs(3)), so the only way to return correct values *and* have df + # print something sensible is to set f_bsize and f_frsize to the same value. (cf. + # http://bugs.debian.org/671490) + stat_.f_bsize = stat_.f_frsize + + # size of fs in f_frsize units. Since backend is supposed to be unlimited, + # always return a half-full filesystem, but at least 1 TB) + fs_size = max(2 * size, 1024 ** 4) + + stat_.f_blocks = fs_size // stat_.f_frsize + stat_.f_bfree = (fs_size - size) // stat_.f_frsize + stat_.f_bavail = stat_.f_bfree # free for non-root + + total_inodes = max(2 * inodes, 1000000) + stat_.f_files = total_inodes + stat_.f_ffree = total_inodes - inodes + stat_.f_favail = total_inodes - inodes # free for non-root + + return stat_ + + def open(self, id_, flags, ctx): + log.debug('started with %d', id_) + if ((flags & os.O_RDWR or flags & os.O_WRONLY) + and (self.failsafe or self.inodes[id_].locked)): + raise FUSEError(errno.EPERM) + + return id_ + + def access(self, id_, mode, ctx): + '''Check if requesting process has `mode` rights on `inode`. + + This method always returns true, since it should only be called + when permission checking is disabled (if permission checking is + enabled, the `default_permissions` FUSE option should be set). + ''' + # Yeah, could be a function and has unused arguments + #pylint: disable=R0201,W0613 + + log.debug('started with %d', id_) + return True + + def create(self, id_p, name, mode, flags, ctx): + log.debug('started with id_p=%d, %s', id_p, name) + if self.failsafe: + raise FUSEError(errno.EPERM) + + try: + id_ = self.db.get_val("SELECT inode FROM contents_v WHERE name=? AND parent_inode=?", + (name, id_p)) + except NoSuchRowError: + inode = self._create(id_p, name, mode, ctx) + else: + self.open(id_, flags, ctx) + inode = self.inodes[id_] + + self.open_inodes[inode.id] += 1 + return (inode.id, inode.entry_attributes()) + + def _create(self, id_p, name, mode, ctx, rdev=0, size=0): + if name == CTRL_NAME: + log.warning('Attempted to create s3ql control file at %s', + get_path(id_p, self.db, name)) + raise FUSEError(errno.EACCES) + + now_ns = time_ns() + inode_p = self.inodes[id_p] + + if inode_p.locked: + raise FUSEError(errno.EPERM) + + if inode_p.refcount == 0: + log.warning('Attempted to create entry %s with unlinked parent %d', + name, id_p) + raise FUSEError(errno.EINVAL) + inode_p.mtime_ns = now_ns + inode_p.ctime_ns = now_ns + + if inode_p.mode & stat.S_ISGID: + gid = inode_p.gid + if stat.S_ISDIR(mode): + mode |= stat.S_ISGID + else: + gid = ctx.gid + try: + inode = self.inodes.create_inode(mtime_ns=now_ns, ctime_ns=now_ns, atime_ns=now_ns, + uid=ctx.uid, gid=gid, mode=mode, refcount=1, + rdev=rdev, size=size) + except OutOfInodesError: + log.warning('Could not find a free inode') + raise FUSEError(errno.ENOSPC) + + self.db.execute("INSERT INTO contents(name_id, inode, parent_inode) VALUES(?,?,?)", + (self._add_name(name), inode.id, id_p)) + + return inode + + + def read(self, fh, offset, length): + '''Read `size` bytes from `fh` at position `off` + + Unless EOF is reached, returns exactly `size` bytes. + + This method releases the global lock while it is running. + ''' + #log.debug('started with %d, %d, %d', fh, offset, length) + buf = BytesIO() + inode = self.inodes[fh] + + # Make sure that we don't read beyond the file size. This + # should not happen unless direct_io is activated, but it's + # cheap and nice for testing. + size = inode.size + length = min(size - offset, length) + + while length > 0: + tmp = self._readwrite(fh, offset, length=length) + buf.write(tmp) + length -= len(tmp) + offset += len(tmp) + + # Inode may have expired from cache + inode = self.inodes[fh] + + if inode.atime_ns < inode.ctime_ns or inode.atime_ns < inode.mtime_ns: + inode.atime_ns = time_ns() + + return buf.getvalue() + + + def write(self, fh, offset, buf): + '''Handle FUSE write requests. + + This method releases the global lock while it is running. + ''' + #log.debug('started with %d, %d, datalen=%d', fh, offset, len(buf)) + + if self.failsafe or self.inodes[fh].locked: + raise FUSEError(errno.EPERM) + + total = len(buf) + minsize = offset + total + while buf: + written = self._readwrite(fh, offset, buf=buf) + offset += written + buf = buf[written:] + + # Update file size if changed + # Fuse does not ensure that we do not get concurrent write requests, + # so we have to be careful not to undo a size extension made by + # a concurrent write (because _readwrite() releases the global + # lock). + now_ns = time_ns() + inode = self.inodes[fh] + inode.size = max(inode.size, minsize) + inode.mtime_ns = now_ns + inode.ctime_ns = now_ns + + return total + + def _readwrite(self, id_, offset, *, buf=None, length=None): + """Read or write as much as we can. + + If *buf* is None, read and return up to *length* bytes. + + If *length* is None, write from *buf* and return the number of + bytes written. + + This is one method to reduce code duplication. + + This method releases the global lock while it is running. + """ + + # Calculate required block + blockno = offset // self.max_obj_size + offset_rel = offset - blockno * self.max_obj_size + + if id_ in self.broken_blocks and blockno in self.broken_blocks[id_]: + raise FUSEError(errno.EIO) + + if length is None: + write = True + length = len(buf) + elif buf is None: + write = False + else: + raise TypeError("Don't know what to do!") + + # Don't try to write/read into the next block + if offset_rel + length > self.max_obj_size: + length = self.max_obj_size - offset_rel + + try: + with self.cache.get(id_, blockno) as fh: + fh.seek(offset_rel) + if write: + fh.write(buf[:length]) + else: + buf = fh.read(length) + + except NoSuchObject as exc: + log.error('Backend lost block %d of inode %d (id %s)!', + blockno, id_, exc.key) + self.failsafe = True + self.broken_blocks[id_].add(blockno) + raise FUSEError(errno.EIO) + + except CorruptedObjectError as exc: + log.error('Backend returned malformed data for block %d of inode %d (%s)', + blockno, id_, exc) + self.failsafe = True + self.broken_blocks[id_].add(blockno) + raise FUSEError(errno.EIO) + + if write: + return length + elif len(buf) == length: + return buf + else: + # If we can't read enough, add null bytes + return buf + b"\0" * (length - len(buf)) + + def fsync(self, fh, datasync): + log.debug('started with %d, %s', fh, datasync) + if not datasync: + self.inodes.flush_id(fh) + + for blockno in range(0, self.inodes[fh].size // self.max_obj_size + 1): + self.cache.flush(fh, blockno) + + def forget(self, forget_list): + log.debug('started with %s', forget_list) + + for (id_, nlookup) in forget_list: + self.open_inodes[id_] -= nlookup + + if self.open_inodes[id_] == 0: + del self.open_inodes[id_] + if id_ in self.broken_blocks: + del self.broken_blocks[id_] + + inode = self.inodes[id_] + if inode.refcount == 0: + log.debug('removing %d from cache', id_) + self.cache.remove(id_, 0, inode.size // self.max_obj_size + 1) + # Since the inode is not open, it's not possible that new blocks + # get created at this point and we can safely delete the inode + self.db.execute('UPDATE names SET refcount = refcount - 1 WHERE ' + 'id IN (SELECT name_id FROM ext_attributes WHERE inode=?)', + (id_,)) + self.db.execute('DELETE FROM names WHERE refcount=0 AND ' + 'id IN (SELECT name_id FROM ext_attributes WHERE inode=?)', + (id_,)) + self.db.execute('DELETE FROM ext_attributes WHERE inode=?', (id_,)) + self.db.execute('DELETE FROM symlink_targets WHERE inode=?', (id_,)) + del self.inodes[id_] + + + def fsyncdir(self, fh, datasync): + log.debug('started with %d, %s', fh, datasync) + if not datasync: + self.inodes.flush_id(fh) + + def releasedir(self, fh): + log.debug('started with %d', fh) + + def release(self, fh): + log.debug('started with %d', fh) + + def flush(self, fh): + log.debug('started with %d', fh) + + +def update_logging(level, modules): + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + if level == logging.DEBUG: + logging.disable(logging.NOTSET) + if 'all' in modules: + root_logger.setLevel(logging.DEBUG) + else: + for module in modules: + logging.getLogger(module).setLevel(logging.DEBUG) + + else: + logging.disable(logging.DEBUG) diff --git a/src/s3ql/fsck.py b/src/s3ql/fsck.py new file mode 100644 index 0000000..3b03414 --- /dev/null +++ b/src/s3ql/fsck.py @@ -0,0 +1,1370 @@ +''' +fsck.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging, setup_logging, QuietError +from . import CURRENT_FS_REV, BUFSIZE, CTRL_INODE, ROOT_INODE +from .backends.common import NoSuchObject +from .backends.comprenc import ComprencBackend +from .backends.local import Backend as LocalBackend +from .common import (inode_for_path, sha256_fh, get_path, get_backend_cachedir, + get_seq_no, is_mounted, get_backend, load_params, + save_params, time_ns) +from .database import NoSuchRowError, Connection +from .metadata import create_tables, dump_and_upload_metadata, download_metadata +from .parse_args import ArgumentParser +from os.path import basename +import apsw +import os +import re +import shutil +import itertools +import stat +import sys +import textwrap +import time +import atexit + +log = logging.getLogger(__name__) + +S_IFMT = (stat.S_IFDIR | stat.S_IFREG | stat.S_IFSOCK | stat.S_IFBLK | + stat.S_IFCHR | stat.S_IFIFO | stat.S_IFLNK) + +class Fsck(object): + + def __init__(self, cachedir_, backend_, param, conn): + + self.cachedir = cachedir_ + self.backend = backend_ + self.expect_errors = False + self.found_errors = False + self.uncorrectable_errors = False + self.max_obj_size = param['max_obj_size'] + self.conn = conn + + # Set of blocks that have been unlinked by check_cache. + # check_block_refcounts() will not report errors if these blocks still + # exist even though they have refcount=0 + self.unlinked_blocks = set() + + # Similarly for objects + self.unlinked_objects = set() + + # Set of inodes that have been moved to lost+found (so that we + # don't move them there repeatedly) + self.moved_inodes = set() + + def check(self): + """Check file system + + Sets instance variable `found_errors`. + """ + + # Create indices required for reference checking + log.info('Creating temporary extra indices...') + for idx in ('tmp1', 'tmp2', 'tmp3', 'tmp4', 'tmp5'): + self.conn.execute('DROP INDEX IF EXISTS %s' % idx) + self.conn.execute('CREATE INDEX tmp1 ON blocks(obj_id)') + self.conn.execute('CREATE INDEX tmp2 ON inode_blocks(block_id)') + self.conn.execute('CREATE INDEX tmp3 ON contents(inode)') + self.conn.execute('CREATE INDEX tmp4 ON contents(name_id)') + self.conn.execute('CREATE INDEX tmp5 ON ext_attributes(name_id)') + try: + self.check_lof() + self.check_cache() + self.check_names_refcount() + + self.check_contents_name() + self.check_contents_inode() + self.check_contents_parent_inode() + + self.check_objects_temp() + self.check_objects_refcount() + self.check_objects_id() + self.check_objects_size() + + self.check_blocks_obj_id() + self.check_blocks_refcount() + self.check_blocks_checksum() + + self.check_inode_blocks_block_id() + self.check_inode_blocks_inode() + + self.check_inodes_refcount() + self.check_inodes_size() + + self.check_ext_attributes_name() + self.check_ext_attributes_inode() + + self.check_symlinks_inode() + + self.check_loops() + self.check_unix() + self.check_foreign_keys() + finally: + log.info('Dropping temporary indices...') + for idx in ('tmp1', 'tmp2', 'tmp3', 'tmp4', 'tmp5'): + self.conn.execute('DROP INDEX %s' % idx) + + def log_error(self, *a, **kw): + '''Log file system error if not expected''' + + if self.expect_errors: + return log.debug(*a, **kw) + else: + return log.warning(*a, **kw) + + def check_foreign_keys(self): + '''Check for referential integrity + + Checks that all foreign keys in the SQLite tables actually resolve. + This is necessary, because we disable runtime checking by SQLite + for performance reasons. + + Note: any problems should have already been caught by the more + specific checkers. + ''' + + log.info("Checking referential integrity...") + + for (table,) in self.conn.query("SELECT name FROM sqlite_master WHERE type='table'"): + for row in self.conn.query('PRAGMA foreign_key_list(%s)' % table): + sql_objs = { 'src_table': table, + 'dst_table': row[2], + 'src_col': row[3], + 'dst_col': row[4] } + + for (val,) in self.conn.query('SELECT %(src_table)s.%(src_col)s ' + 'FROM %(src_table)s LEFT JOIN %(dst_table)s ' + 'ON %(src_table)s.%(src_col)s = %(dst_table)s.%(dst_col)s ' + 'WHERE %(dst_table)s.%(dst_col)s IS NULL ' + 'AND %(src_table)s.%(src_col)s IS NOT NULL' + % sql_objs): + self.found_errors = True + sql_objs['val'] = val + self.log_error('%(src_table)s.%(src_col)s refers to non-existing key %(val)s ' + 'in %(dst_table)s.%(dst_col)s, deleting.', sql_objs) + log.error('This should not happen, please report a bug.') + self.uncorrectable_errors = True + + + def check_cache(self): + """Commit uncommitted cache files""" + + log.info("Checking cached objects...") + if not os.path.exists(self.cachedir): + return + + for filename in os.listdir(self.cachedir): + self.found_errors = True + + match = re.match('^(\\d+)-(\\d+)$', filename) + if match: + inode = int(match.group(1)) + blockno = int(match.group(2)) + else: + raise RuntimeError('Strange file in cache directory: %s' % filename) + + self.log_error("Committing block %d of inode %d to backend", blockno, inode) + + with open(os.path.join(self.cachedir, filename), "rb") as fh: + size = os.fstat(fh.fileno()).st_size + hash_ = sha256_fh(fh) + + try: + (block_id, obj_id) = self.conn.get_row('SELECT id, obj_id FROM blocks WHERE hash=?', (hash_,)) + + except NoSuchRowError: + obj_id = self.conn.rowid('INSERT INTO objects (refcount, size) VALUES(1, -1)') + block_id = self.conn.rowid('INSERT INTO blocks (refcount, hash, obj_id, size) ' + 'VALUES(?, ?, ?, ?)', (1, hash_, obj_id, size)) + def do_write(obj_fh): + with open(os.path.join(self.cachedir, filename), "rb") as fh: + shutil.copyfileobj(fh, obj_fh, BUFSIZE) + return obj_fh + + obj_size = self.backend.perform_write(do_write, 's3ql_data_%d' % obj_id).get_obj_size() + + self.conn.execute('UPDATE objects SET size=? WHERE id=?', (obj_size, obj_id)) + + else: + self.conn.execute('UPDATE blocks SET refcount=refcount+1 WHERE id=?', (block_id,)) + + try: + old_block_id = self.conn.get_val('SELECT block_id FROM inode_blocks ' + 'WHERE inode=? AND blockno=?', (inode, blockno)) + except NoSuchRowError: + self.conn.execute('INSERT INTO inode_blocks (block_id, inode, blockno) VALUES(?,?,?)', + (block_id, inode, blockno)) + else: + self.conn.execute('UPDATE inode_blocks SET block_id=? WHERE inode=? AND blockno=?', + (block_id, inode, blockno)) + + # We just decrease the refcount, but don't take any action + # because the reference count might be wrong + self.conn.execute('UPDATE blocks SET refcount=refcount-1 WHERE id=?', (old_block_id,)) + self.unlinked_blocks.add(old_block_id) + + os.unlink(os.path.join(self.cachedir, filename)) + + + def check_lof(self): + """Ensure that there is a lost+found directory""" + + log.info('Checking lost+found...') + + now_ns = time_ns() + try: + (inode_l, name_id) = self.conn.get_row("SELECT inode, name_id FROM contents_v " + "WHERE name=? AND parent_inode=?", (b"lost+found", ROOT_INODE)) + + except NoSuchRowError: + self.found_errors = True + self.log_error("Recreating missing lost+found directory") + inode_l = self.create_inode(mode=stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR, + atime_ns=now_ns, ctime_ns=now_ns, mtime_ns=now_ns, + refcount=1) + self.conn.execute("INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)", + (self._add_name(b"lost+found"), inode_l, ROOT_INODE)) + + + mode = self.conn.get_val('SELECT mode FROM inodes WHERE id=?', (inode_l,)) + if not stat.S_ISDIR(mode): + self.found_errors = True + self.log_error('/lost+found is not a directory! Old entry will be saved as ' + '/lost+found/inode-%s*', inode_l) + # We leave the old inode unassociated, so that it will be added + # to lost+found later on. + inode_l = self.create_inode(mode=stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR, + atime_ns=now_ns, ctime_ns=now_ns, mtime_ns=now_ns, + refcount=1) + self.conn.execute('UPDATE contents SET inode=? WHERE name_id=? AND parent_inode=?', + (inode_l, name_id, ROOT_INODE)) + + def check_contents_name(self): + """Check contents.name_id""" + + log.info('Checking contents (names)...') + + for (rowid, name_id, inode_p, + inode) in self.conn.query('SELECT contents.rowid, name_id, parent_inode, inode ' + 'FROM contents LEFT JOIN names ' + 'ON name_id = names.id WHERE names.id IS NULL'): + self.found_errors = True + try: + path = get_path(inode_p, self.conn)[1:] + except NoSuchRowError: + newname = ('-%d' % inode).encode() + else: + newname = escape(path) + ('-%d' % inode).encode() + (id_p_new, newname) = self.resolve_free(b"/lost+found", newname) + + self.log_error('Content entry for inode %d refers to non-existing name with id %d, ' + 'moving to /lost+found/%s', inode, name_id, to_str(newname)) + + self.conn.execute('UPDATE contents SET name_id=?, parent_inode=? WHERE rowid=?', + (self._add_name(newname), id_p_new, rowid)) + + + def check_contents_parent_inode(self): + """Check contents.parent_inode""" + + log.info('Checking contents (parent inodes)...') + + for (rowid, inode_p, + name_id) in self.conn.query('SELECT contents.rowid, parent_inode, name_id ' + 'FROM contents LEFT JOIN inodes ' + 'ON parent_inode = inodes.id WHERE inodes.id IS NULL'): + self.found_errors = True + name = self.conn.get_val('SELECT name FROM names WHERE id = ?', (name_id,)) + (id_p_new, newname) = self.resolve_free(b"/lost+found", + ('[%d]-%s' % (inode_p, name)).encode()) + + self.log_error('Parent inode %d for "%s" vanished, moving to /lost+found', + inode_p, to_str(name)) + self._del_name(name_id) + self.conn.execute('UPDATE contents SET name_id=?, parent_inode=? WHERE rowid=?', + (self._add_name(newname), id_p_new, rowid)) + + + def check_contents_inode(self): + """Check contents.inode""" + + log.info('Checking contents (inodes)...') + + to_delete = list() + for (rowid, inode_p, inode, name_id) in self.conn.query('SELECT contents.rowid, parent_inode, inode, ' + 'name_id FROM contents LEFT JOIN inodes ' + 'ON inode = inodes.id WHERE inodes.id IS NULL'): + self.found_errors = True + try: + path = get_path(inode, self.conn)[1:] + except NoSuchRowError: + path = '[inode %d, parent %d]' % (inode, inode_p) + + self.log_error('Inode for %s vanished, deleting', to_str(path)) + self._del_name(name_id) + to_delete.append(rowid) + + for rowid in to_delete: + self.conn.execute('DELETE FROM contents WHERE rowid=?', (rowid,)) + + def check_ext_attributes_name(self): + """Check ext_attributes.name_id""" + + log.info('Checking extended attributes (names)...') + + for (rowid, name_id, inode) in self.conn.query('SELECT ext_attributes.rowid, name_id, inode ' + 'FROM ext_attributes LEFT JOIN names ' + 'ON name_id = names.id WHERE names.id IS NULL'): + + self.found_errors = True + for (name, id_p) in self.conn.query('SELECT name, parent_inode ' + 'FROM contents_v WHERE inode=?', (inode,)): + path = get_path(id_p, self.conn, name) + self.log_error('Extended attribute %d of %s refers to non-existing name %d, renaming..', + rowid, to_str(path), name_id) + + while True: + name_id = self._add_name('lost+found_%d' % rowid) + if not self.conn.has_val("SELECT 1 FROM ext_attributes WHERE name_id=? AND inode=?", + (name_id, inode)): + self.conn.execute('UPDATE ext_attributes SET name_id=? WHERE rowid=?', + (name_id, rowid)) + break + self._del_name('lost+found_%d' % rowid) + rowid += 1 + + def check_ext_attributes_inode(self): + """Check ext_attributes.inode""" + + log.info('Checking extended attributes (inodes)...') + + to_delete = list() + for (rowid, inode, name_id) in self.conn.query('SELECT ext_attributes.rowid, inode, name_id ' + 'FROM ext_attributes LEFT JOIN inodes ' + 'ON inode = inodes.id WHERE inodes.id IS NULL'): + self.found_errors = True + self.log_error('Extended attribute %d refers to non-existing inode %d, deleting', + rowid, inode) + to_delete.append(rowid) + self._del_name(name_id) + + for rowid in to_delete: + self.conn.execute('DELETE FROM ext_attributes WHERE rowid=?', (rowid,)) + + def check_loops(self): + """Ensure that all directories can be reached from root""" + + log.info('Checking directory reachability...') + + self.conn.execute('CREATE TEMPORARY TABLE loopcheck (inode INTEGER PRIMARY KEY, ' + 'parent_inode INTEGER)') + self.conn.execute('CREATE INDEX ix_loopcheck_parent_inode ON loopcheck(parent_inode)') + self.conn.execute('INSERT INTO loopcheck (inode) ' + 'SELECT parent_inode FROM contents GROUP BY parent_inode') + self.conn.execute('UPDATE loopcheck SET parent_inode = ' + '(SELECT contents.parent_inode FROM contents ' + ' WHERE contents.inode = loopcheck.inode LIMIT 1)') + self.conn.execute('CREATE TEMPORARY TABLE loopcheck2 (inode INTEGER PRIMARY KEY)') + self.conn.execute('INSERT INTO loopcheck2 (inode) SELECT inode FROM loopcheck') + + + def delete_tree(inode_p): + for (inode,) in self.conn.query("SELECT inode FROM loopcheck WHERE parent_inode=?", + (inode_p,)): + delete_tree(inode) + self.conn.execute('DELETE FROM loopcheck2 WHERE inode=?', (inode_p,)) + + root = ROOT_INODE + while True: + delete_tree(root) + + if not self.conn.has_val("SELECT 1 FROM loopcheck2"): + break + + self.found_errors = True + + # Try obvious culprits first + try: + inode = self.conn.get_val('SELECT loopcheck2.inode FROM loopcheck2 JOIN contents ' + 'ON loopcheck2.inode = contents.inode ' + 'WHERE parent_inode = contents.inode LIMIT 1') + except NoSuchRowError: + inode = self.conn.get_val("SELECT inode FROM loopcheck2 ORDER BY inode ASC LIMIT 1") + + (name, name_id) = self.conn.get_row("SELECT name, name_id FROM contents_v " + "WHERE inode=? LIMIT 1", (inode,)) + (id_p, name) = self.resolve_free(b"/lost+found", name) + + self.log_error("Found unreachable filesystem entries, re-anchoring %s [%d] " + "in /lost+found", to_str(name), inode) + self.conn.execute('UPDATE contents SET parent_inode=?, name_id=? ' + 'WHERE inode=? AND name_id=?', + (id_p, self._add_name(name), inode, name_id)) + self._del_name(name_id) + self.conn.execute('UPDATE loopcheck SET parent_inode=? WHERE inode=?', + (id_p, inode)) + root = inode + + self.conn.execute("DROP TABLE loopcheck") + self.conn.execute("DROP TABLE loopcheck2") + + def check_inodes_size(self): + """Check inodes.size""" + + log.info('Checking inodes (sizes)...') + + self.conn.execute('CREATE TEMPORARY TABLE min_sizes ' + '(id INTEGER PRIMARY KEY, min_size INTEGER NOT NULL)') + try: + self.conn.execute(''' + INSERT INTO min_sizes (id, min_size) + SELECT inode, MAX(blockno * ? + size) + FROM inode_blocks JOIN blocks ON block_id == blocks.id + GROUP BY inode''', (self.max_obj_size,)) + + self.conn.execute(''' + CREATE TEMPORARY TABLE wrong_sizes AS + SELECT id, size, min_size + FROM inodes JOIN min_sizes USING (id) + WHERE size < min_size''') + + for (id_, size_old, size) in self.conn.query('SELECT * FROM wrong_sizes'): + + self.found_errors = True + self.log_error("Size of inode %d (%s) does not agree with number of blocks, " + "setting from %d to %d", + id_, to_str(get_path(id_, self.conn)), size_old, size) + self.conn.execute("UPDATE inodes SET size=? WHERE id=?", (size, id_)) + finally: + self.conn.execute('DROP TABLE min_sizes') + self.conn.execute('DROP TABLE IF EXISTS wrong_sizes') + + def check_inodes_refcount(self): + """Check inodes.refcount""" + + log.info('Checking inodes (refcounts)...') + + self.conn.execute('CREATE TEMPORARY TABLE refcounts ' + '(id INTEGER PRIMARY KEY, refcount INTEGER NOT NULL)') + try: + self.conn.execute('INSERT INTO refcounts (id, refcount) ' + 'SELECT inode, COUNT(name_id) FROM contents GROUP BY inode') + + self.conn.execute(''' + CREATE TEMPORARY TABLE wrong_refcounts AS + SELECT id, refcounts.refcount, inodes.refcount + FROM inodes LEFT JOIN refcounts USING (id) + WHERE inodes.refcount != refcounts.refcount + OR refcounts.refcount IS NULL''') + + for (id_, cnt, cnt_old) in self.conn.query('SELECT * FROM wrong_refcounts'): + # No checks for root and control + if id_ in (ROOT_INODE, CTRL_INODE): + continue + + self.found_errors = True + if cnt is None: + (id_p, name) = self.resolve_free(b"/lost+found", ("inode-%d" % id_).encode()) + self.log_error("Inode %d not referenced, adding as /lost+found/%s", + id_, to_str(name)) + self.conn.execute("INSERT INTO contents (name_id, inode, parent_inode) " + "VALUES (?,?,?)", (self._add_name(basename(name)), id_, id_p)) + self.conn.execute("UPDATE inodes SET refcount=? WHERE id=?", (1, id_)) + + else: + self.log_error("Inode %d (%s) has wrong reference count, setting from %d to %d", + id_, to_str(get_path(id_, self.conn)), cnt_old, cnt) + self.conn.execute("UPDATE inodes SET refcount=? WHERE id=?", (cnt, id_)) + finally: + self.conn.execute('DROP TABLE refcounts') + self.conn.execute('DROP TABLE IF EXISTS wrong_refcounts') + + def check_blocks_obj_id(self): + """Check blocks.obj_id""" + + log.info('Checking blocks (referenced objects)...') + + for (block_id, obj_id) in self.conn.query('SELECT blocks.id, obj_id FROM blocks LEFT JOIN objects ' + 'ON obj_id = objects.id WHERE objects.id IS NULL'): + self.found_errors = True + self.log_error('Block %d refers to non-existing object %d', block_id, obj_id) + for (inode,) in self.conn.query('SELECT inode FROM inode_blocks WHERE block_id = ? ', + (block_id,)): + if inode in self.moved_inodes: + continue + self.moved_inodes.add(inode) + + affected_entries = list(self.conn.query('SELECT name, name_id, parent_inode ' + 'FROM contents_v WHERE inode=?', (inode,))) + for (name, name_id, id_p) in affected_entries: + path = get_path(id_p, self.conn, name) + self.log_error("File may lack data, moved to /lost+found: %s", to_str(path)) + (lof_id, newname) = self.resolve_free(b"/lost+found", escape(path)) + + self.conn.execute('UPDATE contents SET name_id=?, parent_inode=? ' + 'WHERE name_id=? AND parent_inode=?', + (self._add_name(newname), lof_id, name_id, id_p)) + self._del_name(name_id) + + self.conn.execute('DELETE FROM inode_blocks WHERE block_id=?', (block_id,)) + self.conn.execute("DELETE FROM blocks WHERE id=?", (block_id,)) + + + def check_inode_blocks_inode(self): + """Check inode_blocks.inode""" + + log.info('Checking inode-block mapping (inodes)...') + + to_delete = list() + for (rowid, inode, block_id) in self.conn.query('SELECT inode_blocks.rowid, inode, block_id ' + 'FROM inode_blocks LEFT JOIN inodes ' + 'ON inode = inodes.id WHERE inodes.id IS NULL'): + self.found_errors = True + self.log_error('Inode-block mapping %d refers to non-existing inode %d, deleting', + rowid, inode) + to_delete.append(rowid) + self.unlinked_blocks.add(block_id) + + for rowid in to_delete: + self.conn.execute('DELETE FROM inode_blocks WHERE rowid=?', (rowid,)) + + def check_inode_blocks_block_id(self): + """Check inode_blocks.block_id""" + + log.info('Checking inode-block mapping (blocks)...') + + to_delete = list() + for (rowid, block_id, inode) in self.conn.query('SELECT inode_blocks.rowid, block_id, inode FROM inode_blocks ' + 'LEFT JOIN blocks ON block_id = blocks.id ' + 'WHERE blocks.id IS NULL'): + self.found_errors = True + self.log_error('Inode-block mapping for inode %d refers to non-existing block %d', + inode, block_id) + to_delete.append(rowid) + + if inode in self.moved_inodes: + continue + self.moved_inodes.add(inode) + + affected_entries = list(self.conn.query('SELECT name, name_id, parent_inode ' + 'FROM contents_v WHERE inode=?', (inode,))) + for (name, name_id, id_p) in affected_entries: + path = get_path(id_p, self.conn, name) + self.log_error("File may lack data, moved to /lost+found: %s", to_str(path)) + (lof_id, newname) = self.resolve_free(b"/lost+found", escape(path)) + + self.conn.execute('UPDATE contents SET name_id=?, parent_inode=? ' + 'WHERE name_id=? AND parent_inode=?', + (self._add_name(newname), lof_id, name_id, id_p)) + self._del_name(name_id) + + for rowid in to_delete: + self.conn.execute('DELETE FROM inode_blocks WHERE rowid=?', (rowid,)) + + def check_symlinks_inode(self): + """Check symlinks.inode""" + + log.info('Checking symlinks (inodes)...') + + to_delete = list() + for (rowid, inode) in self.conn.query('SELECT symlink_targets.rowid, inode FROM symlink_targets ' + 'LEFT JOIN inodes ON inode = inodes.id WHERE inodes.id IS NULL'): + self.found_errors = True + self.log_error('Symlink %d refers to non-existing inode %d, deleting', + rowid, inode) + to_delete.append(rowid) + + for rowid in to_delete: + self.conn.execute('DELETE FROM symlink_targets WHERE rowid=?', (rowid,)) + + def check_blocks_refcount(self): + """Check blocks.refcount""" + + log.info('Checking blocks (refcounts)...') + + self.conn.execute('CREATE TEMPORARY TABLE refcounts ' + '(id INTEGER PRIMARY KEY, refcount INTEGER NOT NULL)') + try: + self.conn.execute(''' + INSERT INTO refcounts (id, refcount) + SELECT block_id, COUNT(blockno) + FROM inode_blocks + GROUP BY block_id + ''') + + self.conn.execute(''' + CREATE TEMPORARY TABLE wrong_refcounts AS + SELECT id, refcounts.refcount, blocks.refcount, obj_id + FROM blocks LEFT JOIN refcounts USING (id) + WHERE blocks.refcount != refcounts.refcount + OR refcounts.refcount IS NULL''') + + for (id_, cnt, cnt_old, obj_id) in self.conn.query('SELECT * FROM wrong_refcounts'): + if cnt is None and id_ in self.unlinked_blocks: + # Block was unlinked by check_cache and can now really be + # removed (since we have checked that there are truly no + # other references) + self.conn.execute('DELETE FROM blocks WHERE id=?', (id_,)) + + # We can't remove associated objects yet, because their refcounts + # might be wrong, too. + self.conn.execute('UPDATE objects SET refcount=refcount-1 WHERE id=?', (obj_id,)) + self.unlinked_objects.add(obj_id) + + elif cnt is None: + self.found_errors = True + (id_p, name) = self.resolve_free(b"/lost+found", ("block-%d" % id_).encode()) + self.log_error("Block %d not referenced, adding as /lost+found/%s", + id_, to_str(name)) + now_ns = time_ns() + size = self.conn.get_val('SELECT size FROM blocks WHERE id=?', (id_,)) + inode = self.create_inode(mode=stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR, + mtime_ns=now_ns, atime_ns=now_ns, ctime_ns=now_ns, + refcount=1, size=size) + self.conn.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)', + (inode, 0, id_)) + self.conn.execute("INSERT INTO contents (name_id, inode, parent_inode) VALUES (?,?,?)", + (self._add_name(basename(name)), inode, id_p)) + self.conn.execute("UPDATE blocks SET refcount=? WHERE id=?", (1, id_)) + + else: + self.found_errors = True + self.log_error("Block %d has wrong reference count, setting from %d to %d", + id_, cnt_old, cnt) + self.conn.execute("UPDATE blocks SET refcount=? WHERE id=?", (cnt, id_)) + finally: + self.conn.execute('DROP TABLE refcounts') + self.conn.execute('DROP TABLE IF EXISTS wrong_refcounts') + + def check_blocks_checksum(self): + """Check blocks.hash""" + + log.info('Checking blocks (checksums)...') + + for (block_id, obj_id) in list(self.conn.query('SELECT id, obj_id FROM blocks ' + 'WHERE hash IS NULL')): + self.found_errors = True + + # This should only happen when there was an error during upload, + # so the object must not have been stored correctly. We cannot + # just recalculate the hash for the block, because then we may + # be modifying the contents of the inode that refers to this + # block. + self.log_error("No checksum for block %d, removing from table...", block_id) + + # At the moment, we support only one block per object + assert self.conn.get_val('SELECT refcount FROM objects WHERE id=?', + (obj_id,)) == 1 + + self.conn.execute('DELETE FROM blocks WHERE id=?', (block_id,)) + self.conn.execute('DELETE FROM objects WHERE id=?', (obj_id,)) + + def create_inode(self, mode, uid=os.getuid(), gid=os.getgid(), + mtime_ns=None, atime_ns=None, ctime_ns=None, refcount=None, + size=0): + '''Create inode''' + + id_ = self.conn.rowid('INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,' + 'refcount,size) VALUES (?,?,?,?,?,?,?,?)', + (mode, uid, gid, mtime_ns, atime_ns, ctime_ns, refcount, size)) + + return id_ + + def check_names_refcount(self): + """Check names.refcount""" + + log.info('Checking names (refcounts)...') + + self.conn.execute('CREATE TEMPORARY TABLE refcounts ' + '(id INTEGER PRIMARY KEY, refcount INTEGER NOT NULL)') + try: + self.conn.execute('INSERT INTO refcounts (id, refcount) ' + 'SELECT id, 0 FROM names') + self.conn.execute('UPDATE refcounts SET refcount=' + '(SELECT COUNT(name_id) FROM contents WHERE name_id = refcounts.id)' + '+ (SELECT COUNT(name_id) FROM ext_attributes ' + ' WHERE name_id = refcounts.id)') + + self.conn.execute(''' + CREATE TEMPORARY TABLE wrong_refcounts AS + SELECT id, refcounts.refcount, names.refcount + FROM names LEFT JOIN refcounts USING (id) + WHERE names.refcount != refcounts.refcount + OR refcounts.refcount IS NULL''') + + for (id_, cnt, cnt_old) in self.conn.query('SELECT * FROM wrong_refcounts'): + self.found_errors = True + if cnt is None: + self.log_error("Name %d not referenced, removing (old refcount: %d)", + id_, cnt_old) + self.conn.execute('DELETE FROM names WHERE id=?', (id_,)) + else: + self.log_error("Name %d has wrong reference count, setting from %d to %d", + id_, cnt_old, cnt) + self.conn.execute("UPDATE names SET refcount=? WHERE id=?", (cnt, id_)) + finally: + self.conn.execute('DROP TABLE refcounts') + self.conn.execute('DROP TABLE IF EXISTS wrong_refcounts') + + def check_unix(self): + """Check if file systems for agreement with UNIX conventions + + This means: + - Only directories should have child entries + - Only regular files should have data blocks and a size + - Only symlinks should have a target + - Only devices should have a device number + - symlink size is length of target + - names are not longer than 255 bytes + - All directory entries have a valid mode + + Note that none of this is enforced by S3QL. However, as long as S3QL + only communicates with the UNIX FUSE module, none of the above should + happen (and if it does, it would probably confuse the system quite a + lot). + """ + + log.info('Checking unix conventions...') + + for (inode, mode, size, target, rdev) \ + in self.conn.query("SELECT id, mode, size, target, rdev " + "FROM inodes LEFT JOIN symlink_targets ON id = inode"): + + has_children = self.conn.has_val('SELECT 1 FROM contents WHERE parent_inode=? LIMIT 1', + (inode,)) + + if stat.S_IFMT(mode) == 0: + if has_children: + mode = mode | stat.S_IFDIR + made_to = 'directory' + else: + mode = mode | stat.S_IFREG + made_to = 'regular file' + + self.found_errors = True + self.log_error('Inode %d (%s): directory entry has no type, changed ' + 'to %s.', inode, to_str(get_path(inode, self.conn)), made_to) + self.conn.execute('UPDATE inodes SET mode=? WHERE id=?', (mode, inode)) + + if stat.S_ISLNK(mode) and target is None: + self.found_errors = True + self.log_error('Inode %d (%s): symlink does not have target. ' + 'This is probably going to confuse your system!', + inode, to_str(get_path(inode, self.conn))) + + if stat.S_ISLNK(mode) and target is not None and size != len(target): + self.found_errors = True + self.log_error('Inode %d (%s): symlink size (%d) does not agree with target ' + 'length (%d). This is probably going to confuse your system!', + inode, to_str(get_path(inode, self.conn)), size, len(target)) + + if size != 0 and (not stat.S_ISREG(mode) + and not stat.S_ISLNK(mode) + and not stat.S_ISDIR(mode)): + self.found_errors = True + self.log_error('Inode %d (%s) is not regular file but has non-zero size. ' + 'This is may confuse your system!', + inode, to_str(get_path(inode, self.conn))) + + if target is not None and not stat.S_ISLNK(mode): + self.found_errors = True + self.log_error('Inode %d (%s) is not symlink but has symlink target. ' + 'This is probably going to confuse your system!', + inode, to_str(get_path(inode, self.conn))) + + if rdev != 0 and not (stat.S_ISBLK(mode) or stat.S_ISCHR(mode)): + self.found_errors = True + self.log_error('Inode %d (%s) is not device but has device number. ' + 'This is probably going to confuse your system!', + inode, to_str(get_path(inode, self.conn))) + + + if has_children and not stat.S_ISDIR(mode): + self.found_errors = True + self.log_error('Inode %d (%s) is not a directory but has child entries. ' + 'This is probably going to confuse your system!', + inode, to_str(get_path(inode, self.conn))) + + if (not stat.S_ISREG(mode) and + self.conn.has_val('SELECT 1 FROM inode_blocks WHERE inode=?', (inode,))): + self.found_errors = True + self.log_error('Inode %d (%s) is not a regular file but has data blocks. ' + 'This is probably going to confuse your system!', + inode, to_str(get_path(inode, self.conn))) + + + + + + for (name, id_p) in self.conn.query('SELECT name, parent_inode FROM contents_v ' + 'WHERE LENGTH(name) > 255'): + path = get_path(id_p, self.conn, name) + self.log_error('Entry name %s... in %s has more than 255 characters, ' + 'this could cause problems', to_str(name[:40]), + to_str(path[:-len(name)])) + self.found_errors = True + + def check_objects_refcount(self): + """Check objects.refcount""" + + log.info('Checking objects (reference counts)...') + + self.conn.execute('CREATE TEMPORARY TABLE refcounts ' + '(id INTEGER PRIMARY KEY, refcount INTEGER NOT NULL)') + try: + self.conn.execute('INSERT INTO refcounts (id, refcount) ' + 'SELECT obj_id, COUNT(obj_id) FROM blocks GROUP BY obj_id') + + self.conn.execute(''' + CREATE TEMPORARY TABLE wrong_refcounts AS + SELECT id, refcounts.refcount, objects.refcount + FROM objects LEFT JOIN refcounts USING (id) + WHERE objects.refcount != refcounts.refcount + OR refcounts.refcount IS NULL''') + + for (id_, cnt, cnt_old) in self.conn.query('SELECT * FROM wrong_refcounts'): + if cnt is None and id_ in self.unlinked_objects and cnt_old == 0: + # Object was unlinked by check_block_refcounts + self.conn.execute('DELETE FROM objects WHERE id=?', (id_,)) + + else: + self.found_errors = True + self.log_error("Object %s has invalid refcount, setting from %d to %d", + id_, cnt_old, cnt or 0) + + if cnt is not None: + self.conn.execute("UPDATE objects SET refcount=? WHERE id=?", + (cnt, id_)) + else: + # Orphaned object will be picked up by check_keylist + self.conn.execute('DELETE FROM objects WHERE id=?', (id_,)) + finally: + self.conn.execute('DROP TABLE refcounts') + self.conn.execute('DROP TABLE IF EXISTS wrong_refcounts') + + # Delete objects which (correctly had) refcount=0 + for obj_id in self.conn.query('SELECT id FROM objects WHERE refcount = 0'): + del self.backend['s3ql_data_%d' % obj_id] + self.conn.execute("DELETE FROM objects WHERE refcount = 0") + + + def check_objects_temp(self): + """Remove temporary objects""" + + # Tests may provide a plain backend directly, but in regular operation + # we'll always work with a ComprencBackend (even if there is neiter + # compression nor encryption) + if isinstance(self.backend, ComprencBackend): + plain_backend = self.backend.backend + else: + assert isinstance(self.backend, LocalBackend) + plain_backend = self.backend + + if not isinstance(plain_backend, LocalBackend): + return + + log.info('Checking for temporary objects (backend)...') + + for (path, dirnames, filenames) in os.walk(plain_backend.prefix, topdown=True): + for name in filenames: + if not re.search(r'^[^#]+#[0-9]+--?[0-9]+\.tmp$', name): + continue + + self.found_errors = True + self.log_error("removing temporary file %s", name) + os.unlink(os.path.join(path, name)) + + def check_objects_id(self): + """Check objects.id""" + + log.info('Checking objects (backend)...') + + lof_id = self.conn.get_val("SELECT inode FROM contents_v " + "WHERE name=? AND parent_inode=?", (b"lost+found", ROOT_INODE)) + + # We use this table to keep track of the objects that we have seen + if sys.stdout.isatty(): + stamp1 = 0 + else: + stamp1 = float('inf') + self.conn.execute("CREATE TEMP TABLE obj_ids (id INTEGER PRIMARY KEY)") + try: + for (i, obj_name) in enumerate(self.backend.list('s3ql_data_')): + stamp2 = time.time() + if stamp2 - stamp1 > 1: + sys.stdout.write('\r..processed %d objects so far..' % i) + sys.stdout.flush() + stamp1 = stamp2 + + # We only bother with data objects + try: + obj_id = int(obj_name[10:]) + except ValueError: + log.warning("Ignoring unexpected object %r", obj_name) + continue + + self.conn.execute('INSERT INTO obj_ids VALUES(?)', (obj_id,)) + + for (obj_id,) in self.conn.query('SELECT id FROM obj_ids ' + 'EXCEPT SELECT id FROM objects'): + try: + if obj_id in self.unlinked_objects: + del self.backend['s3ql_data_%d' % obj_id] + else: + # TODO: Save the data in lost+found instead + del self.backend['s3ql_data_%d' % obj_id] + self.found_errors = True + self.log_error("Deleted spurious object %d", obj_id) + except NoSuchObject: + pass + + self.conn.execute('CREATE TEMPORARY TABLE missing AS ' + 'SELECT id FROM objects EXCEPT SELECT id FROM obj_ids') + for (obj_id,) in self.conn.query('SELECT * FROM missing'): + if ('s3ql_data_%d' % obj_id) in self.backend: + # Object was just not in list yet + continue + + self.found_errors = True + self.log_error("object %s only exists in table but not in backend, deleting", obj_id) + + for (id_,) in self.conn.query('SELECT inode FROM inode_blocks JOIN blocks ON block_id = id ' + 'WHERE obj_id=? ', (obj_id,)): + + # Same file may lack several blocks, but we want to move it + # only once + if id_ in self.moved_inodes: + continue + self.moved_inodes.add(id_) + + # Copy the list, or we may pick up the same entry again and again + # (first from the original location, then from lost+found) + affected_entries = list(self.conn.query('SELECT name, name_id, parent_inode ' + 'FROM contents_v WHERE inode=?', (id_,))) + for (name, name_id, id_p) in affected_entries: + path = get_path(id_p, self.conn, name) + self.log_error("File may lack data, moved to /lost+found: %s", to_str(path)) + (_, newname) = self.resolve_free(b"/lost+found", escape(path)) + + self.conn.execute('UPDATE contents SET name_id=?, parent_inode=? ' + 'WHERE name_id=? AND parent_inode=?', + (self._add_name(newname), lof_id, name_id, id_p)) + self._del_name(name_id) + + # Unlink missing blocks + for (block_id,) in self.conn.query('SELECT id FROM blocks WHERE obj_id=?', (obj_id,)): + self.conn.execute('DELETE FROM inode_blocks WHERE block_id=?', (block_id,)) + + self.conn.execute("DELETE FROM blocks WHERE obj_id=?", (obj_id,)) + self.conn.execute("DELETE FROM objects WHERE id=?", (obj_id,)) + finally: + if sys.stdout.isatty(): + sys.stdout.write('\n') + + self.conn.execute('DROP TABLE obj_ids') + self.conn.execute('DROP TABLE IF EXISTS missing') + + + def check_objects_size(self): + """Check objects.size""" + + log.info('Checking objects (sizes)...') + + for (obj_id,) in self.conn.query('SELECT id FROM objects WHERE size = -1 OR size IS NULL'): + self.found_errors = True + self.log_error("Object %d has no size information, retrieving from backend...", obj_id) + + self.conn.execute('UPDATE objects SET size=? WHERE id=?', + (self.backend.get_size('s3ql_data_%d' % obj_id), obj_id)) + + + def resolve_free(self, path, name): + '''Return parent inode and name of an unused directory entry + + The directory entry will be in `path`. If an entry `name` already + exists there, we append a numeric suffix. + ''' + + if not isinstance(path, bytes): + raise TypeError('path must be of type bytes') + if not isinstance(name, bytes): + raise TypeError('name must be of type bytes') + + inode_p = inode_for_path(path, self.conn) + + # Debugging http://code.google.com/p/s3ql/issues/detail?id=217 + # and http://code.google.com/p/s3ql/issues/detail?id=261 + if len(name) > 255 - 4: + name = b''.join((name[0:120], b' ... ', name[-120:])) + + i = 0 + newname = name + name += b'-' + try: + while True: + self.conn.get_val("SELECT inode FROM contents_v " + "WHERE name=? AND parent_inode=?", (newname, inode_p)) + i += 1 + newname = name + str(i).encode() + + except NoSuchRowError: + pass + + return (inode_p, newname) + + def _add_name(self, name): + '''Get id for *name* and increase refcount + + Name is inserted in table if it does not yet exist. + ''' + + try: + name_id = self.conn.get_val('SELECT id FROM names WHERE name=?', (name,)) + except NoSuchRowError: + name_id = self.conn.rowid('INSERT INTO names (name, refcount) VALUES(?,?)', + (name, 1)) + else: + self.conn.execute('UPDATE names SET refcount=refcount+1 WHERE id=?', (name_id,)) + return name_id + + def _del_name(self, name_id): + '''Decrease refcount for name_id, remove if it reaches 0''' + + self.conn.execute('UPDATE names SET refcount=refcount-1 WHERE id=?', (name_id,)) + self.conn.execute('DELETE FROM names WHERE refcount=0 AND id=?', (name_id,)) + + +class ROFsck(Fsck): + ''' + Check file system database only, and don't correct any errors. + ''' + + def __init__(self, path): + + db = Connection(path + '.db') + db.execute('PRAGMA journal_mode = WAL') + + param = load_params(path) + super().__init__(None, None, param, db) + + def check(self): + + self.conn.execute('BEGIN TRANSACTION') + try: + log.info('Creating temporary indices...') + for idx in ('tmp1', 'tmp2', 'tmp3', 'tmp4', 'tmp5'): + self.conn.execute('DROP INDEX IF EXISTS %s' % idx) + self.conn.execute('CREATE INDEX tmp1 ON blocks(obj_id)') + self.conn.execute('CREATE INDEX tmp2 ON inode_blocks(block_id)') + self.conn.execute('CREATE INDEX tmp3 ON contents(inode)') + self.conn.execute('CREATE INDEX tmp4 ON contents(name_id)') + self.conn.execute('CREATE INDEX tmp5 ON ext_attributes(name_id)') + + self.check_lof() + self.check_names_refcount() + + self.check_contents_name() + self.check_contents_inode() + self.check_contents_parent_inode() + + self.check_objects_refcount() + self.check_objects_size() + + self.check_blocks_obj_id() + self.check_blocks_refcount() + self.check_blocks_checksum() + + self.check_inode_blocks_block_id() + self.check_inode_blocks_inode() + + self.check_inodes_refcount() + self.check_inodes_size() + + self.check_ext_attributes_name() + self.check_ext_attributes_inode() + + self.check_symlinks_inode() + + self.check_loops() + self.check_unix() + self.check_foreign_keys() + + finally: + log.info('Dropping temporary indices...') + self.conn.execute('ROLLBACK') + + def check_blocks_checksum(self): + """Check blocks.hash""" + + log.info('Checking blocks (checksums)...') + + for (block_id,) in self.conn.query('SELECT id FROM blocks WHERE hash IS NULL'): + self.found_errors = True + self.log_error("No cached checksum for block %d!", block_id) + + def check_objects_size(self): + """Check objects.size""" + + log.info('Checking objects (sizes)...') + + for (obj_id,) in self.conn.query('SELECT id FROM objects WHERE size IS NULL'): + self.found_errors = True + self.log_error("Object %d has no size information!", obj_id) + +def parse_args(args): + + parser = ArgumentParser( + description="Checks and repairs an S3QL filesystem.") + + parser.add_log('~/.s3ql/fsck.log') + parser.add_cachedir() + parser.add_authfile() + parser.add_debug() + parser.add_quiet() + parser.add_backend_options() + parser.add_version() + parser.add_storage_url() + + parser.add_argument("--batch", action="store_true", default=False, + help="If user input is required, exit without prompting.") + parser.add_argument("--force", action="store_true", default=False, + help="Force checking even if file system is marked clean.") + options = parser.parse_args(args) + + return options + +def main(args=None): + + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + # Check if fs is mounted on this computer + # This is not foolproof but should prevent common mistakes + if is_mounted(options.storage_url): + raise QuietError('Can not check mounted file system.', exitcode=40) + + backend = get_backend(options) + atexit.register(backend.close) + + log.info('Starting fsck of %s', options.storage_url) + + cachepath = get_backend_cachedir(options.storage_url, options.cachedir) + seq_no = get_seq_no(backend) + db = None + + if os.path.exists(cachepath + '.params'): + assert os.path.exists(cachepath + '.db') + param = load_params(cachepath) + if param['seq_no'] < seq_no: + log.info('Ignoring locally cached metadata (outdated).') + param = backend.lookup('s3ql_metadata') + else: + log.info('Using cached metadata.') + db = Connection(cachepath + '.db') + assert not os.path.exists(cachepath + '-cache') or param['needs_fsck'] + + if param['seq_no'] > seq_no: + log.warning('File system has not been unmounted cleanly.') + param['needs_fsck'] = True + + elif backend.lookup('s3ql_metadata')['seq_no'] != param['seq_no']: + log.warning('Remote metadata is outdated.') + param['needs_fsck'] = True + + else: + param = backend.lookup('s3ql_metadata') + assert not os.path.exists(cachepath + '-cache') + # .db might exist if mount.s3ql is killed at exactly the right instant + # and should just be ignored. + + # Check revision + if param['revision'] < CURRENT_FS_REV: + raise QuietError('File system revision too old, please run `s3qladm upgrade` first.', + exitcode=32) + elif param['revision'] > CURRENT_FS_REV: + raise QuietError('File system revision too new, please update your ' + 'S3QL installation.', exitcode=33) + + if param['seq_no'] < seq_no: + print(textwrap.fill(textwrap.dedent('''\ + Backend reports that file system is still mounted elsewhere. Either the file system + has not been unmounted cleanly or the data has not yet propagated through the backend. + In the later case, waiting for a while should fix the problem, in the former case you + should try to run fsck on the computer where the file system has been mounted most + recently. + + You may also continue and use whatever metadata is available in the backend. + However, in that case YOU MAY LOOSE ALL DATA THAT HAS BEEN UPLOADED OR MODIFIED + SINCE THE LAST SUCCESSFULL METADATA UPLOAD. Moreover, files and directories that + you have deleted since then MAY REAPPEAR WITH SOME OF THEIR CONTENT LOST. + '''))) + + print('Enter "continue, I know what I am doing" to use the outdated data anyway:', + '> ', sep='\n', end='') + if options.batch: + raise QuietError('(in batch mode, exiting)', exitcode=41) + if sys.stdin.readline().strip() != 'continue, I know what I am doing': + raise QuietError(exitcode=42) + + param['seq_no'] = seq_no + param['needs_fsck'] = True + + if not db and os.path.exists(cachepath + '-cache'): + for i in itertools.count(): + bak_name = '%s-cache.bak%d' % (cachepath, i) + if not os.path.exists(bak_name): + break + log.warning('Found outdated cache directory (%s), renaming to .bak%d', + cachepath + '-cache', i) + log.warning('You should delete this directory once you are sure that ' + 'everything is in order.') + os.rename(cachepath + '-cache', bak_name) + + if (not param['needs_fsck'] + and param['max_inode'] < 2 ** 31 + and (time.time() - param['last_fsck']) + < 60 * 60 * 24 * 31): # last check more than 1 month ago + if options.force: + log.info('File system seems clean, checking anyway.') + else: + log.info('File system is marked as clean. Use --force to force checking.') + return + + # If using local metadata, check consistency + if db: + log.info('Checking DB integrity...') + try: + # get_list may raise CorruptError itself + res = db.get_list('PRAGMA integrity_check(20)') + if res[0][0] != 'ok': + log.error('\n'.join(x[0] for x in res)) + raise apsw.CorruptError() + except apsw.CorruptError: + raise QuietError('Local metadata is corrupted. Remove or repair the following ' + 'files manually and re-run fsck:\n' + + cachepath + '.db (corrupted)\n' + + cachepath + '.param (intact)', exitcode=43) + else: + db = download_metadata(backend, cachepath + '.db') + + # Increase metadata sequence no + param['seq_no'] += 1 + param['needs_fsck'] = True + backend['s3ql_seq_no_%d' % param['seq_no']] = b'Empty' + save_params(cachepath, param) + + fsck = Fsck(cachepath + '-cache', backend, param, db) + fsck.check() + param['max_inode'] = db.get_val('SELECT MAX(id) FROM inodes') + + if fsck.uncorrectable_errors: + raise QuietError("Uncorrectable errors found, aborting.", exitcode=44+128) + + if os.path.exists(cachepath + '-cache'): + os.rmdir(cachepath + '-cache') + + if param['max_inode'] >= 2 ** 31: + renumber_inodes(db) + param['inode_gen'] += 1 + param['max_inode'] = db.get_val('SELECT MAX(id) FROM inodes') + + if fsck.found_errors and not param['needs_fsck']: + log.error('File system was marked as clean, yet fsck found problems.') + log.error('Please report this to the S3QL mailing list, http://groups.google.com/group/s3ql') + + param['needs_fsck'] = False + param['last_fsck'] = time.time() + param['last-modified'] = time.time() + + dump_and_upload_metadata(backend, db, param) + save_params(cachepath, param) + + log.info('Cleaning up local metadata...') + db.execute('ANALYZE') + db.execute('VACUUM') + db.close() + + log.info('Completed fsck of %s', options.storage_url) + + if fsck.found_errors: + sys.exit(128) + else: + sys.exit(0) + +def renumber_inodes(db): + '''Renumber inodes''' + + log.info('Renumbering inodes...') + for table in ('inodes', 'inode_blocks', 'symlink_targets', + 'contents', 'names', 'blocks', 'objects', 'ext_attributes'): + db.execute('ALTER TABLE %s RENAME TO %s_old' % (table, table)) + + for table in ('contents_v', 'ext_attributes_v'): + db.execute('DROP VIEW %s' % table) + + create_tables(db) + for table in ('names', 'blocks', 'objects'): + db.execute('DROP TABLE %s' % table) + db.execute('ALTER TABLE %s_old RENAME TO %s' % (table, table)) + + log.info('..mapping..') + db.execute('CREATE TEMPORARY TABLE inode_map (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER UNIQUE)') + db.execute('INSERT INTO inode_map (rowid, id) VALUES(?,?)', (ROOT_INODE, ROOT_INODE)) + db.execute('INSERT INTO inode_map (rowid, id) VALUES(?,?)', (CTRL_INODE, CTRL_INODE)) + db.execute('INSERT INTO inode_map (id) SELECT id FROM inodes_old WHERE id > ? ORDER BY id ASC', + (CTRL_INODE,)) + + log.info('..inodes..') + db.execute('INSERT INTO inodes (id,mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size,locked,rdev) ' + 'SELECT (SELECT rowid FROM inode_map WHERE inode_map.id = inodes_old.id), ' + ' mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size,locked,rdev FROM inodes_old') + + log.info('..inode_blocks..') + db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) ' + 'SELECT (SELECT rowid FROM inode_map WHERE inode_map.id = inode_blocks_old.inode), ' + ' blockno, block_id FROM inode_blocks_old') + + log.info('..contents..') + db.execute('INSERT INTO contents (inode, parent_inode, name_id) ' + 'SELECT (SELECT rowid FROM inode_map WHERE inode_map.id = contents_old.inode), ' + ' (SELECT rowid FROM inode_map WHERE inode_map.id = contents_old.parent_inode), ' + ' name_id FROM contents_old') + + log.info('..symlink_targets..') + db.execute('INSERT INTO symlink_targets (inode, target) ' + 'SELECT (SELECT rowid FROM inode_map WHERE inode_map.id = symlink_targets_old.inode), ' + ' target FROM symlink_targets_old') + + log.info('..ext_attributes..') + db.execute('INSERT INTO ext_attributes (inode, name_id, value) ' + 'SELECT (SELECT rowid FROM inode_map WHERE inode_map.id = ext_attributes_old.inode), ' + ' name_id, value FROM ext_attributes_old') + + for table in ('inodes', 'inode_blocks', 'symlink_targets', + 'contents', 'ext_attributes'): + db.execute('DROP TABLE %s_old' % table) + + db.execute('DROP TABLE inode_map') + + +def escape(path): + '''Escape slashes in path so that is usable as a file name''' + + return path[1:].replace(b'_', b'__').replace(b'/', b'_') + +def to_str(name): + '''Decode path name for printing''' + + return str(name, encoding='utf-8', errors='replace') + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/src/s3ql/inherit_docstrings.py b/src/s3ql/inherit_docstrings.py new file mode 100644 index 0000000..de8f3a7 --- /dev/null +++ b/src/s3ql/inherit_docstrings.py @@ -0,0 +1,120 @@ +''' +inherit_docstrings.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. + +--- + +This module defines a metaclass and function decorator that allows +to inherit the docstring for a function from the superclass. +''' + + +from functools import partial +from abc import ABCMeta +from .calc_mro import calc_mro + +__all__ = [ 'copy_ancestor_docstring', 'prepend_ancestor_docstring', + 'InheritableDocstrings', 'ABCDocstMeta' ] + +# This definition is only used to assist static code analyzers +def copy_ancestor_docstring(fn): + '''Copy docstring for method from superclass + + For this decorator to work, the class has to use the `InheritableDocstrings` + metaclass. + ''' + raise RuntimeError('Decorator can only be used in classes ' + 'using the `InheritableDocstrings` metaclass') + +def _copy_ancestor_docstring(mro, fn): + '''Decorator to set docstring for *fn* from *mro*''' + + if fn.__doc__ is not None: + raise RuntimeError('Function already has docstring') + + # Search for docstring in superclass + for cls in mro: + super_fn = getattr(cls, fn.__name__, None) + if super_fn is None: + continue + fn.__doc__ = super_fn.__doc__ + break + else: + raise RuntimeError("Can't inherit docstring for %s: method does not " + "exist in superclass" % fn.__name__) + + return fn + +# This definition is only used to assist static code analyzers +def prepend_ancestor_docstring(fn): + '''Prepend docstring from superclass method + + For this decorator to work, the class has to use the `InheritableDocstrings` + metaclass. + ''' + raise RuntimeError('Decorator can only be used in classes ' + 'using the `InheritableDocstrings` metaclass') + +def _prepend_ancestor_docstring(mro, fn): + '''Decorator to prepend ancestor docstring to *fn*''' + + if fn.__doc__ is None: + fn.__doc__ = '' + + # Search for docstring in superclass + for cls in mro: + super_fn = getattr(cls, fn.__name__, None) + if super_fn is None: + continue + if super_fn.__doc__.endswith('\n') and fn.__doc__.startswith('\n'): + fn.__doc__ = super_fn.__doc__ + fn.__doc__ + else: + fn.__doc__ = '%s\n%s' % (super_fn.__doc__, fn.__doc__) + break + else: + raise RuntimeError("Can't find ancestor docstring for %s: method does not " + "exist in superclass" % fn.__name__) + + return fn + +DECORATORS = (('copy_ancestor_docstring', _copy_ancestor_docstring), + ('prepend_ancestor_docstring', _prepend_ancestor_docstring)) + +class InheritableDocstrings(type): + + @classmethod + def __prepare__(cls, name, bases, **kwds): + classdict = super().__prepare__(name, bases, *kwds) + mro = calc_mro(*bases) + + # Inject decorators into class namespace + for (name, fn) in DECORATORS: + classdict[name] = partial(fn, mro) + + return classdict + + def __new__(cls, name, bases, classdict): + for (dec_name, fn) in DECORATORS: + # Decorators may not exist in class dict if the class (metaclass + # instance) was constructed with an explicit call to `type` + # (Pythonbug? reported as http://bugs.python.org/issue18334) + if dec_name not in classdict: + continue + + # Make sure that class definition hasn't messed with decorator + if getattr(classdict[dec_name], 'func', None) is not fn: + raise RuntimeError('No %s attribute may be created in classes using ' + 'the InheritableDocstrings metaclass' % name) + + + # Delete decorator from class namespace + del classdict[dec_name] + + return super().__new__(cls, name, bases, classdict) + +# Derive new metaclass to add docstring inheritance +class ABCDocstMeta(ABCMeta, InheritableDocstrings): + pass diff --git a/src/s3ql/inode_cache.py b/src/s3ql/inode_cache.py new file mode 100644 index 0000000..77c8985 --- /dev/null +++ b/src/s3ql/inode_cache.py @@ -0,0 +1,271 @@ +''' +inode_cache.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging # Ensure use of custom logger class +from .database import NoSuchRowError +import llfuse +import sys + +log = logging.getLogger(__name__) + +CACHE_SIZE = 100 +ATTRIBUTES = ('mode', 'refcount', 'uid', 'gid', 'size', 'locked', + 'rdev', 'atime_ns', 'mtime_ns', 'ctime_ns', 'id') +ATTRIBUTE_STR = ', '.join(ATTRIBUTES) +UPDATE_ATTRS = ('mode', 'refcount', 'uid', 'gid', 'size', 'locked', + 'rdev', 'atime_ns', 'mtime_ns', 'ctime_ns') +UPDATE_STR = ', '.join('%s=?' % x for x in UPDATE_ATTRS) + +MAX_INODE = 2 ** 32 - 1 + +class _Inode: + '''An inode with its attributes''' + + __slots__ = ATTRIBUTES + ('dirty', 'generation') + + def __init__(self, generation): + super().__init__() + self.dirty = False + self.generation = generation + + def entry_attributes(self): + attr = llfuse.EntryAttributes() + attr.st_nlink = self.refcount + attr.st_blocks = (self.size + 511) // 512 + attr.st_ino = self.id + + # Timeout, can effectively be infinite since attribute changes + # are only triggered by the kernel's own requests + attr.attr_timeout = 3600 + attr.entry_timeout = 3600 + + # We want our blocksize for IO as large as possible to get large + # write requests + attr.st_blksize = 128 * 1024 + + attr.st_mode = self.mode + attr.st_uid = self.uid + attr.st_gid = self.gid + attr.st_size = self.size + attr.st_rdev = self.rdev + attr.st_atime_ns = self.atime_ns + attr.st_mtime_ns = self.mtime_ns + attr.st_ctime_ns = self.ctime_ns + attr.generation = self.generation + + return attr + + def __eq__(self, other): + # Ill defined - should we compare the inode id or all the attributes? + # What does it even mean to have the same id but different attributes? + # Maybe we should we raise an Exception in that case? + return NotImplemented + + def __hash__(self): + return self.id + + def copy(self): + copy = _Inode(self.generation) + + for attr in ATTRIBUTES: + setattr(copy, attr, getattr(self, attr)) + + return copy + + def __setattr__(self, name, value): + if name != 'dirty': + object.__setattr__(self, 'dirty', True) + object.__setattr__(self, name, value) + + def __del__(self): + if not self.dirty: + return + + # Force execution of sys.excepthook (exceptions raised + # by __del__ are ignored) + try: + raise RuntimeError('BUG ALERT: Dirty inode was destroyed!') + except RuntimeError: + exc_info = sys.exc_info() + + sys.excepthook(*exc_info) + +class InodeCache(object): + ''' + This class maps the `inode` SQL table to a dict, caching the rows. + + If the cache is full and a row is not in the cache, the least-recently + retrieved row is deleted from the cache. This means that accessing + cached rows will *not* change the order of their expiration. + + Attributes: + ----------- + :attrs: inode indexed dict holding the attributes + :cached_rows: list of the inodes that are in cache + :pos: position of the most recently retrieved inode in + 'cached_rows'. + + Notes + ----- + + Callers should keep in mind that the changes of the returned inode + object will only be written to the database if the inode is still + in the cache when its attributes are updated: it is possible for + the caller to keep a reference to an inode when that + inode has already been expired from the InodeCache. Modifications + to this inode object will be lost(!). + + Callers should therefore use the returned inode objects only + as long as they can guarantee that no other calls to InodeCache + are made that may result in expiration of inodes from the cache. + + Moreover, the caller must make sure that he does not call + InodeCache methods while a database transaction is active that + may be rolled back. This would rollback database updates + performed by InodeCache, which are generally for inodes that + are expired from the cache and therefore *not* directly related + to the effects of the current method call. + ''' + + def __init__(self, db, inode_gen): + self.attrs = dict() + self.cached_rows = list() + self.db = db + self.generation = inode_gen + + # Fill the cache with dummy data, so that we don't have to + # check if the cache is full or not (it will always be full) + for _ in range(CACHE_SIZE): + self.cached_rows.append(None) + + self.pos = 0 + + + def __delitem__(self, inode): + if self.db.execute('DELETE FROM inodes WHERE id=?', (inode,)) != 1: + raise KeyError('No such inode') + inode = self.attrs.pop(inode, None) + if inode is not None: + inode.dirty = False + + def __getitem__(self, id_): + try: + return self.attrs[id_] + except KeyError: + try: + inode = self.getattr(id_) + except NoSuchRowError: + raise KeyError('No such inode: %d' % id_) + + old_id = self.cached_rows[self.pos] + self.cached_rows[self.pos] = id_ + self.pos = (self.pos + 1) % CACHE_SIZE + if old_id is not None: + try: + old_inode = self.attrs[old_id] + except KeyError: + # We may have deleted that inode + pass + else: + del self.attrs[old_id] + self.setattr(old_inode) + self.attrs[id_] = inode + return inode + + def getattr(self, id_): #@ReservedAssignment + attrs = self.db.get_row("SELECT %s FROM inodes WHERE id=? " % ATTRIBUTE_STR, + (id_,)) + inode = _Inode(self.generation) + + for (i, id_) in enumerate(ATTRIBUTES): + setattr(inode, id_, attrs[i]) + + inode.dirty = False + + return inode + + def create_inode(self, **kw): + + bindings = tuple(kw[x] for x in ATTRIBUTES if x in kw) + columns = ', '.join(x for x in ATTRIBUTES if x in kw) + values = ', '.join('?' * len(kw)) + + id_ = self.db.rowid('INSERT INTO inodes (%s) VALUES(%s)' % (columns, values), + bindings) + if id_ > MAX_INODE - 1: + self.db.execute('DELETE FROM inodes WHERE id=?', (id_,)) + raise OutOfInodesError() + + return self[id_] + + + def setattr(self, inode): + if not inode.dirty: + return + inode.dirty = False + + self.db.execute("UPDATE inodes SET %s WHERE id=?" % UPDATE_STR, + [ getattr(inode, x) for x in UPDATE_ATTRS ] + [inode.id]) + + def flush_id(self, id_): + if id_ in self.attrs: + self.setattr(self.attrs[id_]) + + def destroy(self): + '''Flush all entries and empty cache''' + + for i in range(len(self.cached_rows)): + id_ = self.cached_rows[i] + self.cached_rows[i] = None + if id_ is not None: + try: + inode = self.attrs[id_] + except KeyError: + # We may have deleted that inode + pass + else: + del self.attrs[id_] + self.setattr(inode) + + assert len(self.attrs) == 0 + + def flush(self): + '''Flush all entries to database''' + + # We don't want to use dict.itervalues() since + # the dict may change while we iterate + for i in range(len(self.cached_rows)): + id_ = self.cached_rows[i] + if id_ is not None: + try: + inode = self.attrs[id_] + except KeyError: + # We may have deleted that inode + pass + else: + self.setattr(inode) + + def __del__(self): + if len(self.attrs) == 0: + return + + # Force execution of sys.excepthook (exceptions raised + # by __del__ are ignored) + try: + raise RuntimeError('InodeCache instance was destroyed without calling destroy()') + except RuntimeError: + exc_info = sys.exc_info() + + sys.excepthook(*exc_info) + + + +class OutOfInodesError(Exception): + + def __str__(self): + return 'Could not find free rowid in inode table' diff --git a/src/s3ql/lock.py b/src/s3ql/lock.py new file mode 100644 index 0000000..645e321 --- /dev/null +++ b/src/s3ql/lock.py @@ -0,0 +1,57 @@ +''' +lock.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging, setup_logging, QuietError +from .common import assert_fs_owner +from .parse_args import ArgumentParser +import llfuse +import os +import sys +import textwrap + +log = logging.getLogger(__name__) + +def parse_args(args): + '''Parse command line''' + + parser = ArgumentParser( + description=textwrap.dedent('''\ + Makes the given directory tree(s) immutable. No changes of any sort can + be performed on the tree after that. Immutable entries can only be + deleted with s3qlrm. + ''')) + + parser.add_debug() + parser.add_quiet() + parser.add_version() + + parser.add_argument('path', metavar='', nargs='+', + help='Directories to make immutable.', + type=(lambda x: x.rstrip('/'))) + + return parser.parse_args(args) + + +def main(args=None): + '''Make directory tree immutable''' + + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + for name in options.path: + if os.path.ismount(name): + raise QuietError('%s is a mount point.' % name) + ctrlfile = assert_fs_owner(name) + fstat = os.stat(name) + llfuse.setxattr(ctrlfile, 'lock', ('%d' % fstat.st_ino).encode()) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/src/s3ql/logging.py b/src/s3ql/logging.py new file mode 100644 index 0000000..b66482e --- /dev/null +++ b/src/s3ql/logging.py @@ -0,0 +1,177 @@ +''' +logging.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +import logging +import logging.handlers +import warnings +import sys +import os.path + +class QuietError(Exception): + ''' + QuietError is the base class for exceptions that should not result + in a stack trace being printed. + + It is typically used for exceptions that are the result of the user + supplying invalid input data. The exception argument should be a + string containing sufficient information about the problem. + ''' + + def __init__(self, msg='', exitcode=1): + super().__init__() + self.msg = msg + + #: Exit code to use when terminating process + self.exitcode = exitcode + + def __str__(self): + return self.msg + +class MyFormatter(logging.Formatter): + '''Prepend severity to log message if it exceeds threshold''' + + def format(self, record): + s = super().format(record) + if record.levelno > logging.INFO: + s = '%s: %s' % (record.levelname, s) + return s + +def create_handler(target): + '''Create logging handler for given target''' + + if target.lower() == 'syslog': + handler = logging.handlers.SysLogHandler('/dev/log') + formatter = logging.Formatter(os.path.basename(sys.argv[0]) + + '[%(process)s:%(threadName)s] ' + + '%(name)s.%(funcName)s: %(message)s') + + else: + fullpath = os.path.expanduser(target) + dirname = os.path.dirname(fullpath) + if dirname and not os.path.exists(dirname): + try: + os.makedirs(dirname) + except PermissionError: + raise QuietError('No permission to create log file %s' % fullpath, + exitcode=10) + + try: + handler = logging.handlers.RotatingFileHandler(fullpath, + maxBytes=10 * 1024**2, backupCount=5) + except PermissionError: + raise QuietError('No permission to write log file %s' % fullpath, + exitcode=10) + + formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(process)s:%(threadName)s ' + '%(name)s.%(funcName)s: %(message)s', + datefmt="%Y-%m-%d %H:%M:%S") + + handler.setFormatter(formatter) + return handler + +def setup_logging(options): + + # We want to be able to detect warnings and higher severities + # in the captured test output. 'critical' has too many potential + # false positives, so we rename this level to "FATAL". + logging.addLevelName(logging.CRITICAL, 'FATAL') + + root_logger = logging.getLogger() + if root_logger.handlers: + root_logger.debug("Logging already initialized.") + return + + stdout_handler = add_stdout_logging(options.quiet) + if hasattr(options, 'log') and options.log: + root_logger.addHandler(create_handler(options.log)) + elif options.debug and (not hasattr(options, 'log') or not options.log): + # When we have debugging enabled but no separate log target, + # make stdout logging more detailed. + formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(process)s %(levelname)-8s ' + '%(threadName)s %(name)s.%(funcName)s: %(message)s', + datefmt="%Y-%m-%d %H:%M:%S") + stdout_handler.setFormatter(formatter) + stdout_handler.setLevel(logging.NOTSET) + + setup_excepthook() + + if options.debug: + if 'all' in options.debug: + root_logger.setLevel(logging.DEBUG) + else: + root_logger.setLevel(logging.INFO) + for module in options.debug: + logging.getLogger(module).setLevel(logging.DEBUG) + + logging.disable(logging.NOTSET) + else: + root_logger.setLevel(logging.INFO) + logging.disable(logging.DEBUG) + logging.captureWarnings(capture=True) + + return stdout_handler + +def setup_excepthook(): + '''Modify sys.excepthook to log exceptions + + Also makes sure that exceptions derived from `QuietException` + do not result in stacktraces. + ''' + + def excepthook(type_, val, tb): + root_logger = logging.getLogger() + if isinstance(val, QuietError): + root_logger.error(val.msg) + sys.exit(val.exitcode) + else: + root_logger.error('Uncaught top-level exception:', + exc_info=(type_, val, tb)) + sys.exit(1) + + sys.excepthook = excepthook + +def add_stdout_logging(quiet=False): + '''Add stdout logging handler to root logger''' + + root_logger = logging.getLogger() + formatter = MyFormatter('%(message)s') + handler = logging.StreamHandler(sys.stderr) + if quiet: + handler.setLevel(logging.WARNING) + else: + handler.setLevel(logging.INFO) + handler.setFormatter(formatter) + root_logger.addHandler(handler) + return handler + +class Logger(logging.getLoggerClass()): + ''' + This class has the following features in addition to `logging.Logger`: + + * Log messages that are emitted with an *log_once* attribute in the + *extra* parameter are only emitted once per logger. + ''' + + def __init__(self, name): + super().__init__(name) + self.log_cache = set() + + def handle(self, record): + if hasattr(record, 'log_once') and record.log_once: + id_ = hash((record.name, record.levelno, record.msg, + record.args, record.exc_info)) + if id_ in self.log_cache: + return + self.log_cache.add(id_) + + return super().handle(record) +logging.setLoggerClass(Logger) + +# Convenience object for use in logging calls, e.g. +# log.warning('This will be printed only once', extra=LOG_ONCE) +LOG_ONCE = { 'log_once': True } diff --git a/src/s3ql/metadata.py b/src/s3ql/metadata.py new file mode 100644 index 0000000..7e0d2f2 --- /dev/null +++ b/src/s3ql/metadata.py @@ -0,0 +1,324 @@ +''' +metadata.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging # Ensure use of custom logger class +from .database import Connection +from . import BUFSIZE +from .common import pretty_print_size +from .deltadump import INTEGER, BLOB, dump_table, load_table +from .backends.common import NoSuchObject, CorruptedObjectError +import os +import tempfile +import bz2 +import stat + +log = logging.getLogger(__name__) + +# Has to be kept in sync with create_tables()! +DUMP_SPEC = [ + ('objects', 'id', (('id', INTEGER, 1), + ('size', INTEGER), + ('refcount', INTEGER))), + + ('blocks', 'id', (('id', INTEGER, 1), + ('hash', BLOB, 32), + ('size', INTEGER), + ('obj_id', INTEGER, 1), + ('refcount', INTEGER))), + + ('inodes', 'id', (('id', INTEGER, 1), + ('uid', INTEGER), + ('gid', INTEGER), + ('mode', INTEGER), + ('mtime_ns', INTEGER), + ('atime_ns', INTEGER), + ('ctime_ns', INTEGER), + ('size', INTEGER), + ('rdev', INTEGER), + ('locked', INTEGER), + ('refcount', INTEGER))), + + ('inode_blocks', 'inode, blockno', + (('inode', INTEGER), + ('blockno', INTEGER, 1), + ('block_id', INTEGER, 1))), + + ('symlink_targets', 'inode', (('inode', INTEGER, 1), + ('target', BLOB))), + + ('names', 'id', (('id', INTEGER, 1), + ('name', BLOB), + ('refcount', INTEGER))), + + ('contents', 'parent_inode, name_id', + (('name_id', INTEGER, 1), + ('inode', INTEGER, 1), + ('parent_inode', INTEGER))), + + ('ext_attributes', 'inode', (('inode', INTEGER), + ('name_id', INTEGER), + ('value', BLOB))), +] + + + +def restore_metadata(fh, dbfile): + '''Read metadata from *fh* and write into *dbfile* + + Return database connection to *dbfile*. + + *fh* must be able to return an actual file descriptor from + its `fileno` method. + + *dbfile* will be created with 0600 permissions. Data is + first written into a temporary file *dbfile* + '.tmp', and + the file is renamed once all data has been loaded. + ''' + + tmpfile = dbfile + '.tmp' + fd = os.open(tmpfile, os.O_RDWR | os.O_CREAT | os.O_TRUNC, + stat.S_IRUSR | stat.S_IWUSR) + try: + os.close(fd) + + db = Connection(tmpfile) + db.execute('PRAGMA locking_mode = NORMAL') + db.execute('PRAGMA synchronous = OFF') + db.execute('PRAGMA journal_mode = OFF') + create_tables(db) + + for (table, _, columns) in DUMP_SPEC: + log.info('..%s..', table) + load_table(table, columns, db=db, fh=fh) + db.execute('ANALYZE') + + # We must close the database to rename it + db.close() + except: + os.unlink(tmpfile) + raise + + os.rename(tmpfile, dbfile) + + return Connection(dbfile) + +def cycle_metadata(backend, keep=10): + '''Rotate metadata backups''' + + # Since we always overwrite the source afterwards, we can + # use either copy or rename - so we pick whatever is faster. + if backend.has_native_rename: + cycle_fn = backend.rename + else: + cycle_fn = backend.copy + + log.info('Backing up old metadata...') + for i in range(keep)[::-1]: + try: + cycle_fn("s3ql_metadata_bak_%d" % i, "s3ql_metadata_bak_%d" % (i + 1)) + except NoSuchObject: + pass + + # However, the current metadata object should always be copied, + # so that even if there's a crash we don't end up without it + try: + backend.copy("s3ql_metadata", "s3ql_metadata_bak_0") + except NoSuchObject: + # In case of mkfs, there may be no metadata object yet + pass + cycle_fn("s3ql_metadata_new", "s3ql_metadata") + + if cycle_fn is backend.copy: + backend.delete('s3ql_metadata_new') + +def dump_metadata(db, fh): + '''Dump metadata into fh + + *fh* must be able to return an actual file descriptor from + its `fileno` method. + ''' + + locking_mode = db.get_val('PRAGMA locking_mode') + try: + # Ensure that we don't hold a lock on the db + # (need to access DB to actually release locks) + db.execute('PRAGMA locking_mode = NORMAL') + db.has_val('SELECT rowid FROM %s LIMIT 1' % DUMP_SPEC[0][0]) + + for (table, order, columns) in DUMP_SPEC: + log.info('..%s..', table) + dump_table(table, order, columns, db=db, fh=fh) + + finally: + db.execute('PRAGMA locking_mode = %s' % locking_mode) + + +def create_tables(conn): + # Table of storage objects + # Refcount is included for performance reasons + # size == -1 indicates block has not been uploaded yet + conn.execute(""" + CREATE TABLE objects ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + refcount INT NOT NULL, + size INT NOT NULL + )""") + + # Table of known data blocks + # Refcount is included for performance reasons + conn.execute(""" + CREATE TABLE blocks ( + id INTEGER PRIMARY KEY, + hash BLOB(16) UNIQUE, + refcount INT, + size INT NOT NULL, + obj_id INTEGER NOT NULL REFERENCES objects(id) + )""") + + # Table with filesystem metadata + # The number of links `refcount` to an inode can in theory + # be determined from the `contents` table. However, managing + # this separately should be significantly faster (the information + # is required for every getattr!) + conn.execute(""" + CREATE TABLE inodes ( + -- id has to specified *exactly* as follows to become + -- an alias for the rowid. + id INTEGER PRIMARY KEY AUTOINCREMENT, + uid INT NOT NULL, + gid INT NOT NULL, + mode INT NOT NULL, + mtime_ns INT NOT NULL, + atime_ns INT NOT NULL, + ctime_ns INT NOT NULL, + refcount INT NOT NULL, + size INT NOT NULL DEFAULT 0, + rdev INT NOT NULL DEFAULT 0, + locked BOOLEAN NOT NULL DEFAULT 0 + )""") + + # Further Blocks used by inode (blockno >= 1) + conn.execute(""" + CREATE TABLE inode_blocks ( + inode INTEGER NOT NULL REFERENCES inodes(id), + blockno INT NOT NULL, + block_id INTEGER NOT NULL REFERENCES blocks(id), + PRIMARY KEY (inode, blockno) + )""") + + # Symlinks + conn.execute(""" + CREATE TABLE symlink_targets ( + inode INTEGER PRIMARY KEY REFERENCES inodes(id), + target BLOB NOT NULL + )""") + + # Names of file system objects + conn.execute(""" + CREATE TABLE names ( + id INTEGER PRIMARY KEY, + name BLOB NOT NULL, + refcount INT NOT NULL, + UNIQUE (name) + )""") + + # Table of filesystem objects + # rowid is used by readdir() to restart at the correct position + conn.execute(""" + CREATE TABLE contents ( + rowid INTEGER PRIMARY KEY AUTOINCREMENT, + name_id INT NOT NULL REFERENCES names(id), + inode INT NOT NULL REFERENCES inodes(id), + parent_inode INT NOT NULL REFERENCES inodes(id), + + UNIQUE (parent_inode, name_id) + )""") + + # Extended attributes + conn.execute(""" + CREATE TABLE ext_attributes ( + inode INTEGER NOT NULL REFERENCES inodes(id), + name_id INTEGER NOT NULL REFERENCES names(id), + value BLOB NOT NULL, + + PRIMARY KEY (inode, name_id) + )""") + + # Shortcuts + conn.execute(""" + CREATE VIEW contents_v AS + SELECT * FROM contents JOIN names ON names.id = name_id + """) + conn.execute(""" + CREATE VIEW ext_attributes_v AS + SELECT * FROM ext_attributes JOIN names ON names.id = name_id + """) + +def stream_write_bz2(ifh, ofh): + '''Compress *ifh* into *ofh* using bz2 compression''' + + compr = bz2.BZ2Compressor(9) + while True: + buf = ifh.read(BUFSIZE) + if not buf: + break + buf = compr.compress(buf) + if buf: + ofh.write(buf) + buf = compr.flush() + if buf: + ofh.write(buf) + +def stream_read_bz2(ifh, ofh): + '''Uncompress bz2 compressed *ifh* into *ofh*''' + + decompressor = bz2.BZ2Decompressor() + while True: + buf = ifh.read(BUFSIZE) + if not buf: + break + buf = decompressor.decompress(buf) + if buf: + ofh.write(buf) + + if decompressor.unused_data or ifh.read(1) != b'': + raise CorruptedObjectError('Data after end of bz2 stream') + +def download_metadata(backend, db_file, name='s3ql_metadata'): + with tempfile.TemporaryFile() as tmpfh: + def do_read(fh): + tmpfh.seek(0) + tmpfh.truncate() + stream_read_bz2(fh, tmpfh) + + log.info('Downloading and decompressing metadata...') + backend.perform_read(do_read, name) + + log.info("Reading metadata...") + tmpfh.seek(0) + return restore_metadata(tmpfh, db_file) + +def dump_and_upload_metadata(backend, db, param): + with tempfile.TemporaryFile() as fh: + log.info('Dumping metadata...') + dump_metadata(db, fh) + upload_metadata(backend, fh, param) + +def upload_metadata(backend, fh, param): + log.info("Compressing and uploading metadata...") + def do_write(obj_fh): + fh.seek(0) + stream_write_bz2(fh, obj_fh) + return obj_fh + obj_fh = backend.perform_write(do_write, "s3ql_metadata_new", + metadata=param, is_compressed=True) + log.info('Wrote %s of compressed metadata.', + pretty_print_size(obj_fh.get_obj_size())) + + log.info('Cycling metadata backups...') + cycle_metadata(backend) diff --git a/src/s3ql/mkfs.py b/src/s3ql/mkfs.py new file mode 100644 index 0000000..310a78a --- /dev/null +++ b/src/s3ql/mkfs.py @@ -0,0 +1,184 @@ +''' +mkfs.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging, setup_logging, QuietError +from . import CURRENT_FS_REV, CTRL_INODE, ROOT_INODE +from .backends.comprenc import ComprencBackend +from .backends import s3 +from .common import (get_backend_cachedir, get_backend, split_by_n, + freeze_basic_mapping, time_ns) +from .database import Connection +from .metadata import dump_and_upload_metadata, create_tables +from .parse_args import ArgumentParser +from getpass import getpass +from base64 import b64encode +import os +import shutil +import stat +import sys +import time +import atexit + +log = logging.getLogger(__name__) + +def parse_args(args): + + parser = ArgumentParser( + description="Initializes an S3QL file system") + + parser.add_cachedir() + parser.add_authfile() + parser.add_debug() + parser.add_quiet() + parser.add_backend_options() + parser.add_version() + parser.add_storage_url() + + parser.add_argument("-L", default='', help="Filesystem label", + dest="label", metavar='',) + parser.add_argument("--max-obj-size", type=int, default=10240, metavar='', + help="Maximum size of storage objects in KiB. Files bigger than this " + "will be spread over multiple objects in the storage backend. " + "Default: %(default)d KiB.") + parser.add_argument("--plain", action="store_true", default=False, + help="Create unencrypted file system.") + parser.add_argument("--force", action="store_true", default=False, + help="Overwrite any existing data.") + + options = parser.parse_args(args) + + return options + +def init_tables(conn): + # Insert root directory + now_ns = time_ns() + conn.execute("INSERT INTO inodes (id,mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) " + "VALUES (?,?,?,?,?,?,?,?)", + (ROOT_INODE, stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR + | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, + os.getuid(), os.getgid(), now_ns, now_ns, now_ns, 1)) + + # Insert control inode, the actual values don't matter that much + conn.execute("INSERT INTO inodes (id,mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) " + "VALUES (?,?,?,?,?,?,?,?)", + (CTRL_INODE, stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, now_ns, now_ns, now_ns, 42)) + + # Insert lost+found directory + inode = conn.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) " + "VALUES (?,?,?,?,?,?,?)", + (stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR, + os.getuid(), os.getgid(), now_ns, now_ns, now_ns, 1)) + name_id = conn.rowid('INSERT INTO names (name, refcount) VALUES(?,?)', + (b'lost+found', 1)) + conn.execute("INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)", + (name_id, inode, ROOT_INODE)) + +def main(args=None): + + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + if options.max_obj_size < 1024: + # This warning should never be converrted to an exception + log.warning('Maximum object sizes less than 1 MiB will degrade ' + 'performance.', extra={ 'force_log': True }) + + plain_backend = get_backend(options, raw=True) + atexit.register(plain_backend.close) + + log.info("Before using S3QL, make sure to read the user's guide, especially\n" + "the 'Important Rules to Avoid Loosing Data' section.") + + if isinstance(plain_backend, s3.Backend) and '.' in plain_backend.bucket_name: + log.warning('S3 Buckets with names containing dots cannot be ' + 'accessed using SSL!' + '(cf. https://forums.aws.amazon.com/thread.jspa?threadID=130560)') + + if 's3ql_metadata' in plain_backend: + if not options.force: + raise QuietError("Found existing file system! Use --force to overwrite") + + log.info('Purging existing file system data..') + plain_backend.clear() + log.info('Please note that the new file system may appear inconsistent\n' + 'for a while until the removals have propagated through the backend.') + + if not options.plain: + if sys.stdin.isatty(): + wrap_pw = getpass("Enter encryption password: ") + if not wrap_pw == getpass("Confirm encryption password: "): + raise QuietError("Passwords don't match.") + else: + wrap_pw = sys.stdin.readline().rstrip() + wrap_pw = wrap_pw.encode('utf-8') + + # Generate data encryption passphrase + log.info('Generating random encryption key...') + fh = open('/dev/urandom', "rb", 0) # No buffering + data_pw = fh.read(32) + fh.close() + + backend = ComprencBackend(wrap_pw, ('lzma', 2), plain_backend) + backend['s3ql_passphrase'] = data_pw + backend['s3ql_passphrase_bak1'] = data_pw + backend['s3ql_passphrase_bak2'] = data_pw + backend['s3ql_passphrase_bak3'] = data_pw + else: + data_pw = None + + backend = ComprencBackend(data_pw, ('lzma', 2), plain_backend) + atexit.unregister(plain_backend.close) + atexit.register(backend.close) + + # Setup database + cachepath = get_backend_cachedir(options.storage_url, options.cachedir) + + # There can't be a corresponding backend, so we can safely delete + # these files. + if os.path.exists(cachepath + '.db'): + os.unlink(cachepath + '.db') + if os.path.exists(cachepath + '-cache'): + shutil.rmtree(cachepath + '-cache') + + log.info('Creating metadata tables...') + db = Connection(cachepath + '.db') + create_tables(db) + init_tables(db) + + param = dict() + param['revision'] = CURRENT_FS_REV + param['seq_no'] = 1 + param['label'] = options.label + param['max_obj_size'] = options.max_obj_size * 1024 + param['needs_fsck'] = False + param['inode_gen'] = 0 + param['max_inode'] = db.get_val('SELECT MAX(id) FROM inodes') + param['last_fsck'] = time.time() + param['last-modified'] = time.time() + + log.info('Dumping metadata...') + dump_and_upload_metadata(backend, db, param) + backend.store('s3ql_seq_no_%d' % param['seq_no'], b'Empty') + with open(cachepath + '.params', 'wb') as fh: + fh.write(freeze_basic_mapping(param)) + + if data_pw is not None: + print('Please store the following master key in a safe location. It allows ', + 'decryption of the S3QL file system in case the storage objects holding ', + 'this information get corrupted:', + '---BEGIN MASTER KEY---', + ' '.join(split_by_n(b64encode(data_pw).decode(), 4)), + '---END MASTER KEY---', + sep='\n') + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/src/s3ql/mount.py b/src/s3ql/mount.py new file mode 100644 index 0000000..20a6e87 --- /dev/null +++ b/src/s3ql/mount.py @@ -0,0 +1,757 @@ +''' +mount.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging, setup_logging, QuietError +from . import fs, CURRENT_FS_REV +from .backends.pool import BackendPool +from .block_cache import BlockCache +from .common import (get_backend_cachedir, get_seq_no, get_backend_factory, + load_params, save_params) +from .daemonize import daemonize +from .database import Connection +from .inode_cache import InodeCache +from .metadata import (download_metadata, upload_metadata, dump_and_upload_metadata, + dump_metadata) +from .parse_args import ArgumentParser +from .exit_stack import ExitStack +from threading import Thread +import _thread +import argparse +import faulthandler +import llfuse +import itertools +import os +import platform +import subprocess +import re +import signal +import resource +import sys +import tempfile +import threading +import time +import shutil +import atexit + +try: + from systemd.daemon import notify as sd_notify +except ImportError: + sd_notify = None + +log = logging.getLogger(__name__) + +def install_thread_excepthook(): + """work around sys.excepthook thread bug + + See http://bugs.python.org/issue1230540. + + Call once from __main__ before creating any threads. If using + psyco, call psyco.cannotcompile(threading.Thread.run) since this + replaces a new-style class method. + """ + + init_old = threading.Thread.__init__ + def init(self, *args, **kwargs): + init_old(self, *args, **kwargs) + run_old = self.run + def run_with_except_hook(*args, **kw): + try: + run_old(*args, **kw) + except SystemExit: + raise + except: + sys.excepthook(*sys.exc_info()) + self.run = run_with_except_hook + + threading.Thread.__init__ = init +install_thread_excepthook() + +def main(args=None): + '''Mount S3QL file system''' + + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + + # Save handler so that we can remove it when daemonizing + stdout_log_handler = setup_logging(options) + + if not os.path.exists(options.mountpoint): + raise QuietError('Mountpoint does not exist.', exitcode=36) + + if options.threads is None: + options.threads = determine_threads(options) + + avail_fd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] + if avail_fd == resource.RLIM_INFINITY: + avail_fd = 4096 + resource.setrlimit(resource.RLIMIT_NOFILE, (avail_fd, avail_fd)) + + # Subtract some fd's for random things we forgot, and a fixed number for + # each upload thread (because each thread is using at least one socket and + # at least one temporary file) + avail_fd -= 32 + 3 * options.threads + + if options.max_cache_entries is None: + if avail_fd <= 64: + raise QuietError("Not enough available file descriptors.", + exitcode=37) + log.info('Autodetected %d file descriptors available for cache entries', + avail_fd) + options.max_cache_entries = avail_fd + else: + if options.max_cache_entries > avail_fd: + log.warning("Up to %d cache entries requested, but detected only %d " + "available file descriptors.", options.max_cache_entries, avail_fd) + options.max_cache_entries = avail_fd + + if options.profile: + import cProfile + import pstats + prof = cProfile.Profile() + + backend_factory = get_backend_factory(options.storage_url, options.backend_options, + options.authfile, options.compress) + backend_pool = BackendPool(backend_factory) + atexit.register(backend_pool.flush) + + # Get paths + cachepath = get_backend_cachedir(options.storage_url, options.cachedir) + + # Retrieve metadata + with backend_pool() as backend: + (param, db) = get_metadata(backend, cachepath) + + #if param['max_obj_size'] < options.min_obj_size: + # raise QuietError('Maximum object size must be bigger than minimum object size.', + # exitcode=2) + + # Handle --cachesize + rec_cachesize = options.max_cache_entries * param['max_obj_size'] / 2 + avail_cache = shutil.disk_usage(os.path.dirname(cachepath))[2] / 1024 + if options.cachesize is None: + options.cachesize = min(rec_cachesize, 0.8 * avail_cache) + log.info('Setting cache size to %d MB', options.cachesize / 1024) + elif options.cachesize > avail_cache: + log.warning('Requested cache size %d MB, but only %d MB available', + options.cachesize / 1024, avail_cache / 1024) + + if options.nfs: + # NFS may try to look up '..', so we have to speed up this kind of query + log.info('Creating NFS indices...') + db.execute('CREATE INDEX IF NOT EXISTS ix_contents_inode ON contents(inode)') + + else: + db.execute('DROP INDEX IF EXISTS ix_contents_inode') + + metadata_upload_thread = MetadataUploadThread(backend_pool, param, db, + options.metadata_upload_interval) + block_cache = BlockCache(backend_pool, db, cachepath + '-cache', + options.cachesize * 1024, options.max_cache_entries) + commit_thread = CommitThread(block_cache) + operations = fs.Operations(block_cache, db, max_obj_size=param['max_obj_size'], + inode_cache=InodeCache(db, param['inode_gen']), + upload_event=metadata_upload_thread.event) + metadata_upload_thread.fs = operations + + with ExitStack() as cm: + log.info('Mounting %s at %s...', options.storage_url, options.mountpoint) + try: + llfuse.init(operations, options.mountpoint, get_fuse_opts(options)) + except RuntimeError as exc: + raise QuietError(str(exc), exitcode=39) + + unmount_clean = False + def unmount(): + log.info("Unmounting file system...") + llfuse.close(unmount=unmount_clean) + cm.callback(unmount) + + if options.fg: + faulthandler.enable() + faulthandler.register(signal.SIGUSR1) + else: + if stdout_log_handler: + logging.getLogger().removeHandler(stdout_log_handler) + global crit_log_fh + crit_log_fh = open(os.path.join(options.cachedir, 'mount.s3ql_crit.log'), 'a') + faulthandler.enable(crit_log_fh) + faulthandler.register(signal.SIGUSR1, file=crit_log_fh) + daemonize(options.cachedir) + + mark_metadata_dirty(backend, cachepath, param) + + block_cache.init(options.threads) + cm.callback(block_cache.destroy) + + metadata_upload_thread.start() + cm.callback(metadata_upload_thread.join) + cm.callback(metadata_upload_thread.stop) + + commit_thread.start() + cm.callback(commit_thread.join) + cm.callback(commit_thread.stop) + + if options.upstart: + os.kill(os.getpid(), signal.SIGSTOP) + if sd_notify is not None: + sd_notify('READY=1') + sd_notify('MAINPID=%d' % os.getpid()) + + exc_info = setup_exchook() + workers = 1 if options.single else None # use default + + if options.profile: + prof.runcall(llfuse.main, workers) + else: + llfuse.main(workers) + + # Allow operations to terminate while block_cache is still available + # (destroy() will be called again when from llfuse.close(), but at that + # point the block cache is no longer available). + with llfuse.lock: + operations.destroy() + + # Re-raise if main loop terminated due to exception in other thread + if exc_info: + (exc_inst, exc_tb) = exc_info + raise exc_inst.with_traceback(exc_tb) + + log.info("FUSE main loop terminated.") + + unmount_clean = True + + # At this point, there should be no other threads left + + # Do not update .params yet, dump_metadata() may fail if the database is + # corrupted, in which case we want to force an fsck. + param['max_inode'] = db.get_val('SELECT MAX(id) FROM inodes') + if operations.failsafe: + log.warning('File system errors encountered, marking for fsck.') + param['needs_fsck'] = True + with backend_pool() as backend: + seq_no = get_seq_no(backend) + if metadata_upload_thread.db_mtime == os.stat(cachepath + '.db').st_mtime: + log.info('File system unchanged, not uploading metadata.') + del backend['s3ql_seq_no_%d' % param['seq_no']] + param['seq_no'] -= 1 + save_params(cachepath, param) + elif seq_no == param['seq_no']: + param['last-modified'] = time.time() + dump_and_upload_metadata(backend, db, param) + save_params(cachepath, param) + else: + log.error('Remote metadata is newer than local (%d vs %d), ' + 'refusing to overwrite!', seq_no, param['seq_no']) + log.error('The locally cached metadata will be *lost* the next time the file system ' + 'is mounted or checked and has therefore been backed up.') + for name in (cachepath + '.params', cachepath + '.db'): + for i in range(4)[::-1]: + if os.path.exists(name + '.%d' % i): + os.rename(name + '.%d' % i, name + '.%d' % (i + 1)) + os.rename(name, name + '.0') + + log.info('Cleaning up local metadata...') + db.execute('ANALYZE') + db.execute('VACUUM') + db.close() + + if options.profile: + with tempfile.NamedTemporaryFile() as tmp, \ + open('s3ql_profile.txt', 'w') as fh: + prof.dump_stats(tmp.name) + p = pstats.Stats(tmp.name, stream=fh) + p.strip_dirs() + p.sort_stats('cumulative') + p.print_stats(50) + p.sort_stats('time') + p.print_stats(50) + + log.info('All done.') + + +def get_system_memory(): + '''Attempt to determine total system memory + + If amount cannot be determined, emits warning and + returns -1. + ''' + + # MacOS X doesn't support sysconf('SC_PHYS_PAGES') + if platform.system() == 'Darwin': + try: + out = subprocess.check_output(['sysctl', 'hw.memsize'], + universal_newlines=True) + except subprocess.CalledProcessError as exc: + log.warning('Cannot determine system memory, sysctl failed with %s', + exc.output) + return -1 + + # output of sysctl is 'hw.memsize: #'. Strip the prefix. + hit = re.match(r'^hw.memsize: ([0-9]+)$', out) + if not hit: + log.warning('Cannot determine system memory, unable to parse sysctl output.') + return -1 + + return int(hit.group(1)) + + else: + try: + return os.sysconf('SC_PHYS_PAGES') * os.sysconf('SC_PAGESIZE') + except ValueError: + log.warning('Unable to determine number of CPU cores (sysconf failed).') + return -1 + + +# Memory required for LZMA compression in MB (from xz(1)) +LZMA_MEMORY = { 0: 3, 1: 9, 2: 17, 3: 32, 4: 48, + 5: 94, 6: 94, 7: 186, 8: 370, 9: 674 } +def determine_threads(options): + '''Return optimum number of upload threads''' + + try: + cores = os.sysconf('SC_NPROCESSORS_ONLN') + except ValueError: + log.warning('Unable to determine number of CPU cores (sysconf failed).') + cores = -1 + + memory = get_system_memory() + + if options.compress[0] == 'lzma': + # Keep this in sync with compression level in backends/common.py + # Memory usage according to man xz(1) + mem_per_thread = LZMA_MEMORY[options.compress[1]] * 1024 ** 2 + else: + # Only check LZMA memory usage + mem_per_thread = 0 + + if cores == -1: + log.warning("Can't determine number of cores, using 2 upload threads.") + return 2 + elif memory == -1 and mem_per_thread != 0: + log.warning("Can't determine available memory, using 2 upload threads.") + return 2 + elif 2 * cores * mem_per_thread > (memory / 2): + threads = min(int((memory / 2) // mem_per_thread), 10) + if threads > 0: + log.info('Using %d upload threads (memory limited).', threads) + else: + log.warning('Compression will require %d MiB memory ' + '(%d%% of total system memory', mem_per_thread / 1024 ** 2, + mem_per_thread * 100 / memory) + threads = 1 + return threads + else: + threads = min(2 * cores, 10) + log.info("Using %d upload threads.", threads) + return threads + +def get_metadata(backend, cachepath): + '''Retrieve metadata''' + + seq_no = get_seq_no(backend) + + # Check for cached metadata + db = None + if os.path.exists(cachepath + '.params'): + param = load_params(cachepath) + if param['seq_no'] < seq_no: + log.info('Ignoring locally cached metadata (outdated).') + param = backend.lookup('s3ql_metadata') + elif param['seq_no'] > seq_no: + raise QuietError("File system not unmounted cleanly, run fsck!", + exitcode=30) + else: + log.info('Using cached metadata.') + db = Connection(cachepath + '.db') + else: + param = backend.lookup('s3ql_metadata') + + # Check for unclean shutdown + if param['seq_no'] < seq_no: + raise QuietError('Backend reports that fs is still mounted elsewhere, aborting.', + exitcode=31) + + # Check revision + if param['revision'] < CURRENT_FS_REV: + raise QuietError('File system revision too old, please run `s3qladm upgrade` first.', + exitcode=32) + elif param['revision'] > CURRENT_FS_REV: + raise QuietError('File system revision too new, please update your ' + 'S3QL installation.', exitcode=33) + + # Check that the fs itself is clean + if param['needs_fsck']: + raise QuietError("File system damaged or not unmounted cleanly, run fsck!", + exitcode=30) + if time.time() - param['last_fsck'] > 60 * 60 * 24 * 31: + log.warning('Last file system check was more than 1 month ago, ' + 'running fsck.s3ql is recommended.') + + if param['max_inode'] > 2 ** 32 - 50000: + raise QuietError('Insufficient free inodes, fsck run required.', + exitcode=34) + elif param['max_inode'] > 2 ** 31: + log.warning('Few free inodes remaining, running fsck is recommended') + + if os.path.exists(cachepath + '-cache'): + for i in itertools.count(): + bak_name = '%s-cache.bak%d' % (cachepath, i) + if not os.path.exists(bak_name): + break + log.warning('Found outdated cache directory (%s), renaming to .bak%d', + cachepath + '-cache', i) + log.warning('You should delete this directory once you are sure that ' + 'everything is in order.') + os.rename(cachepath + '-cache', bak_name) + + # Download metadata + if not db: + db = download_metadata(backend, cachepath + '.db') + save_params(cachepath, param) + + return (param, db) + +def mark_metadata_dirty(backend, cachepath, param): + '''Mark metadata as dirty and increase sequence number''' + + param['seq_no'] += 1 + param['needs_fsck'] = True + save_params(cachepath, param) + backend['s3ql_seq_no_%d' % param['seq_no']] = b'Empty' + param['needs_fsck'] = False + +def get_fuse_opts(options): + '''Return fuse options for given command line options''' + + fuse_opts = [ "nonempty", 'fsname=%s' % options.storage_url, + 'subtype=s3ql', 'big_writes', 'max_write=131072', + 'no_remote_lock' ] + + if platform.system() == 'Darwin': + # FUSE4X and OSXFUSE claim to support nonempty, but + # neither of them actually do. + fuse_opts.remove('nonempty') + + if options.allow_other: + fuse_opts.append('allow_other') + if options.allow_root: + fuse_opts.append('allow_root') + if options.allow_other or options.allow_root: + fuse_opts.append('default_permissions') + + return fuse_opts + + +def parse_args(args): + '''Parse command line''' + + # Parse fstab-style -o options + if '--' in args: + max_idx = args.index('--') + else: + max_idx = len(args) + if '-o' in args[:max_idx]: + pos = args.index('-o') + val = args[pos + 1] + del args[pos] + del args[pos] + for opt in reversed(val.split(',')): + if '=' in opt: + (key, val) = opt.split('=') + args.insert(pos, val) + args.insert(pos, '--' + key) + else: + if opt in ('rw', 'defaults', 'auto', 'noauto', 'user', 'nouser', 'dev', 'nodev', + 'suid', 'nosuid', 'atime', 'diratime', 'exec', 'noexec', 'group', + 'mand', 'nomand', '_netdev', 'nofail', 'norelatime', 'strictatime', + 'owner', 'users', 'nobootwait'): + continue + elif opt == 'ro': + raise QuietError('Read-only mounting not supported.', + exitcode=35) + args.insert(pos, '--' + opt) + + def compression_type(s): + hit = re.match(r'^([a-z0-9]+)(?:-([0-9]))?$', s) + if not hit: + raise argparse.ArgumentTypeError('%s is not a valid --compress value' % s) + alg = hit.group(1) + lvl = hit.group(2) + if alg not in ('none', 'zlib', 'bzip2', 'lzma'): + raise argparse.ArgumentTypeError('Invalid compression algorithm: %s' % alg) + if lvl is None: + lvl = 6 + else: + lvl = int(lvl) + if alg == 'none': + alg = None + return (alg, lvl) + + parser = ArgumentParser( + description="Mount an S3QL file system.") + + parser.add_log('~/.s3ql/mount.log') + parser.add_cachedir() + parser.add_authfile() + parser.add_debug() + parser.add_quiet() + parser.add_backend_options() + parser.add_version() + parser.add_storage_url() + + parser.add_argument("mountpoint", metavar='', type=os.path.abspath, + help='Where to mount the file system') + parser.add_argument("--cachesize", type=int, default=None, metavar='', + help="Cache size in KiB (default: autodetect).") + parser.add_argument("--max-cache-entries", type=int, default=None, metavar='', + help="Maximum number of entries in cache (default: autodetect). " + 'Each cache entry requires one file descriptor, so if you increase ' + 'this number you have to make sure that your process file descriptor ' + 'limit (as set with `ulimit -n`) is high enough (at least the number ' + 'of cache entries + 100).') + parser.add_argument("--allow-other", action="store_true", default=False, help= + 'Normally, only the user who called `mount.s3ql` can access the mount ' + 'point. This user then also has full access to it, independent of ' + 'individual file permissions. If the `--allow-other` option is ' + 'specified, other users can access the mount point as well and ' + 'individual file permissions are taken into account for all users.') + parser.add_argument("--allow-root", action="store_true", default=False, + help='Like `--allow-other`, but restrict access to the mounting ' + 'user and the root user.') + parser.add_argument("--fg", action="store_true", default=False, + help="Do not daemonize, stay in foreground") + parser.add_argument("--upstart", action="store_true", default=False, + help="Stay in foreground and raise SIGSTOP once mountpoint " + "is up.") + parser.add_argument("--compress", action="store", default='lzma-6', + metavar='', type=compression_type, + help="Compression algorithm and compression level to use when " + "storing new data. *algorithm* may be any of `lzma`, `bzip2`, " + "`zlib`, or none. *lvl* may be any integer from 0 (fastest) " + "to 9 (slowest). Default: `%(default)s`") + parser.add_argument("--metadata-upload-interval", action="store", type=int, + default=24 * 60 * 60, metavar='', + help='Interval in seconds between complete metadata uploads. ' + 'Set to 0 to disable. Default: 24h.') + parser.add_argument("--threads", action="store", type=int, + default=None, metavar='', + help='Number of parallel upload threads to use (default: auto).') + parser.add_argument("--nfs", action="store_true", default=False, + help='Enable some optimizations for exporting the file system ' + 'over NFS. (default: %(default)s)') + parser.add_argument("--single", action="store_true", default=False, + help=argparse.SUPPRESS) + parser.add_argument("--profile", action="store_true", default=False, + help=argparse.SUPPRESS) + + # Not yet implemented. When implementing this, don't forget to + # uncomment check against param['max_obj_size'] in main(). + #parser.add_argument("--min-obj-size", type=int, default=512, metavar='', + # help="Minimum size of storage objects in KiB. Files smaller than this " + # "may be combined into groups that are stored as single objects " + # "in the storage backend. Default: %(default)d KB.") + + options = parser.parse_args(args) + + if options.allow_other and options.allow_root: + parser.error("--allow-other and --allow-root are mutually exclusive.") + + if not options.log and not options.fg: + parser.error("Please activate logging to a file or syslog, or use the --fg option.") + + if options.profile: + options.single = True + + if options.upstart: + options.fg = True + + if options.metadata_upload_interval == 0: + options.metadata_upload_interval = None + + return options + +class MetadataUploadThread(Thread): + ''' + Periodically upload metadata. Upload is done every `interval` + seconds, and whenever `event` is set. To terminate thread, + set `quit` attribute as well as `event` event. + + This class uses the llfuse global lock. When calling objects + passed in the constructor, the global lock is acquired first. + ''' + + def __init__(self, backend_pool, param, db, interval): + super().__init__() + self.backend_pool = backend_pool + self.param = param + self.db = db + self.interval = interval + self.daemon = True + self.db_mtime = os.stat(db.file).st_mtime + self.event = threading.Event() + self.quit = False + self.name = 'Metadata-Upload-Thread' + + # Can't assign in constructor, because Operations instance needs + # access to self.event as well. + self.fs = None + + def run(self): + log.debug('started') + + assert self.fs is not None + + while not self.quit: + self.event.wait(self.interval) + self.event.clear() + + if self.quit: + break + + with llfuse.lock: + if self.quit: + break + new_mtime = os.stat(self.db.file).st_mtime + if self.db_mtime == new_mtime: + log.info('File system unchanged, not uploading metadata.') + continue + + log.info('Dumping metadata...') + fh = tempfile.TemporaryFile() + dump_metadata(self.db, fh) + + with self.backend_pool() as backend: + seq_no = get_seq_no(backend) + if seq_no > self.param['seq_no']: + log.error('Remote metadata is newer than local (%d vs %d), ' + 'refusing to overwrite and switching to failsafe mode!', + seq_no, self.param['seq_no']) + self.fs.failsafe = True + fh.close() + break + + fh.seek(0) + self.param['last-modified'] = time.time() + + # Temporarily decrease sequence no, this is not the final upload + self.param['seq_no'] -= 1 + upload_metadata(backend, fh, self.param) + self.param['seq_no'] += 1 + + fh.close() + self.db_mtime = new_mtime + + # Break reference loop + self.fs = None + + log.debug('finished') + + def stop(self): + '''Signal thread to terminate''' + + log.debug('started') + self.quit = True + self.event.set() + +def setup_exchook(): + '''Send SIGTERM if any other thread terminates with an exception + + The exc_info will be saved in the list object returned + by this function. + ''' + + main_thread = _thread.get_ident() + old_exchook = sys.excepthook + exc_info = [] + + def exchook(exc_type, exc_inst, tb): + reporting_thread = _thread.get_ident() + if reporting_thread != main_thread: + if exc_info: + log.warning("Unhandled top-level exception during shutdown " + "(will not be re-raised)") + else: + log.debug('recording exception %s', exc_inst) + os.kill(os.getpid(), signal.SIGTERM) + exc_info.append(exc_inst) + exc_info.append(tb) + old_exchook(exc_type, exc_inst, tb) + + # If the main thread re-raised exception, there is no need to call + # excepthook again + elif exc_info and exc_info[0] is exc_inst: + log.debug('Suppressing exception hook for re-raised %s', exc_inst) + else: + old_exchook(exc_type, exc_inst, tb) + + sys.excepthook = exchook + + return exc_info + + +class CommitThread(Thread): + ''' + Periodically upload dirty blocks. + + This class uses the llfuse global lock. When calling objects + passed in the constructor, the global lock is acquired first. + ''' + + + def __init__(self, block_cache): + super().__init__() + self.block_cache = block_cache + self.stop_event = threading.Event() + self.name = 'CommitThread' + + def run(self): + log.debug('started') + + while not self.stop_event.is_set(): + did_sth = False + + with llfuse.lock: + stamp = time.time() + # Need to make copy, since we aren't allowed to change + # dict while iterating through it. The performance hit doesn't seem + # to be that bad: + # >>> from timeit import timeit + # >>> timeit("k=0\nfor el in list(d.values()):\n k += el", + # ... setup='\nfrom collections import OrderedDict\nd = OrderedDict()\nfor i in range(5000):\n d[i]=i\n', + # ... number=500)/500 * 1e3 + # 1.3769531380003173 + # >>> timeit("k=0\nfor el in d.values(n:\n k += el", + # ... setup='\nfrom collections import OrderedDict\nd = OrderedDict()\nfor i in range(5000):\n d[i]=i\n', + # ... number=500)/500 * 1e3 + # 1.456586996000624 + for el in list(self.block_cache.cache.values()): + if self.stop_event.is_set(): + break + if stamp - el.last_access < 10: + break + if not el.dirty or el in self.block_cache.in_transit: + continue + + self.block_cache.upload(el) + did_sth = True + + if not did_sth: + self.stop_event.wait(5) + + log.debug('finished') + + def stop(self): + '''Signal thread to terminate''' + + log.debug('started') + self.stop_event.set() + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/src/s3ql/multi_lock.py b/src/s3ql/multi_lock.py new file mode 100644 index 0000000..59b23a8 --- /dev/null +++ b/src/s3ql/multi_lock.py @@ -0,0 +1,67 @@ +''' +multi_lock.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +import threading +import logging +from contextlib import contextmanager + +__all__ = [ "MultiLock" ] + +log = logging.getLogger(__name__) + +class MultiLock: + """Provides locking for multiple objects. + + This class provides locking for a dynamically changing set of objects: The + `acquire` and `release` methods have an additional argument, the locking + key. Only locks with the same key can actually see each other, so that + several threads can hold locks with different locking keys at the same time. + + MultiLock instances can be used as context managers. + + Note that it is actually possible for one thread to release a lock that has + been obtained by a different thread. This is not a bug but a feature. + """ + + def __init__(self): + self.locked_keys = set() + self.cond = threading.Condition(threading.Lock()) + + @contextmanager + def __call__(self, *key): + self.acquire(*key) + try: + yield + finally: + self.release(*key) + + def acquire(self, *key, timeout=None): + '''Acquire lock for given key + + If timeout is exceeded, return False. Otherwise return True. + ''' + + with self.cond: + if not self.cond.wait_for(lambda: key not in self.locked_keys, timeout): + return False + + self.locked_keys.add(key) + + def release(self, *key, noerror=False): + """Release lock on given key + + If noerror is False, do not raise exception if *key* is + not locked. + """ + + with self.cond: + if noerror: + self.locked_keys.discard(key) + else: + self.locked_keys.remove(key) + self.cond.notifyAll() diff --git a/src/s3ql/oauth_client.py b/src/s3ql/oauth_client.py new file mode 100644 index 0000000..30c5da0 --- /dev/null +++ b/src/s3ql/oauth_client.py @@ -0,0 +1,106 @@ +''' +oauth_client.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging, setup_logging, QuietError +from .parse_args import ArgumentParser +import sys +import textwrap +import requests +import time + +log = logging.getLogger(__name__) + +# S3QL client id and client secret for Google APIs. +# Don't get your hopes up, this isn't truly secret. +CLIENT_ID = '381875429714-6pch5vnnmqab454c68pkt8ugm86ef95v.apps.googleusercontent.com' +CLIENT_SECRET = 'HGl8fJeVML-gZ-1HSZRNZPz_' + +def parse_args(args): + '''Parse command line''' + + parser = ArgumentParser( + description=textwrap.dedent('''\ + Obtain OAuth2 refresh token for Google Storage + ''')) + + parser.add_debug() + parser.add_quiet() + parser.add_version() + + return parser.parse_args(args) + +def _log_response(r): + '''Log server response''' + + if not log.isEnabledFor(logging.DEBUG): + return + + s = [ 'Server response:', + '%03d %s' % (r.status_code, r.reason) ] + for tup in r.headers.items(): + s.append('%s: %s' % tup) + + s.append('') + s.append(r.text) + + log.debug('\n'.join(s)) + +def _parse_response(r): + + _log_response(r) + if r.status_code != requests.codes.ok: + raise QuietError('Connection failed with: %d %s' + % (r.status_code, r.reason)) + + return r.json() + +def main(args=None): + + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + cli = requests.Session() + + # We need full control in order to be able to update metadata + # cf. https://stackoverflow.com/questions/24718787 + r = cli.post('https://accounts.google.com/o/oauth2/device/code', + data={ 'client_id': CLIENT_ID, + 'scope': 'https://www.googleapis.com/auth/devstorage.full_control' }, + verify=True, allow_redirects=False, timeout=20) + req_json = _parse_response(r) + + print(textwrap.fill('Please open %s in your browser and enter the following ' + 'user code: %s' % (req_json['verification_url'], + req_json['user_code']))) + + while True: + log.debug('polling..') + time.sleep(req_json['interval']) + + r = cli.post('https://accounts.google.com/o/oauth2/token', + data={ 'client_id': CLIENT_ID, + 'client_secret': CLIENT_SECRET, + 'code': req_json['device_code'], + 'grant_type': 'http://oauth.net/grant_type/device/1.0' }, + verify=True, allow_redirects=False, timeout=20) + resp_json = _parse_response(r) + r.close() + + if 'error' in resp_json: + if resp_json['error'] == 'authorization_pending': + continue + else: + raise QuietError('Authentication failed: ' + resp_json['error']) + else: + break + + print('Success. Your refresh token is:\n', + resp_json['refresh_token']) diff --git a/src/s3ql/parse_args.py b/src/s3ql/parse_args.py new file mode 100644 index 0000000..163b997 --- /dev/null +++ b/src/s3ql/parse_args.py @@ -0,0 +1,257 @@ +''' +argparse.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. + +This module provides a customized ArgumentParser class. Differences +are: + + * a --version argument is added by default + + * convenience functions are available for adding --quiet, + --debug, --cachedir, --log and --authfile options. + + * instead of the usage string one can pass a usage list. The first + element will be prefixed with ``usage: `` as usual. Additional + elements will be printed on separate lines and prefixed with + `` or: ``. + + * When element of an usage list, the ``DEFAULT_USAGE`` object + will be replaced by the automatically generated usage message, + excluding any --help arguments. + + * When specified on its own, the replacement will be done including + any --help arguments. + + * The ``usage`` and ``add_help`` settings are inherited from the + parent parser to the subparsers. + +''' + +# Pylint really gets confused by this module +#pylint: disable-all + + +from . import RELEASE +from argparse import ArgumentTypeError, ArgumentError +import argparse +from .logging import logging # Ensure use of custom logger class +import os +import re + +DEFAULT_USAGE = object() +log = logging.getLogger(__name__) + +class HelpFormatter(argparse.HelpFormatter): + + def _format_usage(self, usage, actions, groups, prefix): + '''Special handling for usage lists + + If usage is a list object, its elements will be printed on + separate lines. DEFAULT_USAGE will be replaced by the + default usage string of the parser (but, if `usage`` is a list, + excluding any --help arguments)). + ''' + + if isinstance(usage, list): + # Omit help argument + actions = [ x for x in actions if not isinstance(x, argparse._HelpAction) ] + res = [] + for s in usage: + if not res: + res.append('usage: ') + else: + res.append(' or: ') + if s is DEFAULT_USAGE: + res.append(super()._format_usage(None, actions, groups, '')[:-1]) + else: + res.append(s % dict(prog=self._prog)) + res.append('\n') + + return '%s\n\n' % ''.join(res) + + elif usage is DEFAULT_USAGE: + return super()._format_usage(None, actions, groups, prefix) + else: + return super()._format_usage(usage, actions, groups, prefix) + + def format_help(self): + help_ = super().format_help() + if help_.count('\n') > 2: + return help_ + '\n' + else: + return help_ + + +class SubParsersAction(argparse._SubParsersAction): + '''A replacement for _SubParsersAction that keeps + track of the parent parser''' + + def __init__(self, **kw): + self.parent = kw.pop('parent') + super().__init__(**kw) + + def add_parser(self, *a, **kwargs): + '''Pass parent usage and add_help attributes to new parser''' + + if 'usage' not in kwargs: + # Inherit, but preserve old progs attribute + usage = self.parent.usage + repl = dict(prog=self.parent.prog) + if isinstance(usage, list): + usage = [ (x % repl if isinstance(x, str) else x) + for x in usage ] + elif usage: + usage = usage % repl + kwargs['usage'] = usage + + if 'help' in kwargs: + kwargs.setdefault('description', kwargs['help'].capitalize() + '.') + + kwargs.setdefault('add_help', self.parent.add_help) + kwargs.setdefault('formatter_class', self.parent.formatter_class) + + if 'parents' in kwargs: + for p in kwargs['parents']: + if p.epilog: + kwargs.setdefault('epilog', p.epilog % dict(prog=self.parent.prog)) + + return super().add_parser(*a, **kwargs) + + +class ArgumentParser(argparse.ArgumentParser): + + def __init__(self, *a, **kw): + if 'formatter_class' not in kw: + kw['formatter_class'] = HelpFormatter + + super().__init__(*a, **kw) + self.register('action', 'parsers', SubParsersAction) + + def add_version(self): + self.add_argument('--version', action='version', + help="just print program version and exit", + version='S3QL %s' % RELEASE) + + def add_quiet(self): + self.add_argument("--quiet", action="store_true", default=False, + help="be really quiet") + + def add_backend_options(self): + self.add_argument("--backend-options", default={}, type=suboptions_type, + metavar='', + help="Backend specific options (separate by commas). See " + "backend documentation for available options.") + + def add_debug(self): + destnote = ('Debug messages will be written to the target ' + 'specified by the ``--log`` option.') + self.add_argument("--debug-modules", metavar='', + type=lambda s: s.split(','), dest='debug', + help="Activate debugging output from specified modules " + "(use commas to separate multiple modules). " + + destnote) + self.add_argument("--debug", action='append_const', const='s3ql', + help="Activate debugging output from all S3QL modules. " + + destnote) + + def add_authfile(self): + self.add_argument("--authfile", type=str, metavar='', + default=os.path.expanduser("~/.s3ql/authinfo2"), + help='Read authentication credentials from this file ' + '(default: `~/.s3ql/authinfo2)`') + + def add_cachedir(self): + self.add_argument("--cachedir", type=str, metavar='', + default=os.path.expanduser("~/.s3ql"), + help='Store cached data in this directory ' + '(default: `~/.s3ql)`') + + def add_log(self, default=None): + self.add_argument("--log", type=str_or_None_type, metavar='', default=default, + help='Destination for log messages. Specify ``none`` for standard ' + 'output or ``syslog`` for the system logging daemon. ' + 'Anything else will be interpreted as a file name. Log files ' + 'will be rotated when they reach 1 MiB, and at most 5 old log ' + 'files will be kept. Default: ``%(default)s``') + + def add_storage_url(self): + self.add_argument("storage_url", metavar='', + type=storage_url_type, + help='Storage URL of the backend that contains the file system') + + def add_subparsers(self, **kw): + '''Pass parent and set prog to default usage message''' + kw.setdefault('parser_class', argparse.ArgumentParser) + + kw['parent'] = self + + # prog defaults to the usage message of this parser, skipping + # optional arguments and with no "usage:" prefix + if kw.get('prog') is None: + formatter = self._get_formatter() + positionals = self._get_positional_actions() + groups = self._mutually_exclusive_groups + formatter.add_usage(None, positionals, groups, '') + kw['prog'] = formatter.format_help().strip() + + return super().add_subparsers(**kw) + + def parse_args(self, *args, **kwargs): + + try: + return super().parse_args(*args, **kwargs) + except ArgumentError as exc: + self.exit(str(exc)) + +def storage_url_type(s): + '''Validate and canonicalize storage url''' + + if not re.match(r'^([a-zA-Z0-9]+)://(.+)$', s): + raise ArgumentTypeError('%s is not a valid storage url.' % s) + + if s.startswith('local://'): + # Append trailing slash so that we can match patterns with + # trailing slash in authinfo2 file. + return 'local://%s/' % os.path.abspath(s[len('local://'):]) + + # If there is no prefix specified, then e.g. s3://foo and s3://foo/ point to + # the same location (even though s3://foo/bar and s3://foo/bar/ are pointing + # to *different* locations). However, the first two storage urls would + # nevertheless get different cache directories, which is undesired. + # Therefore, we handle these special cases right when parsing the command + # line. In retrospect, it would have been better to always add an implicit + # slash (even when using a prefix), but we can't do that now because it + # would make file systems created without trailing slash inaccessible. + if (re.match(r'^(s3|gs)://[^/]+$', s) + or re.match(r'^(s3c|swift(ks)?|rackspace)://[^/]+/[^/]+$', s)): + s += '/' + + return s + +def str_or_None_type(s): + if s.lower() == 'none': + return None + return s + +def suboptions_type(s): + '''An argument converter for suboptions + + A suboption takes a comma separated list of additional + options, e.g. --backend-options ssl,timeout=42,sse + ''' + + assert isinstance(s, str) + + opts = dict() + for opt in s.split(','): + if '=' in opt: + (key, val) = opt.split('=', 1) + else: + key = opt + val = True + opts[key] = val + + return opts diff --git a/src/s3ql/remove.py b/src/s3ql/remove.py new file mode 100644 index 0000000..b0f1395 --- /dev/null +++ b/src/s3ql/remove.py @@ -0,0 +1,59 @@ +''' +remove.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging, setup_logging, QuietError +from .common import assert_fs_owner, path2bytes +from .parse_args import ArgumentParser +import llfuse +import os +import sys +import textwrap + +log = logging.getLogger(__name__) + +def parse_args(args): + '''Parse command line''' + + parser = ArgumentParser( + description=textwrap.dedent('''\ + Recursively delete files and directories in an S3QL file system, + including immutable entries. + ''')) + + parser.add_debug() + parser.add_quiet() + parser.add_version() + + parser.add_argument('path', metavar='', nargs='+', + help='Directories to remove', + type=(lambda x: x.rstrip('/'))) + + return parser.parse_args(args) + + +def main(args=None): + '''Recursively delete files and directories in an S3QL file system''' + + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + for name in options.path: + if os.path.ismount(name): + raise QuietError('%s is a mount point.' % name) + + ctrlfile = assert_fs_owner(name) + fstat_p = os.stat(os.path.dirname(os.path.abspath(name))) + cmd = ('(%d, %r)' % (fstat_p.st_ino, + path2bytes(os.path.basename(name)))).encode() + llfuse.setxattr(ctrlfile, 'rmtree', cmd) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/src/s3ql/statfs.py b/src/s3ql/statfs.py new file mode 100644 index 0000000..57c3408 --- /dev/null +++ b/src/s3ql/statfs.py @@ -0,0 +1,80 @@ +''' +statfs.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging, setup_logging +from .common import assert_fs_owner, pretty_print_size +from .parse_args import ArgumentParser +import llfuse +import struct +import sys + +log = logging.getLogger(__name__) + +def parse_args(args): + '''Parse command line''' + + parser = ArgumentParser( + description="Print file system statistics.") + + parser.add_debug() + parser.add_quiet() + parser.add_version() + + parser.add_argument("mountpoint", metavar='', + type=(lambda x: x.rstrip('/')), + help='Mount point of the file system to examine') + + parser.add_argument("--raw", action="store_true", default=False, + help="Do not pretty-print numbers") + + return parser.parse_args(args) + +def main(args=None): + '''Print file system statistics to sys.stdout''' + + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + if options.raw: + pprint = lambda x: '%d bytes' % x + else: + pprint = pretty_print_size + + ctrlfile = assert_fs_owner(options.mountpoint, mountpoint=True) + + # Use a decent sized buffer, otherwise the statistics have to be + # calculated three(!) times because we need to invoke getxattr + # three times. + buf = llfuse.getxattr(ctrlfile, 's3qlstat', size_guess=256) + + (entries, blocks, inodes, fs_size, dedup_size, + compr_size, db_size, cache_cnt, cache_size, dirty_cnt, + dirty_size, removal_cnt) = struct.unpack('QQQQQQQQQQQQ', buf) + p_dedup = dedup_size * 100 / fs_size if fs_size else 0 + p_compr_1 = compr_size * 100 / fs_size if fs_size else 0 + p_compr_2 = compr_size * 100 / dedup_size if dedup_size else 0 + print ('Directory entries: %d' % entries, + 'Inodes: %d' % inodes, + 'Data blocks: %d' % blocks, + 'Total data size: %s' % pprint(fs_size), + 'After de-duplication: %s (%.2f%% of total)' + % (pprint(dedup_size), p_dedup), + 'After compression: %s (%.2f%% of total, %.2f%% of de-duplicated)' + % (pprint(compr_size), p_compr_1, p_compr_2), + 'Database size: %s (uncompressed)' % pprint(db_size), + 'Cache size: %s, %d entries' % (pprint(cache_size), cache_cnt), + 'Cache size (dirty): %s, %d entries' % (pprint(dirty_size), dirty_cnt), + 'Queued object removals: %d' % (removal_cnt,), + sep='\n') + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/src/s3ql/umount.py b/src/s3ql/umount.py new file mode 100644 index 0000000..cd55146 --- /dev/null +++ b/src/s3ql/umount.py @@ -0,0 +1,198 @@ +''' +umount.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging, setup_logging +from . import CTRL_NAME +from .common import assert_s3ql_mountpoint, parse_literal +from .parse_args import ArgumentParser +import llfuse +import os +import subprocess +import platform +import sys +import textwrap +import time + +log = logging.getLogger(__name__) + +def parse_args(args): + '''Parse command line + + This function writes to stdout/stderr and may call `system.exit()` instead + of throwing an exception if it encounters errors. + ''' + + parser = ArgumentParser( + description=textwrap.dedent('''\ + Unmounts an S3QL file system. The command returns only after all data + has been uploaded to the backend.''')) + + parser.add_debug() + parser.add_quiet() + parser.add_version() + + parser.add_argument("mountpoint", metavar='', + type=(lambda x: x.rstrip('/')), + help='Mount point to un-mount') + + parser.add_argument('--lazy', "-z", action="store_true", default=False, + help="Lazy umount. Detaches the file system immediately, even if there " + 'are still open files. The data will be uploaded in the background ' + 'once all open files have been closed.') + + return parser.parse_args(args) + +class UmountError(Exception): + """ + Base class for unmount errors. + """ + + message = 'internal error' + exitcode = 3 + + def __init__(self, mountpoint): + super().__init__() + self.mountpoint = mountpoint + + def __str__(self): + return self.message + +class UmountSubError(UmountError): + message = 'Unmount subprocess failed.' + exitcode = 2 + +class MountInUseError(UmountError): + message = 'In use.' + exitcode = 1 + +def lazy_umount(mountpoint): + '''Invoke fusermount -u -z for mountpoint''' + + if os.getuid() == 0 or platform.system() == 'Darwin': + # MacOS X always uses umount rather than fusermount + umount_cmd = ('umount', '-l', mountpoint) + else: + umount_cmd = ('fusermount', '-u', '-z', mountpoint) + + if subprocess.call(umount_cmd) != 0: + raise UmountSubError(mountpoint) + +def get_cmdline(pid): + '''Return command line for *pid* + + If *pid* doesn't exists, return None. If command line + cannot be determined for other reasons, log warning + and return None. + ''' + + try: + output = subprocess.check_output(['ps', '-p', str(pid), '-o', 'args='], + universal_newlines=True).strip() + except subprocess.CalledProcessError: + log.warning('Unable to execute ps, assuming process %d has terminated.' + % pid) + return None + + if output: + return output + else: + return None + +def blocking_umount(mountpoint): + '''Invoke fusermount and wait for daemon to terminate.''' + + with open('/dev/null', 'wb') as devnull: + if subprocess.call(['fuser', '-m', mountpoint], stdout=devnull, + stderr=devnull) == 0: + raise MountInUseError(mountpoint) + + ctrlfile = os.path.join(mountpoint, CTRL_NAME) + + log.debug('Flushing cache...') + llfuse.setxattr(ctrlfile, 's3ql_flushcache!', b'dummy') + + # Get pid + log.debug('Trying to get pid') + pid = parse_literal(llfuse.getxattr(ctrlfile, 's3ql_pid?'), int) + log.debug('PID is %d', pid) + + # Get command line to make race conditions less-likely + cmdline = get_cmdline(pid) + + # Unmount + log.debug('Unmounting...') + + if os.getuid() == 0 or platform.system() == 'Darwin': + # MacOS X always uses umount rather than fusermount + umount_cmd = ['umount', mountpoint] + else: + umount_cmd = ['fusermount', '-u', mountpoint] + + if subprocess.call(umount_cmd) != 0: + raise UmountSubError(mountpoint) + + # Wait for daemon + log.debug('Uploading metadata...') + step = 0.1 + while True: + try: + os.kill(pid, 0) + except OSError: + log.debug('Kill failed, assuming daemon has quit.') + break + + # Check that the process did not terminate and the PID + # was reused by a different process + cmdline2 = get_cmdline(pid) + if cmdline2 is None: + log.debug('Reading cmdline failed, assuming daemon has quit.') + break + elif cmdline2 == cmdline: + log.debug('PID still alive and commandline unchanged.') + else: + log.debug('PID still alive, but cmdline changed') + break + + # Process still exists, we wait + log.debug('Daemon seems to be alive, waiting...') + time.sleep(step) + if step < 1: + step += 0.1 + +def main(args=None): + '''Umount S3QL file system''' + + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + assert_s3ql_mountpoint(options.mountpoint) + + try: + if options.lazy: + lazy_umount(options.mountpoint) + else: + blocking_umount(options.mountpoint) + + except MountInUseError as err: + print('Cannot unmount, the following processes still access the mountpoint:', + file=sys.stderr) + subprocess.call(['fuser', '-v', '-m', options.mountpoint], + stdout=sys.stderr, stderr=sys.stderr) + sys.exit(err.exitcode) + + except UmountError as err: + print('%s: %s' % (options.mountpoint, err), file=sys.stderr) + sys.exit(err.exitcode) + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/src/s3ql/verify.py b/src/s3ql/verify.py new file mode 100644 index 0000000..a2118a8 --- /dev/null +++ b/src/s3ql/verify.py @@ -0,0 +1,222 @@ +''' +verify.py - this file is part of S3QL. + +Copyright © 2014 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from .logging import logging, setup_logging +from .mount import get_metadata +from . import BUFSIZE +from .common import (get_backend_factory, get_backend_cachedir, pretty_print_size, + AsyncFn) +from .backends.common import NoSuchObject, CorruptedObjectError +from .parse_args import ArgumentParser +from queue import Queue, Full as QueueFull +import os +import argparse +import time +import signal +import faulthandler +import sys +import textwrap +import atexit + +log = logging.getLogger(__name__) + +def _new_file_type(s, encoding='utf-8'): + '''An argparse type for a file that does not yet exist''' + + if os.path.exists(s): + msg = 'File already exists - refusing to overwrite: %s' % s + raise argparse.ArgumentTypeError(msg) + + fh = open(s, 'w', encoding=encoding) + atexit.register(fh.close) + return fh + +def parse_args(args): + '''Parse command line''' + + parser = ArgumentParser( + description=textwrap.dedent('''\ + Verifies that all data in an S3QL file system can be downloaded + from the storage backend. + + In contrast to fsck.s3ql, this program does not trust the object + listing returned by the backend, but actually attempts to retrieve + every object. It therefore takes a lot longer. + ''')) + + parser.add_debug() + parser.add_quiet() + parser.add_version() + parser.add_cachedir() + parser.add_authfile() + parser.add_backend_options() + parser.add_storage_url() + + parser.add_argument("--missing-file", type=_new_file_type, metavar='', + default='missing_objects.txt', + help="File to store keys of missing objects.") + + parser.add_argument("--corrupted-file", type=_new_file_type, metavar='', + default='corrupted_objects.txt', + help="File to store keys of corrupted objects.") + + parser.add_argument("--data", action="store_true", default=False, + help="Read every object completely, instead of checking " + "just the metadata.") + + parser.add_argument("--parallel", default=4, type=int, + help="Number of connections to use in parallel.") + + parser.add_argument("--start-with", default=0, type=int, metavar='', + help="Skip over first objects and with verifying " + "object +1.") + + options = parser.parse_args(args) + + return options + +def main(args=None): + faulthandler.enable() + faulthandler.register(signal.SIGUSR1) + + if args is None: + args = sys.argv[1:] + + options = parse_args(args) + setup_logging(options) + + backend_factory = get_backend_factory(options.storage_url, options.backend_options, + options.authfile) + + # Get paths + cachepath = get_backend_cachedir(options.storage_url, options.cachedir) + + # Retrieve metadata + with backend_factory() as backend: + (param, db) = get_metadata(backend, cachepath) + + retrieve_objects(db, backend_factory, options.corrupted_file, + options.missing_file, thread_count=options.parallel, + full=options.data, offset=options.start_with) + + if options.corrupted_file.tell() or options.missing_file.tell(): + sys.exit(46) + else: + os.unlink(options.corrupted_file.name) + os.unlink(options.missing_file.name) + sys.exit(0) + +def retrieve_objects(db, backend_factory, corrupted_fh, missing_fh, + thread_count=1, full=False, offset=0): + """Attempt to retrieve every object""" + + log.info('Reading all objects...') + + queue = Queue(thread_count) + threads = [] + for _ in range(thread_count): + t = AsyncFn(_retrieve_loop, queue, backend_factory, corrupted_fh, + missing_fh, full) + # Don't wait for worker threads, gives deadlock if main thread + # terminates with exception + t.daemon = True + t.start() + threads.append(t) + + total_size = db.get_val('SELECT SUM(size) FROM objects') + total_count = db.get_val('SELECT COUNT(id) FROM objects') + size_acc = 0 + + sql = 'SELECT id, size FROM objects ORDER BY id' + i = 0 # Make sure this is set if there are zero objects + stamp1 = 0 + try: + for (i, (obj_id, size)) in enumerate(db.query(sql)): + stamp2 = time.time() + if stamp2 - stamp1 > 1: + stamp1 = stamp2 + progress = '%d objects (%.2f%%)' % (i, i/total_count * 100) + if full: + s = pretty_print_size(size_acc) + progress += ' / %s (%.2f%%)' % (s, size_acc / total_size * 100) + sys.stdout.write('\r..processed %s so far..' % progress) + sys.stdout.flush() + + # Terminate early if any thread failed with an exception + for t in threads: + if not t.is_alive(): + t.join_and_raise() + + size_acc += size + if i < offset: + continue + + # Avoid blocking if all threads terminated + while True: + try: + queue.put(obj_id, timeout=1) + except QueueFull: + pass + else: + break + for t in threads: + if not t.is_alive(): + t.join_and_raise() + + finally: + sys.stdout.write('\n') + + queue.maxsize += len(threads) + for t in threads: + queue.put(None) + + for t in threads: + t.join_and_raise() + + log.info('Verified all %d storage objects.', i) + +def _retrieve_loop(queue, backend_factory, corrupted_fh, missing_fh, full=False): + '''Retrieve object ids arriving in *queue* from *backend* + + If *full* is False, lookup and read metadata. If *full* is True, + read entire object. + + Corrupted objects are written into *corrupted_fh*. Missing objects + are written into *missing_fh*. + + Terminate when None is received. + ''' + + with backend_factory() as backend: + while True: + obj_id = queue.get() + if obj_id is None: + break + + log.debug('reading object %s', obj_id) + def do_read(fh): + while True: + buf = fh.read(BUFSIZE) + if not buf: + break + + key = 's3ql_data_%d' % obj_id + try: + if full: + backend.perform_read(do_read, key) + else: + backend.lookup(key) + except NoSuchObject: + log.warning('Backend seems to have lost object %d', obj_id) + print(key, file=missing_fh) + except CorruptedObjectError: + log.warning('Object %d is corrupted', obj_id) + print(key, file=corrupted_fh) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/tests/common.py b/tests/common.py new file mode 100644 index 0000000..9373654 --- /dev/null +++ b/tests/common.py @@ -0,0 +1,309 @@ +''' +common.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. + + +This module contains common functions used by multiple unit tests. +''' + +from contextlib import contextmanager +from functools import wraps +import re +import time +import os +import subprocess +import stat +import random +import configparser +import logging +import pytest +import functools + +def get_clock_granularity(): + resolution = float('inf') + for i in range(50): + stamp1 = time.time() + stamp2 = stamp1 + while stamp1 == stamp2: + stamp2 = time.time() + resolution = min(resolution, stamp2 - stamp1) + time.sleep(0.01) + return resolution +CLOCK_GRANULARITY = get_clock_granularity() + +# When testing, we want to make sure that we don't sleep for too short a time +# (cause it may cause spurious test failures), and that the sleep interval +# covers at least one timer update. We have to monkeypatch because we especially +# want functions like s3ql.backends.common.retry to use the "safe" sleep +# version. +@functools.wraps(time.sleep) +def safe_sleep(secs, _sleep_real=time.sleep): + '''Like time.sleep(), but sleep for at least *secs* + + `time.sleep` may sleep less than the given period if a signal is + received. This function ensures that we sleep for at least the + desired time. + ''' + + now = time.time() + end = now + secs + while now < end: + _sleep_real(max(end - now, CLOCK_GRANULARITY)) + now = time.time() + +@pytest.fixture(autouse=True, scope='session') +def install_safe_sleep(): + time.sleep = safe_sleep + +def retry(timeout, fn, *a, **kw): + """Wait for fn(*a, **kw) to return True. + + If the return value of fn() returns something True, this value + is returned. Otherwise, the function is called repeatedly for + `timeout` seconds. If the timeout is reached, `RetryTimeoutError` is + raised. + """ + + step = 0.2 + waited = 0 + while waited < timeout: + ret = fn(*a, **kw) + if ret: + return ret + time.sleep(step) + waited += step + if step < waited / 30: + step *= 2 + + raise RetryTimeoutError() + +class RetryTimeoutError(Exception): + '''Raised by `retry()` when a timeout is reached.''' + + pass + +def skip_if_no_fusermount(): + '''Raise SkipTest if fusermount is not available''' + + with subprocess.Popen(['which', 'fusermount'], stdout=subprocess.PIPE, + universal_newlines=True) as which: + fusermount_path = which.communicate()[0].strip() + + if not fusermount_path or which.returncode != 0: + pytest.skip("Can't find fusermount executable") + + if not os.path.exists('/dev/fuse'): + pytest.skip("FUSE kernel module does not seem to be loaded") + + if os.getuid() == 0: + return + + mode = os.stat(fusermount_path).st_mode + if mode & stat.S_ISUID == 0: + pytest.skip('fusermount executable not setuid, and we are not root.') + + try: + fd = os.open('/dev/fuse', os.O_RDWR) + except OSError as exc: + pytest.skip('Unable to open /dev/fuse: %s' % exc.strerror) + else: + os.close(fd) + +def skip_without_rsync(): + try: + with open('/dev/null', 'wb') as null: + subprocess.call(['rsync', '--version'], stdout=null, + stderr=subprocess.STDOUT,) + except FileNotFoundError: + pytest.skip('rsync not installed') + +def populate_dir(path, entries=1000, size=20*1024*1024, + pooldir='/usr/bin', seed=None): + '''Populate directory with random data + + *entries* specifies the total number of directory entries that are created + in the tree. *size* specifies the size occupied by all files together. The + files in *pooldir* are used as a source of directory names and file + contents. + + *seed* is used to initalize the random number generator and can be used to + make the created structure reproducible (provided that the contents of + *pooldir* don't change). + ''' + + poolnames = os.listdir(pooldir) + if seed is None: + # We want tests to be reproducible on a given system, so users + # can report meaningful bugs + seed = len(poolnames) + random.seed(seed) + + # Entries in percentages + subdir_cnt = random.randint(5, 10) + file_cnt = random.randint(60, 70) + fifo_cnt = random.randint(5, 10) + symlink_cnt = random.randint(10, 20) + hardlink_cnt = random.randint(5, 15) + + # Normalize to desired entry count + scale = entries / sum((subdir_cnt, file_cnt, fifo_cnt, symlink_cnt, hardlink_cnt)) + subdir_cnt = int(scale * subdir_cnt) + file_cnt = int(scale * file_cnt) + fifo_cnt = int(scale * fifo_cnt) + symlink_cnt = int(scale * symlink_cnt) + hardlink_cnt = int(scale * hardlink_cnt) + + # Sizes, make sure there is at least one big file + file_sizes = [ random.randint(0, 100) for _ in range(file_cnt-1) ] + scale = 0.5 * size / sum(file_sizes) + file_sizes = [ int(scale * x) for x in file_sizes ] + file_sizes.append(int(0.5 * size)) + + # Special characters for use in filenames + special_chars = [ chr(x) for x in range(128) + if x not in (0, ord('/')) ] + + def random_name(path): + '''Get random, non-existing file name underneath *path* + + Returns a fully qualified path with a filename chosen from *poolnames*. + ''' + while True: + name = poolnames[random.randrange(len(poolnames))] + + # Special characters + len_ = random.randrange(4) + if len_ > 0: + pos = random.choice((-1,0,1)) # Prefix, Middle, Suffix + s = ''.join(special_chars[random.randrange(len(special_chars))] + for _ in range(len_)) + if pos == -1: + name = s + name + elif pos == 1: + name += s + else: + name += s + poolnames[random.randrange(len(poolnames))] + + fullname = os.path.join(path, name) + if not os.path.lexists(fullname): + break + return fullname + + + # + # Step 1: create directory tree + # + dirs = [ path ] + for _ in range(subdir_cnt): + idx = random.randrange(len(dirs)) + name = random_name(dirs[idx]) + os.mkdir(name) + dirs.append(name) + + + # + # Step 2: populate the tree with files + # + files = [] + for size in file_sizes: + idx = random.randrange(len(dirs)) + name = random_name(dirs[idx]) + with open(name, 'wb') as dst: + while size > 0: + idx = random.randrange(len(poolnames)) + srcname = os.path.join(pooldir, poolnames[idx]) + if not os.path.isfile(srcname): + continue + with open(srcname, 'rb') as src: + buf = src.read(size) + dst.write(buf) + size -= len(buf) + files.append(name) + + # + # Step 3: Special files + # + for _ in range(fifo_cnt): + name = random_name(dirs[random.randrange(len(dirs))]) + os.mkfifo(name) + files.append(name) + + # + # Step 4: populate tree with symlinks + # + for _ in range(symlink_cnt): + relative = random.choice((True, False)) + existing = random.choice((True, False)) + idx = random.randrange(len(dirs)) + dir_ = dirs[idx] + name = random_name(dir_) + + if existing: + directory = random.choice((True, False)) + if directory: + target = dirs[random.randrange(len(dirs))] + else: + target = files[random.randrange(len(files))] + else: + target = random_name(dirs[random.randrange(len(dirs))]) + + if relative: + target = os.path.relpath(target, dir_) + else: + target = os.path.abspath(target) + + os.symlink(target, name) + + # + # Step 5: Create some hardlinks + # + for _ in range(hardlink_cnt): + samedir = random.choice((True, False)) + + target = files[random.randrange(len(files))] + if samedir: + dir_ = os.path.dirname(target) + else: + dir_ = dirs[random.randrange(len(dirs))] + name = random_name(dir_) + os.link(target, name) + files.append(name) + + +class NoTestSection(Exception): + ''' + Raised by get_remote_test_info if no matching test + section was found. + ''' + + def __init__(self, reason): + self.reason = reason + +def get_remote_test_info(name): + authfile = os.path.expanduser('~/.s3ql/authinfo2') + if not os.path.exists(authfile): + raise NoTestSection('No authentication file found.') + + mode = os.stat(authfile).st_mode + if mode & (stat.S_IRGRP | stat.S_IROTH): + raise NoTestSection("Authentication file has insecure permissions") + + config = configparser.ConfigParser() + config.read(authfile) + + try: + fs_name = config.get(name, 'test-fs') + backend_login = config.get(name, 'backend-login') + backend_password = config.get(name, 'backend-password') + except (configparser.NoOptionError, configparser.NoSectionError): + raise NoTestSection("Authentication file does not have %s section" % name) + + # Append prefix to make sure that we're starting with an empty bucket + if fs_name[-1] != '/': + fs_name += '/' + fs_name += 's3ql_test_%d/' % time.time() + + return (backend_login, backend_password, fs_name) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7c974d1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,120 @@ +''' +conftest.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. + + +This module is loaded automatically by py.test and is used to +initialize logging and adjust the load path before running +any tests. +''' + +# Python version check +import sys +if sys.version_info < (3,3): + raise SystemExit('Python version is %d.%d.%d, but S3QL requires Python 3.3 or newer' + % sys.version_info[:3]) + +import logging.handlers +import sys +import os.path +import pytest +import faulthandler +import signal +import gc +import time + +# If a test fails, wait a moment before retrieving the captured +# stdout/stderr. When using a server process (like in t4_fuse.py), this makes +# sure that we capture any potential output of the server that comes *after* a +# test has failed. For example, if a request handler raises an exception, the +# server first signals an error to FUSE (causing the test to fail), and then +# logs the exception. Without the extra delay, the exception will go into +# nowhere. +@pytest.mark.hookwrapper +def pytest_pyfunc_call(pyfuncitem): + outcome = yield + failed = outcome.excinfo is not None + if failed: + time.sleep(1) + +@pytest.fixture(scope="class") +def s3ql_cmd_argv(request): + '''Provide argument list to execute s3ql commands in tests''' + + if request.config.getoption('installed'): + request.cls.s3ql_cmd_argv = lambda self, cmd: [ cmd ] + else: + basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + request.cls.s3ql_cmd_argv = lambda self, cmd: [ sys.executable, + os.path.join(basedir, 'bin', cmd) ] + +# Enable output checks +pytest_plugins = ('pytest_checklogs') + +# Ignore DeprecationWarnings when running unit tests. They are +# unfortunately quite often a result of indirect imports via third party +# modules, so we can't actually fix them. +@pytest.fixture(autouse=True) +def ignore_depreciation_warnings(reg_output): + reg_output(r'(Pending)?DeprecationWarning', count=0) + +@pytest.fixture() +def pass_reg_output(request, reg_output): + '''Provide reg_output function to UnitTest instances''' + request.instance.reg_output = reg_output + +def pytest_addoption(parser): + group = parser.getgroup("terminal reporting") + group._addoption("--logdebug", action="append", metavar='', + help="Activate debugging output from for tests. Use `all` " + "to get debug messages from all modules. This option can be " + "specified multiple times.") + + group = parser.getgroup("general") + group._addoption("--installed", action="store_true", default=False, + help="Test the installed package.") + +def pytest_configure(config): + # If we are running from the S3QL source directory, make sure that we + # load modules from here + basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + if not config.getoption('installed'): + if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'src', 's3ql', '__init__.py'))): + sys.path = [os.path.join(basedir, 'src')] + sys.path + + # When running from HG repo, enable all warnings + if os.path.exists(os.path.join(basedir, 'MANIFEST.in')): + import warnings + warnings.resetwarnings() + warnings.simplefilter('default') + + # Enable faulthandler + global faultlog_fh + faultlog_fh = open(os.path.join(basedir, 'tests', 'test_crit.log'), 'a') + faulthandler.enable(faultlog_fh) + faulthandler.register(signal.SIGUSR1, file=faultlog_fh) + + # Configure logging. We don't set a default handler but rely on + # the catchlog pytest plugin. + logdebug = config.getoption('logdebug') + root_logger = logging.getLogger() + if logdebug is not None: + logging.disable(logging.NOTSET) + if 'all' in logdebug: + root_logger.setLevel(logging.DEBUG) + else: + for module in logdebug: + logging.getLogger(module).setLevel(logging.DEBUG) + else: + root_logger.setLevel(logging.INFO) + logging.disable(logging.DEBUG) + logging.captureWarnings(capture=True) + +# Run gc.collect() at the end of every test, so that we get ResourceWarnings +# as early as possible. +def pytest_runtest_teardown(item, nextitem): + gc.collect() diff --git a/tests/mock_server.py b/tests/mock_server.py new file mode 100644 index 0000000..c63c54b --- /dev/null +++ b/tests/mock_server.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +''' +mock_server.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from http.server import BaseHTTPRequestHandler +import re +import socketserver +import logging +import hashlib +import urllib.parse +import sys +import traceback +from xml.sax.saxutils import escape as xml_escape + +log = logging.getLogger(__name__) + +ERROR_RESPONSE_TEMPLATE = '''\ + + + %(code)s + %(message)s + %(resource)s + %(request_id)s + +''' + +COPY_RESPONSE_TEMPLATE = '''\ + + + 2008-02-20T22:13:01 + "%(etag)s" + +''' + +class StorageServer(socketserver.TCPServer): + + def __init__(self, request_handler, server_address): + super().__init__(server_address, request_handler) + self.data = dict() + self.metadata = dict() + self.hostname = self.server_address[0] + self.port = self.server_address[1] + + def serve_forever(self): + # This is a hack to debug sporadic test failures due to exceptions in + # the mock server thread. So far, I have not managed to properly debug + # them because reproducing them is very difficult, and output capture + # for some reason only gets the first line of the traceback. My current + # best theory is that these exceptions just happen at some point during + # cleanup (after the test has passed), and are therefore safe to ignore + # ( because any actual problems should also cause test failures in the + # main thread that runs the actual test). However, it's better to be + # safe than sorry + try: + super().serve_forever() + except Exception: + with open('mock_server.log', 'r+') as fh: + traceback.print_exc(file=fh) + raise + +class ParsedURL: + __slots__ = [ 'bucket', 'key', 'params', 'fragment' ] + +def parse_url(path): + p = ParsedURL() + q = urllib.parse.urlsplit(path) + + path = urllib.parse.unquote(q.path) + + assert path[0] == '/' + (p.bucket, p.key) = path[1:].split('/', maxsplit=1) + + p.params = urllib.parse.parse_qs(q.query) + p.fragment = q.fragment + + return p + +class S3CRequestHandler(BaseHTTPRequestHandler): + '''A request handler implementing a subset of the AWS S3 Interface + + Bucket names are ignored, all keys share the same global + namespace. + ''' + + server_version = "MockHTTP" + protocol_version = 'HTTP/1.1' + meta_header_re = re.compile(r'X-AMZ-Meta-([a-z0-9_.-]+)$', + re.IGNORECASE) + hdr_prefix = 'X-AMZ-' + xml_ns = 'http://s3.amazonaws.com/doc/2006-03-01/' + + def log_message(self, format, *args): + log.debug(format, *args) + + def handle(self): + # Ignore exceptions resulting from the client closing + # the connection. + try: + return super().handle() + except ValueError as exc: + if exc.args == ('I/O operation on closed file.',): + pass + else: + raise + except (BrokenPipeError, ConnectionResetError): + pass + + def do_DELETE(self): + q = parse_url(self.path) + try: + del self.server.data[q.key] + del self.server.metadata[q.key] + except KeyError: + self.send_error(404, code='NoSuchKey', resource=q.key) + return + else: + self.send_response(204) + self.end_headers() + + def _check_encoding(self): + encoding = self.headers['Content-Encoding'] + if 'Content-Length' not in self.headers: + self.send_error(400, message='Missing Content-Length', + code='MissingContentLength') + return + elif encoding and encoding != 'identity': + self.send_error(501, message='Unsupport encoding', + code='NotImplemented') + return + + return int(self.headers['Content-Length']) + + def do_PUT(self): + len_ = self._check_encoding() + q = parse_url(self.path) + meta = dict() + for (name, value) in self.headers.items(): + hit = self.meta_header_re.search(name) + if hit: + meta[hit.group(1)] = value + + src = self.headers.get(self.hdr_prefix + 'copy-source') + if src and len_: + self.send_error(400, message='Upload and copy are mutually exclusive', + code='UnexpectedContent') + return + elif src: + src = urllib.parse.unquote(src) + hit = re.match('^/([a-z0-9._-]+)/(.+)$', src) + if not hit: + self.send_error(400, message='Cannot parse copy-source', + code='InvalidArgument') + return + + metadata_directive = self.headers.get(self.hdr_prefix + 'metadata-directive', + 'COPY') + if metadata_directive not in ('COPY', 'REPLACE'): + self.send_error(400, message='Invalid metadata directive', + code='InvalidArgument') + return + src = hit.group(2) + try: + data = self.server.data[src] + self.server.data[q.key] = data + if metadata_directive == 'COPY': + self.server.metadata[q.key] = self.server.metadata[src] + else: + self.server.metadata[q.key] = meta + except KeyError: + self.send_error(404, code='NoSuchKey', resource=src) + return + else: + data = self.rfile.read(len_) + self.server.metadata[q.key] = meta + self.server.data[q.key] = data + + md5 = hashlib.md5() + md5.update(data) + + if src: + content = (COPY_RESPONSE_TEMPLATE % + {'etag': md5.hexdigest(), + 'ns': self.xml_ns }).encode('utf-8') + self.send_response(200) + self.send_header('ETag', '"%s"' % md5.hexdigest()) + self.send_header('Content-Length', str(len(content))) + self.send_header("Content-Type", 'text/xml') + self.end_headers() + self.wfile.write(content) + else: + self.send_response(201) + self.send_header('ETag', '"%s"' % md5.hexdigest()) + self.send_header('Content-Length', '0') + self.end_headers() + + def handle_expect_100(self): + if self.command == 'PUT': + self._check_encoding() + + self.send_response_only(100) + self.end_headers() + return True + + def do_GET(self): + q = parse_url(self.path) + if not q.key: + return self.do_list(q) + + try: + data = self.server.data[q.key] + meta = self.server.metadata[q.key] + except KeyError: + self.send_error(404, code='NoSuchKey', resource=q.key) + return + + self.send_response(200) + self.send_header("Content-Type", 'application/octet-stream') + self.send_header("Content-Length", str(len(data))) + for (name, value) in meta.items(): + self.send_header(self.hdr_prefix + 'Meta-%s' % name, value) + md5 = hashlib.md5() + md5.update(data) + self.send_header('ETag', '"%s"' % md5.hexdigest()) + self.end_headers() + self.send_data(data) + + def send_data(self, data): + self.wfile.write(data) + + def do_list(self, q): + marker = q.params['marker'][0] if 'marker' in q.params else None + max_keys = int(q.params['max_keys'][0]) if 'max_keys' in q.params else 1000 + prefix = q.params['prefix'][0] if 'prefix' in q.params else '' + + resp = ['', + '' % self.xml_ns, + '%d' % max_keys, + 'false' ] + + count = 0 + for key in sorted(self.server.data): + if not key.startswith(prefix): + continue + if marker and key <= marker: + continue + resp.append('%s' % xml_escape(key)) + count += 1 + if count == max_keys: + resp[3] = 'true' + break + + resp.append('') + body = '\n'.join(resp).encode() + + self.send_response(200) + self.send_header("Content-Type", 'text/xml') + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_HEAD(self): + q = parse_url(self.path) + try: + meta = self.server.metadata[q.key] + data = self.server.data[q.key] + except KeyError: + self.send_error(404, code='NoSuchKey', resource=q.key) + return + + self.send_response(200) + self.send_header("Content-Type", 'application/octet-stream') + self.send_header("Content-Length", str(len(data))) + for (name, value) in meta.items(): + self.send_header(self.hdr_prefix + 'Meta-%s' % name, value) + self.end_headers() + + def send_error(self, status, message=None, code='', resource='', + extra_headers=None): + + if not message: + try: + (_, message) = self.responses[status] + except KeyError: + message = 'Unknown' + + self.log_error("code %d, message %s", status, message) + content = (ERROR_RESPONSE_TEMPLATE % + {'code': code, + 'message': xml_escape(message), + 'request_id': 42, + 'resource': xml_escape(resource)}).encode('utf-8', 'replace') + self.send_response(status, message) + self.send_header("Content-Type", 'text/xml; charset="utf-8"') + self.send_header("Content-Length", str(len(content))) + if extra_headers: + for (name, value) in extra_headers.items(): + self.send_header(name, value) + self.end_headers() + if self.command != 'HEAD' and status >= 200 and status not in (204, 304): + self.wfile.write(content) + + +class GSRequestHandler(S3CRequestHandler): + '''A request handler implementing a subset of the Google Storage API. + + Bucket names are ignored, all keys share the same global namespace. + ''' + + meta_header_re = re.compile(r'x-goog-meta-([a-z0-9_.-]+)$', + re.IGNORECASE) + hdr_prefix = 'x-goog-' + xml_ns = 'http://doc.s3.amazonaws.com/2006-03-01' + +#: A list of the available mock request handlers with +#: corresponding storage urls +handler_list = [ (S3CRequestHandler, 's3c://%(host)s:%(port)d/s3ql_test'), + + # Special syntax only for testing against mock server + (GSRequestHandler, 'gs://!unittest!%(host)s:%(port)d/s3ql_test') ] diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 0000000..a87c1f8 --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +addopts = --verbose --assert=rewrite --exitfirst --tb=native +python_files = t?_*.py diff --git a/tests/pytest_checklogs.py b/tests/pytest_checklogs.py new file mode 100644 index 0000000..046df20 --- /dev/null +++ b/tests/pytest_checklogs.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +''' +py.test plugin to look for suspicious phrases in messages +emitted on stdout/stderr or via the logging module. + +False positives can be registered via a new `reg_output` +fixture (for messages to stdout/stderr), and a `assert_logs` +function (for logging messages). +''' + +import pytest +import re +import functools +import sys +import logging +from contextlib import contextmanager + +def pytest_configure(config): + if not config.pluginmanager.hasplugin('pytest_catchlog'): + raise ImportError('pytest catchlog plugin not found') + +# Fail tests if they result in log messages of severity WARNING or more. +def check_test_log(caplog): + for record in caplog.records: + if (record.levelno >= logging.WARNING and + not getattr(record, 'checklogs_ignore', False)): + raise AssertionError('Logger received warning messages') + +class CountMessagesHandler(logging.Handler): + def __init__(self, level=logging.NOTSET): + super().__init__(level) + self.count = 0 + + def emit(self, record): + self.count += 1 + +@contextmanager +def assert_logs(pattern, level=logging.WARNING, count=None): + '''Assert that suite emits specified log message + + *pattern* is matched against the *unformatted* log message, i.e. before any + arguments are merged. + + If *count* is not None, raise an exception unless exactly *count* matching + messages are caught. + + Matched log records will also be flagged so that the caplog fixture + does not generate exceptions for them (no matter their severity). + ''' + + def filter(record): + if (record.levelno == level and + re.search(pattern, record.msg)): + record.checklogs_ignore = True + return True + return False + + handler = CountMessagesHandler() + handler.setLevel(level) + handler.addFilter(filter) + logger = logging.getLogger() + logger.addHandler(handler) + try: + yield + + finally: + logger.removeHandler(handler) + + if count is not None and handler.count != count: + raise AssertionError('Expected to catch %d %r messages, but got only %d' + % (count, pattern, handler.count)) + +def check_test_output(capfd, item): + (stdout, stderr) = capfd.readouterr() + + # Write back what we've read (so that it will still be printed) + sys.stdout.write(stdout) + sys.stderr.write(stderr) + + # Strip out false positives + try: + false_pos = item.checklogs_fp + except AttributeError: + false_pos = () + for (pattern, flags, count) in false_pos: + cp = re.compile(pattern, flags) + (stdout, cnt) = cp.subn('', stdout, count=count) + if count == 0 or count - cnt > 0: + stderr = cp.sub('', stderr, count=count - cnt) + + for pattern in ('exception', 'error', 'warning', 'fatal', 'traceback', + 'fault', 'crash(?:ed)?', 'abort(?:ed)', 'fishy'): + cp = re.compile(r'\b{}\b'.format(pattern), re.IGNORECASE | re.MULTILINE) + hit = cp.search(stderr) + if hit: + raise AssertionError('Suspicious output to stderr (matched "%s")' % hit.group(0)) + hit = cp.search(stdout) + if hit: + raise AssertionError('Suspicious output to stdout (matched "%s")' % hit.group(0)) + +def register_output(item, pattern, count=1, flags=re.MULTILINE): + '''Register *pattern* as false positive for output checking + + This prevents the test from failing because the output otherwise + appears suspicious. + ''' + + item.checklogs_fp.append((pattern, flags, count)) + +@pytest.fixture() +def reg_output(request): + assert not hasattr(request.node, 'checklogs_fp') + request.node.checklogs_fp = [] + return functools.partial(register_output, request.node) + +def check_output(item): + pm = item.config.pluginmanager + cm = pm.getplugin('capturemanager') + check_test_output(cm._capturing, item) + check_test_log(item.catch_log_handler) + +@pytest.hookimpl(trylast=True) +def pytest_runtest_setup(item): + check_output(item) +@pytest.hookimpl(trylast=True) +def pytest_runtest_call(item): + check_output(item) +@pytest.hookimpl(trylast=True) +def pytest_runtest_teardown(item, nextitem): + check_output(item) diff --git a/tests/t1_backends.py b/tests/t1_backends.py new file mode 100755 index 0000000..8ae7c56 --- /dev/null +++ b/tests/t1_backends.py @@ -0,0 +1,1282 @@ +#!/usr/bin/env python3 +''' +t1_backends.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +from s3ql.logging import logging +import mock_server +from dugong import ConnectionClosed +from s3ql import backends, BUFSIZE +from s3ql.backends.local import Backend as LocalBackend +from s3ql.backends.gs import Backend as GSBackend +from s3ql.backends.common import (NoSuchObject, CorruptedObjectError) +from s3ql.backends.comprenc import ComprencBackend, ObjectNotEncrypted +from s3ql.backends.s3c import BadDigestError, OperationAbortedError, HTTPError, S3Error +from argparse import Namespace +from common import get_remote_test_info, NoTestSection, CLOCK_GRANULARITY +from pytest_checklogs import assert_logs +import s3ql.backends.common +import tempfile +import re +import functools +import time +import pytest +from pytest import raises as assert_raises +import shutil +import struct +import threading + +log = logging.getLogger(__name__) +empty_set = set() + +def brace_expand(s): + hit = re.search('^(.*)\{(.+)\}(.*)$', s) + if not hit: + return [s] + (p, e, s) = hit.groups() + l = [] + for el in e.split(','): + l.append(p+el+s) + return l + +def enable_temp_fail(backend): + if isinstance(backend, ComprencBackend): + backend = backend.backend + backend.unittest_info.may_temp_fail = True + +# It'd be nice if we could use the setup_module hook instead, but +# unfortunately that gets executed *after* pytest_generate_tests. +def _get_backend_info(): + '''Get information about raw backends available for testing''' + + info = [] + + # Local backend + bi = Namespace() + bi.name = 'local' + bi.classname = 'local' + info.append(bi) + + # Backends talking to actual remote servers (if available) + for name in backends.prefix_map: + if name == 'local': # local backend has already been handled + continue + + try: + (login, password, + storage_url) = get_remote_test_info(name + '-test') + except NoTestSection as exc: + log.info('Not running remote tests for %s backend: %s', + name, exc.reason) + continue + + bi = Namespace() + bi.name = 'remote-' + name + bi.classname = name + bi.storage_url = storage_url + bi.login = login + bi.password = password + info.append(bi) + + # Backends talking to local mock servers + for (request_handler, storage_url) in mock_server.handler_list: + name = re.match(r'^([a-zA-Z0-9]+)://', storage_url).group(1) + bi = Namespace() + bi.name = 'mock-' + name + bi.classname = name + bi.request_handler = request_handler + bi.storage_url = storage_url + info.append(bi) + + return info + +def pytest_generate_tests(metafunc, _info_cache=[]): + if _info_cache: + backend_info = _info_cache[0] + else: + backend_info = _get_backend_info() + _info_cache.append(backend_info) + + if 'backend' not in metafunc.fixturenames: + return + + fn = metafunc.function + assert hasattr(fn, 'with_backend') + + test_params = [] + for spec in fn.with_backend.args: + (backend_spec, comprenc_spec) = spec.split('/') + + # Expand compression/encryption specification + # raw == don't use ComprencBackend at all + # plain = use ComprencBackend without compression and encryption + if comprenc_spec == '*': + comprenc_kinds = [ 'aes', 'aes+zlib', 'plain', 'zlib', + 'bzip2', 'lzma', 'raw' ] + else: + comprenc_kinds = brace_expand(comprenc_spec) + + # Expand backend specification + if backend_spec == '*': + test_bi = backend_info + else: + test_bi = [] + for name in brace_expand(backend_spec): + test_bi += [ x for x in backend_info + if x.classname == name ] + + # Filter + if fn.with_backend.kwargs.get('require_mock_server', False): + test_bi = [ x for x in test_bi + if 'request_handler' in x ] + if fn.with_backend.kwargs.get('require_immediate_consistency', False): + test_bi = [ x for x in test_bi + if 'request_handler' in x + or x.classname in ('local', 'gs') ] + + for comprenc_kind in comprenc_kinds: + for bi in test_bi: + test_params.append((bi, comprenc_kind)) + + metafunc.parametrize("backend", test_params, indirect=True, + ids=[ '%s/%s' % (x[0].name, x[1]) + for x in test_params] ) + +@pytest.yield_fixture() +def backend(request): + (backend_info, comprenc_kind) = request.param + + if backend_info.classname == 'local': + gen = yield_local_backend(backend_info) + elif 'request_handler' in backend_info: + gen = yield_mock_backend(backend_info) + else: + gen = yield_remote_backend(backend_info) + + for raw_backend in gen: + if comprenc_kind == 'raw': + backend = raw_backend + elif comprenc_kind == 'plain': + backend = ComprencBackend(None, (None, 6), raw_backend) + elif comprenc_kind == 'aes+zlib': + backend = ComprencBackend(b'schlurz', ('zlib', 6), raw_backend) + elif comprenc_kind == 'aes': + backend = ComprencBackend(b'schlurz', (None, 6), raw_backend) + else: + backend = ComprencBackend(None, (comprenc_kind, 6), raw_backend) + + backend.unittest_info = raw_backend.unittest_info + yield backend + +def yield_local_backend(bi): + backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') + backend = LocalBackend('local://' + backend_dir, None, None) + backend.unittest_info = Namespace() + backend.unittest_info.retry_time = 0 + try: + yield backend + finally: + backend.close() + shutil.rmtree(backend_dir) + +def yield_mock_backend(bi): + backend_class = backends.prefix_map[bi.classname] + server = mock_server.StorageServer(bi.request_handler, ('localhost', 0)) + thread = threading.Thread(target=server.serve_forever) + thread.daemon = True + thread.start() + + storage_url = bi.storage_url % { 'host': server.server_address[0], + 'port': server.server_address[1] } + backend = backend_class(storage_url, 'joe', 'swordfish', { 'no-ssl': True }) + + # Enable OAuth when using Google Backend + if isinstance(backend, GSBackend): + backend.use_oauth2 = True + backend.hdr_prefix = 'x-goog-' # Normally set in __init__ + backend.access_token[backend.password] = 'foobar' + + backend.unittest_info = Namespace() + backend.unittest_info.retry_time = 0 + + # Mock server should not have temporary failures by default + is_temp_failure = backend.is_temp_failure + @functools.wraps(backend.is_temp_failure) + def wrap(exc): + return backend.unittest_info.may_temp_fail and is_temp_failure(exc) + backend.is_temp_failure = wrap + backend.unittest_info.may_temp_fail = False + + try: + yield backend + finally: + backend.close() + server.server_close() + server.shutdown() + +def yield_remote_backend(bi, _ctr=[0]): + # Add timestamp + ctr to prefix so we don't have to deal with cruft from + # previous tests + _ctr[0] += 1 + storage_url = bi.storage_url + if storage_url[-1] != '/': + storage_url += '/' + storage_url += '%d_%d/' % (time.time(), _ctr[0]) + + backend_class = backends.prefix_map[bi.classname] + backend = backend_class(storage_url, bi.login, bi.password, {}) + + backend.unittest_info = Namespace() + backend.unittest_info.retry_time = 600 + try: + yield backend + finally: + backend.clear() + backend.close() + +def newname(name_counter=[0]): + '''Return random, unique string''' + name_counter[0] += 1 + return "random\\'name' %d" % name_counter[0] + +def newvalue(): + return newname().encode() + +def fetch_object(backend, key, sleep_time=1): + '''Read data and metadata for *key* from *backend* + + If `NoSuchObject` exception is encountered, retry for + up to *retry_time* seconds. + ''' + waited=0 + retry_time = backend.unittest_info.retry_time + while True: + try: + return backend.fetch(key) + except NoSuchObject: + if waited >= retry_time: + raise + time.sleep(sleep_time) + waited += sleep_time + +def lookup_object(backend, key, sleep_time=1): + '''Read metadata for *key* from *backend* + + If `NoSuchObject` exception is encountered, retry for + up to *retry_time* seconds. + ''' + retry_time = backend.unittest_info.retry_time + waited=0 + while True: + try: + return backend.lookup(key) + except NoSuchObject: + if waited >= retry_time: + raise + time.sleep(sleep_time) + waited += sleep_time + +def assert_in_index(backend, keys, sleep_time=1): + '''Assert that *keys* will appear in index + + Raises assertion error if *keys* do not show up within + *retry_time* seconds. + ''' + waited=0 + retry_time = backend.unittest_info.retry_time + keys = set(keys) # copy + while True: + index = set(backend.list()) + if not keys - index: + return + elif waited >= retry_time: + assert keys - index == empty_set + time.sleep(sleep_time) + waited += sleep_time + +def assert_not_in_index(backend, keys, sleep_time=1): + '''Assert that *keys* will disappear from index + + Raises assertion error if *keys* do not disappear within + *retry_time* seconds. + ''' + retry_time = backend.unittest_info.retry_time + waited=0 + keys = set(keys) # copy + while True: + index = set(backend.list()) + if keys - index == keys: + return + elif waited >= retry_time: + assert keys - index == keys + time.sleep(sleep_time) + waited += sleep_time + +def assert_not_readable(backend, key, sleep_time=1): + '''Assert that *key* does not exist in *backend* + + Asserts that a `NoSuchObject` exception will be raised when trying to read + the object after at most *retry_time* seconds. + ''' + waited=0 + retry_time = backend.unittest_info.retry_time + while True: + try: + backend.fetch(key) + except NoSuchObject: + return + if waited >= retry_time: + pytest.fail('object %s still present in backend' % key) + time.sleep(sleep_time) + waited += sleep_time + +@pytest.mark.with_backend('*/*') +def test_read_write(backend): + key = newname() + value = newvalue() + metadata = { 'jimmy': 'jups@42' } + + assert key not in backend + assert_raises(NoSuchObject, backend.lookup, key) + assert_raises(NoSuchObject, backend.fetch, key) + + def do_write(fh): + fh.write(value) + backend.perform_write(do_write, key, metadata) + + assert_in_index(backend, [key]) + (value2, metadata2) = fetch_object(backend, key) + + assert value == value2 + assert metadata == metadata2 + assert lookup_object(backend, key) == metadata + +@pytest.mark.with_backend('swift/raw') +def test_issue114(backend, monkeypatch): + key = newname() + value = newvalue() + monkeypatch.setitem(backend.options, 'disable-expect100', True) + backend[key] = value + assert_in_index(backend, [key]) + +@pytest.mark.with_backend('*/raw', 'local/{plain,aes,zlib}') +def test_complex_meta(backend): + key = newname() + value = newvalue() + + metadata = { 'com\nplex: key': 42, + 'farp_': False, 'non-farp': True, + 'blu%rz': 23.283475, + 'görp': b'heelo', + 'sch.al': 'gorroobalp\nfurrö!', + 'lo-ng': 'foobarz' * 40 } + + assert key not in backend + backend.store(key, value, metadata) + (value2, metadata2) = fetch_object(backend, key) + + assert value == value2 + assert metadata == metadata2 + assert lookup_object(backend, key) == metadata + +# No need to run with different encryption/compression settings, +# ComprencBackend should just forward this 1:1 to the raw backend. +@pytest.mark.with_backend('*/aes') +def test_list(backend): + keys = ([ 'prefixa' + newname() for dummy in range(6) ] + + [ 'prefixb' + newname() for dummy in range(6) ]) + values = [ newvalue() for dummy in range(12) ] + + assert set(backend.list()) == empty_set + for i in range(12): + backend[keys[i]] = values[i] + assert_in_index(backend, keys) + + assert set(backend.list('prefixa')) == set(keys[:6]) + assert set(backend.list('prefixb')) == set(keys[6:]) + assert set(backend.list('prefixc')) == empty_set + +@pytest.mark.with_backend('*/raw', 'local/{plain,aes,zlib,aes+zlib}', + require_immediate_consistency=True) +def test_readslowly(backend): + key = newname() + value = newvalue() + metadata = { 'jimmy': 'jups@42' } + + backend.store(key, value, metadata) + + s3ql.backends.common.BUFSIZE = 1 + try: + with backend.open_read(key) as fh: + # Force slow reading from underlying layer + if hasattr(fh, 'fh'): + def read_slowly(size, *, real_read=fh.fh.read): + return real_read(1) + fh.fh.read = read_slowly + + buf = [] + while True: + buf.append(fh.read(1)) + if not buf[-1]: + break + value2 = b''.join(buf) + metadata2 = fh.metadata + finally: + s3ql.backends.common.BUFSIZE = BUFSIZE + + assert value == value2 + assert metadata == metadata2 + +# No need to run with different encryption/compression settings, +# ComprencBackend should just forward this 1:1 to the raw backend. +@pytest.mark.with_backend('*/aes') +def test_delete(backend): + key = newname() + value = newvalue() + + backend[key] = value + + # Wait for object to become visible + assert_in_index(backend, [key]) + fetch_object(backend, key) + + # Delete it + del backend[key] + + # Make sure that it's truly gone + assert_not_in_index(backend, [key]) + assert_not_readable(backend, key) + +# No need to run with different encryption/compression settings, +# ComprencBackend should just forward this 1:1 to the raw backend. +@pytest.mark.with_backend('*/aes') +def test_delete_multi(backend): + keys = [ newname() for _ in range(30) ] + value = newvalue() + + # Create objects + for key in keys: + backend[key] = value + + # Wait for them + assert_in_index(backend, keys) + for key in keys: + fetch_object(backend, key) + + # Delete half of them + # We don't use force=True but catch the exemption to increase the + # chance that some existing objects are not deleted because of the + # error. + to_delete = keys[::2] + to_delete.insert(7, 'not_existing') + try: + backend.delete_multi(to_delete) + except NoSuchObject: + pass + + # Without full consistency, deleting an non-existing object + # may not give an error + assert backend.unittest_info.retry_time or len(to_delete) > 0 + + deleted = set(keys[::2]) - set(to_delete) + assert len(deleted) > 0 + remaining = set(keys) - deleted + + assert_not_in_index(backend, deleted) + for key in deleted: + assert_not_readable(backend, key) + + assert_in_index(backend, remaining) + for key in remaining: + fetch_object(backend, key) + +# No need to run with different encryption/compression settings, +# ComprencBackend should just forward this 1:1 to the raw backend. +@pytest.mark.with_backend('*/aes') +def test_clear(backend): + keys = [ newname() for _ in range(5) ] + value = newvalue() + + # Create objects + for key in keys: + backend[key] = value + + # Wait for them + assert_in_index(backend, keys) + for key in keys: + fetch_object(backend, key) + + # Delete everything + backend.clear() + + assert_not_in_index(backend, keys) + for key in keys: + assert_not_readable(backend, key) + +@pytest.mark.with_backend('*/raw', 'local/{plain,aes,zlib}') +def test_copy(backend): + key1 = newname() + key2 = newname() + value = newvalue() + metadata = { 'jimmy': 'jups@42' } + + backend.store(key1, value, metadata) + + # Wait for object to become visible + assert_in_index(backend, [key1]) + fetch_object(backend, key1) + + assert_not_in_index(backend, [key2]) + assert_not_readable(backend, key2) + + backend.copy(key1, key2) + + assert_in_index(backend, [key2]) + (value2, metadata2) = fetch_object(backend, key2) + + assert value == value2 + assert metadata == metadata2 + +@pytest.mark.with_backend('*/raw') +def test_copy_special(backend): + key1 = 'with_+_char/orig' + key2 = 'with_+_char/dest' + value = b'Just a couple of random bytes' + + backend.store(key1, value) + + assert_not_in_index(backend, [key2]) + assert_not_readable(backend, key2) + + # Wait for object to become visible + assert_in_index(backend, [key1]) + fetch_object(backend, key1) + + backend.copy(key1, key2) + + assert_in_index(backend, [key2]) + value2 = backend[key2] + assert value == value2 + +@pytest.mark.with_backend('*/raw', 'local/{aes,zlib}') +def test_copy_newmeta(backend): + key1 = newname() + key2 = newname() + value = newvalue() + meta1 = { 'jimmy': 'jups@42' } + meta2 = { 'jiy': 'jfobauske42' } + + backend.store(key1, value, meta1) + + # Wait for object to become visible + assert_in_index(backend, [key1]) + fetch_object(backend, key1) + + assert_not_in_index(backend, [key2]) + assert_not_readable(backend, key2) + + backend.copy(key1, key2, meta2) + + assert_in_index(backend, [key2]) + (value2, meta) = fetch_object(backend, key2) + + assert value == value2 + assert meta == meta2 + +@pytest.mark.with_backend('*/raw', 'local/{aes,zlib}') +def test_rename(backend): + key1 = newname() + key2 = newname() + value = newvalue() + metadata = { 'jimmy': 'jups@42' } + + backend.store(key1, value, metadata) + + # Wait for object to become visible + assert_in_index(backend, [key1]) + fetch_object(backend, key1) + + assert_not_in_index(backend, [key2]) + assert_not_readable(backend, key2) + + backend.rename(key1, key2) + + assert_in_index(backend, [key2]) + (value2, metadata2) = fetch_object(backend, key2) + + assert value == value2 + assert metadata == metadata2 + + assert_not_in_index(backend, [key1]) + assert_not_readable(backend, key1) + +@pytest.mark.with_backend('*/raw', 'local/{aes,zlib}') +def test_rename_newmeta(backend): + key1 = newname() + key2 = newname() + value = newvalue() + meta1 = { 'jimmy': 'jups@42' } + meta2 = { 'apple': 'potatoes' } + + backend.store(key1, value, meta1) + + # Wait for object to become visible + assert_in_index(backend, [key1]) + fetch_object(backend, key1) + + assert_not_in_index(backend, [key2]) + assert_not_readable(backend, key2) + + backend.rename(key1, key2, meta2) + + assert_in_index(backend, [key2]) + (value2, meta) = fetch_object(backend, key2) + + assert value == value2 + assert meta == meta2 + +@pytest.mark.with_backend('*/raw', 'local/{aes,zlib}') +def test_update_meta(backend): + key = 'simple' + value = b'not too hard' + meta1 = { 'jimmy': 'jups@42' } + meta2 = { 'apple': 'potatoes' } + + backend.store(key, value, meta1) + + # Wait for object to become visible + assert_in_index(backend, [key]) + fetch_object(backend, key) + + backend.update_meta(key, meta2) + + # Wait for updated metadata + waited=0 + sleep_time = 1 + while True: + (value2, meta) = fetch_object(backend, key) + if meta != meta1: + break + elif waited >= backend.unittest_info.retry_time: + pytest.fail('metadata for %s not updated after %d seconds' + % (key, waited)) + time.sleep(sleep_time) + waited += sleep_time + + assert value == value2 + assert meta == meta2 + +# Choice of compression algorithm should not make a difference +@pytest.mark.with_backend('s3c/{raw,aes,zlib}', + require_mock_server=True) +def test_copy_error(backend, monkeypatch): + value = b'hello there, let us see whats going on' + key1 = 'object-key1' + key2 = 'object-key2' + backend[key1] = value + + # Monkeypatch request handler to produce error + handler_class = mock_server.S3CRequestHandler + def do_PUT(self, real_PUT=handler_class.do_PUT, count=[0]): + count[0] += 1 + if count[0] > 3: + return real_PUT(self) + else: + self.send_error(200, code='OperationAborted') + monkeypatch.setattr(handler_class, 'do_PUT', do_PUT) + assert_raises(OperationAbortedError, backend.copy, key1, key2) + + enable_temp_fail(backend) + backend.copy(key1, key2) + +@pytest.mark.with_backend('local/{aes,aes+zlib,zlib,bzip2,lzma}') +def test_corruption(backend): + plain_backend = backend.backend + + # Create compressed object + key = newname() + value = newvalue() + backend[key] = value + + # Retrieve compressed data + (compr_value, meta) = fetch_object(plain_backend, key) + compr_value = bytearray(compr_value) + + # Overwrite with corrupted data + # (this needs immediate consistency) + compr_value[-3:] = b'000' + plain_backend.store(key, compr_value, meta) + + with pytest.raises(CorruptedObjectError) as exc: + fetch_object(backend, key) + + if backend.passphrase is None: # compression only + assert exc.value.str == 'Invalid compressed stream' + else: + assert exc.value.str == 'HMAC mismatch' + +@pytest.mark.with_backend('local/{aes,aes+zlib,zlib,bzip2,lzma}') +def test_extra_data(backend): + plain_backend = backend.backend + + # Create compressed object + key = newname() + value = newvalue() + backend[key] = value + + # Retrieve compressed data + (compr_value, meta) = fetch_object(plain_backend, key) + compr_value = bytearray(compr_value) + + # Overwrite with extended data + # (this needs immediate consistency) + compr_value += b'000' + plain_backend.store(key, compr_value, meta) + + with pytest.raises(CorruptedObjectError) as exc: + fetch_object(backend, key) + + if backend.passphrase is None: # compression only + assert exc.value.str == 'Data after end of compressed stream' + else: + assert exc.value.str == 'Extraneous data at end of object' + +@pytest.mark.with_backend('*/{raw,plain,aes,aes+zlib,zlib}') +def test_multi_packet(backend): + '''Write and read packet extending over multiple chunks''' + key = newname() + + def do_write(fh): + for i in range(5): + fh.write(b'\xFF' * BUFSIZE) + backend.perform_write(do_write, key) + + def do_read(fh): + buf = bytearray() + while True: + tmp = fh.read(BUFSIZE//2) + if not tmp: + break + buf += tmp + return buf + res = backend.perform_read(do_read, key) + assert res == b'\xFF' * (5*BUFSIZE) + +@pytest.mark.with_backend('local/{raw,plain,aes,aes+zlib,zlib}') +def test_issue431(backend): + key = newname() + hdr_len = struct.calcsize(b' 3: + return real_GET(self) + else: + self.send_error(503, code='OperationAborted') + monkeypatch.setattr(handler_class, 'do_GET', do_GET) + assert_raises(OperationAbortedError, backend.fetch, value) + + enable_temp_fail(backend) + assert backend[key] == value + +@pytest.mark.with_backend('s3c/{raw,aes+zlib}', + require_mock_server=True) +def test_head_s3error(backend, monkeypatch): + value = b'hello there, let us see whats going on' + key = 'quote' + meta = {'bar': 42, 'foo': 42**2} + backend.store(key, value, metadata=meta) + + # Monkeypatch request handler to produce 3 errors + handler_class = mock_server.S3CRequestHandler + def do_HEAD(self, real_HEAD=handler_class.do_HEAD, count=[0]): + count[0] += 1 + if count[0] > 3: + return real_HEAD(self) + else: + self.send_error(503, code='OperationAborted') + monkeypatch.setattr(handler_class, 'do_HEAD', do_HEAD) + with pytest.raises(HTTPError) as exc: + backend.lookup(key) + assert exc.value.status == 503 + + enable_temp_fail(backend) + assert backend.lookup(key) == meta + +@pytest.mark.with_backend('s3c/raw', require_mock_server=True) +def test_delete_s3error(backend, monkeypatch): + value = b'hello there, let us see whats going on' + key = 'quote' + backend[key] = value + + # Monkeypatch request handler to produce 3 errors + handler_class = mock_server.S3CRequestHandler + def do_DELETE(self, real_DELETE=handler_class.do_DELETE, count=[0]): + count[0] += 1 + if count[0] > 3: + return real_DELETE(self) + else: + self.send_error(503, code='OperationAborted') + monkeypatch.setattr(handler_class, 'do_DELETE', do_DELETE) + assert_raises(OperationAbortedError, backend.delete, key) + + enable_temp_fail(backend) + backend.delete(key) + +@pytest.mark.with_backend('s3c/raw', require_mock_server=True) +def test_backoff(backend, monkeypatch): + value = b'hello there, let us see whats going on' + key = 'quote' + backend[key] = value + + # Monkeypatch request handler + handler_class = mock_server.S3CRequestHandler + timestamps = [] + def do_DELETE(self, real_DELETE=handler_class.do_DELETE): + timestamps.append(time.time()) + if len(timestamps) < 3: + self.send_error(503, code='SlowDown', + extra_headers={'Retry-After': '1'}) + else: + return real_DELETE(self) + + monkeypatch.setattr(handler_class, 'do_DELETE', do_DELETE) + enable_temp_fail(backend) + backend.delete(key) + + assert timestamps[1] - timestamps[0] > 1 - CLOCK_GRANULARITY + assert timestamps[2] - timestamps[1] > 1 - CLOCK_GRANULARITY + assert timestamps[2] - timestamps[0] < 10 + +@pytest.mark.with_backend('s3c/raw', require_mock_server=True) +def test_httperror(backend, monkeypatch): + value = b'hello there, let us see whats going on' + key = 'quote' + backend[key] = value + + # Monkeypatch request handler to produce a HTTP Error + handler_class = mock_server.S3CRequestHandler + def do_DELETE(self, real_DELETE=handler_class.do_DELETE, count=[0]): + count[0] += 1 + if count[0] >= 3: + return real_DELETE(self) + content = "I'm a proxy, and I messed up!".encode('utf-8') + self.send_response(502, "Bad Gateway") + self.send_header("Content-Type", 'text/plain; charset="utf-8"') + self.send_header("Content-Length", str(len(content))) + self.end_headers() + if self.command != 'HEAD': + self.wfile.write(content) + + monkeypatch.setattr(handler_class, 'do_DELETE', do_DELETE) + assert_raises(HTTPError, backend.delete, key) + + enable_temp_fail(backend) + backend.delete(key) + +@pytest.mark.with_backend('s3c/{raw,aes+zlib}', + require_mock_server=True) +def test_put_s3error_early(backend, monkeypatch): + '''Fail after expect-100''' + + data = b'hello there, let us see whats going on' + key = 'borg' + + # Monkeypatch request handler to produce 3 errors + handler_class = mock_server.S3CRequestHandler + def handle_expect_100(self, real=handler_class.handle_expect_100, count=[0]): + count[0] += 1 + if count[0] > 3: + return real(self) + else: + self.send_error(503, code='OperationAborted') + return False + monkeypatch.setattr(handler_class, 'handle_expect_100', handle_expect_100) + fh = backend.open_write(key) + fh.write(data) + assert_raises(OperationAbortedError, fh.close) + + enable_temp_fail(backend) + fh.close() + +@pytest.mark.with_backend('s3c/{raw,aes+zlib}', + require_mock_server=True) +def test_put_s3error_med(backend, monkeypatch): + '''Fail as soon as data is received''' + data = b'hello there, let us see whats going on' + key = 'borg' + + # Monkeypatch request handler to produce 3 errors + handler_class = mock_server.S3CRequestHandler + def do_PUT(self, real_PUT=handler_class.do_PUT, count=[0]): + count[0] += 1 + # Note: every time we return an error, the request will be retried + # *twice*: once because of the error, and a second time because the + # connection has been closed by the server. + if count[0] > 2: + return real_PUT(self) + else: + self.send_error(503, code='OperationAborted') + + # Since we don't read all the data, we have to close + # the connection + self.close_connection = True + + monkeypatch.setattr(handler_class, 'do_PUT', do_PUT) + fh = backend.open_write(key) + fh.write(data) + assert_raises(OperationAbortedError, fh.close) + + enable_temp_fail(backend) + fh.close() + +@pytest.mark.with_backend('s3c/{raw,aes+zlib}', + require_mock_server=True) +def test_put_s3error_late(backend, monkeypatch): + '''Fail after reading all data''' + data = b'hello there, let us see whats going on' + key = 'borg' + + # Monkeypatch request handler to produce 3 errors + handler_class = mock_server.S3CRequestHandler + def do_PUT(self, real_PUT=handler_class.do_PUT, count=[0]): + count[0] += 1 + if count[0] > 3: + return real_PUT(self) + else: + self.rfile.read(int(self.headers['Content-Length'])) + self.send_error(503, code='OperationAborted') + + monkeypatch.setattr(handler_class, 'do_PUT', do_PUT) + fh = backend.open_write(key) + fh.write(data) + assert_raises(OperationAbortedError, fh.close) + + enable_temp_fail(backend) + fh.close() + +@pytest.mark.with_backend('s3c/{raw,aes+zlib}', + require_mock_server=True) +def test_issue58(backend, monkeypatch): + '''Send error while client is sending data''' + + # Monkeypatch request handler + handler_class = mock_server.S3CRequestHandler + def do_PUT(self, real=handler_class.do_PUT, count=[0]): + count[0] += 1 + if count[0] > 1: + return real(self) + + # Read half the data + self.rfile.read(min(BUFSIZE, int(self.headers['Content-Length'])//2)) + + # Then generate an error and close the connection + self.send_error(401, code='MalformedXML') + self.close_connection = True + + monkeypatch.setattr(handler_class, 'do_PUT', do_PUT) + + # Write a big object. We need to write random data, or + # compression while make the payload too small + with pytest.raises(S3Error) as exc_info: + with backend.open_write('borg') as fh, \ + open('/dev/urandom', 'rb') as rnd: + for _ in range(5): + fh.write(rnd.read(BUFSIZE)) + assert exc_info.value.code == 'MalformedXML' + + enable_temp_fail(backend) + fh.close() + +@pytest.mark.with_backend('s3c/{raw,aes+zlib}', + require_mock_server=True) +def test_issue58_b(backend, monkeypatch): + '''Close connection while client is sending data''' + + # Monkeypatch request handler + handler_class = mock_server.S3CRequestHandler + def do_PUT(self, real=handler_class.do_PUT, count=[0]): + count[0] += 1 + if count[0] > 1: + return real(self) + + # Read half the data + self.rfile.read(min(BUFSIZE, int(self.headers['Content-Length'])//2)) + + # Then close the connection silently + self.close_connection = True + monkeypatch.setattr(handler_class, 'do_PUT', do_PUT) + + # Write a big object. We need to write random data, or + # compression while make the payload too small + with pytest.raises(ConnectionClosed): + with backend.open_write('borg') as fh, \ + open('/dev/urandom', 'rb') as rnd: + for _ in range(5): + fh.write(rnd.read(BUFSIZE)) + + enable_temp_fail(backend) + fh.close() + +@pytest.mark.with_backend('gs/{raw,aes+zlib}', + require_mock_server=True) +def test_expired_token_get(backend, monkeypatch): + '''Test handling of expired OAuth token''' + + key = 'borg' + data = b'hello there, let us see whats going on' + + # Monkeypatch backend class to check if token is refreshed + token_refreshed = False + def _get_access_token(self): + nonlocal token_refreshed + token_refreshed = True + self.access_token[self.password] = 'foobar' + monkeypatch.setattr(GSBackend, '_get_access_token', + _get_access_token) + + # Store some data + backend[key] = data + + # Monkeypatch request handler to produce error + handler_class = mock_server.GSRequestHandler + def do_GET(self, real=handler_class.do_GET, count=[0]): + count[0] += 1 + if count[0] > 1: + return real(self) + else: + self.send_error(401, code='AuthenticationRequired') + monkeypatch.setattr(handler_class, 'do_GET', do_GET) + + token_refreshed = False + assert backend[key] == data + assert token_refreshed + +@pytest.mark.with_backend('gs/{raw,aes+zlib}', + require_mock_server=True) +def test_expired_token_put(backend, monkeypatch): + '''Test handling of expired OAuth token''' + + key = 'borg' + data = b'hello there, let us see whats going on' + + # Monkeypatch backend class to check if token is refreshed + token_refreshed = False + def _get_access_token(self): + nonlocal token_refreshed + token_refreshed = True + self.access_token[self.password] = 'foobar' + monkeypatch.setattr(GSBackend, '_get_access_token', + _get_access_token) + + # Monkeypatch request handler to produce error + handler_class = mock_server.GSRequestHandler + def do_PUT(self, real=handler_class.do_PUT, count=[0]): + count[0] += 1 + if count[0] > 1: + return real(self) + else: + self.rfile.read(int(self.headers['Content-Length'])) + self.send_error(401, code='AuthenticationRequired') + monkeypatch.setattr(handler_class, 'do_PUT', do_PUT) + + token_refreshed = False + backend[key] = data + assert token_refreshed + +@pytest.mark.with_backend('s3c/{raw,aes+zlib}', + require_mock_server=True) +def test_conn_abort(backend, monkeypatch): + '''Close connection while sending data''' + + data = b'hello there, let us see whats going on' + key = 'borg' + backend[key] = data + + # Monkeypatch request handler + handler_class = mock_server.S3CRequestHandler + def send_data(self, data, count=[0]): + count[0] += 1 + if count[0] >= 3: + self.wfile.write(data) + else: + self.wfile.write(data[:len(data)//2]) + self.close_connection = True + monkeypatch.setattr(handler_class, 'send_data', send_data) + + with pytest.raises(ConnectionClosed): + with assert_logs("^Object closed prematurely, can't check MD5", + count=1, level=logging.WARNING): + backend.fetch(key) + + enable_temp_fail(backend) + assert backend[key] == data diff --git a/tests/t1_dump.py b/tests/t1_dump.py new file mode 100755 index 0000000..cab1ff6 --- /dev/null +++ b/tests/t1_dump.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +''' +t1_dump.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +import unittest +from s3ql import deltadump +import tempfile +from s3ql.database import Connection +import random +import time + +class DumpTests(unittest.TestCase): + def setUp(self): + self.tmpfh1 = tempfile.NamedTemporaryFile() + self.tmpfh2 = tempfile.NamedTemporaryFile() + self.src = Connection(self.tmpfh1.name) + self.dst = Connection(self.tmpfh2.name) + self.fh = tempfile.TemporaryFile() + + # Disable exclusive locking for all tests + self.src.execute('PRAGMA locking_mode = NORMAL') + self.dst.execute('PRAGMA locking_mode = NORMAL') + + self.create_table(self.src) + self.create_table(self.dst) + + def tearDown(self): + self.src.close() + self.dst.close() + self.tmpfh1.close() + self.tmpfh2.close() + self.fh.close() + + def test_transactions(self): + self.fill_vals(self.src) + dumpspec = (('id', deltadump.INTEGER, 0),) + deltadump.dump_table(table='test', order='id', columns=dumpspec, + db=self.src, fh=self.fh) + self.fh.seek(0) + self.dst.execute('PRAGMA journal_mode = WAL') + + deltadump.load_table(table='test', columns=dumpspec, db=self.dst, + fh=self.fh, trx_rows=10) + self.compare_tables(self.src, self.dst) + + def test_1_vals_1(self): + self.fill_vals(self.src) + dumpspec = (('id', deltadump.INTEGER, 0),) + deltadump.dump_table(table='test', order='id', columns=dumpspec, + db=self.src, fh=self.fh) + self.fh.seek(0) + deltadump.load_table(table='test', columns=dumpspec, db=self.dst, + fh=self.fh) + self.compare_tables(self.src, self.dst) + + def test_1_vals_2(self): + self.fill_vals(self.src) + dumpspec = (('id', deltadump.INTEGER, 1),) + deltadump.dump_table(table='test', order='id', columns=dumpspec, + db=self.src, fh=self.fh) + self.fh.seek(0) + deltadump.load_table(table='test', columns=dumpspec, db=self.dst, + fh=self.fh) + self.compare_tables(self.src, self.dst) + + def test_1_vals_3(self): + self.fill_vals(self.src) + dumpspec = (('id', deltadump.INTEGER, -1),) + deltadump.dump_table(table='test', order='id', columns=dumpspec, + db=self.src, fh=self.fh) + self.fh.seek(0) + deltadump.load_table(table='test', columns=dumpspec, db=self.dst, + fh=self.fh) + self.compare_tables(self.src, self.dst) + + def test_2_buf_auto(self): + self.fill_vals(self.src) + self.fill_buf(self.src) + dumpspec = (('id', deltadump.INTEGER), + ('buf', deltadump.BLOB)) + deltadump.dump_table(table='test', order='id', columns=dumpspec, + db=self.src, fh=self.fh) + self.fh.seek(0) + deltadump.load_table(table='test', columns=dumpspec, db=self.dst, + fh=self.fh) + self.compare_tables(self.src, self.dst) + + def test_2_buf_fixed(self): + BUFLEN = 32 + self.fill_vals(self.src) + self.fill_buf(self.src, BUFLEN) + dumpspec = (('id', deltadump.INTEGER), + ('buf', deltadump.BLOB, BUFLEN)) + deltadump.dump_table(table='test', order='id', columns=dumpspec, + db=self.src, fh=self.fh) + self.fh.seek(0) + deltadump.load_table(table='test', columns=dumpspec, db=self.dst, + fh=self.fh) + self.compare_tables(self.src, self.dst) + + def test_3_deltas_1(self): + self.fill_deltas(self.src) + dumpspec = (('id', deltadump.INTEGER, 0),) + deltadump.dump_table(table='test', order='id', columns=dumpspec, + db=self.src, fh=self.fh) + self.fh.seek(0) + deltadump.load_table(table='test', columns=dumpspec, db=self.dst, + fh=self.fh) + self.compare_tables(self.src, self.dst) + + def test_3_deltas_2(self): + self.fill_deltas(self.src) + dumpspec = (('id', deltadump.INTEGER, 1),) + deltadump.dump_table(table='test', order='id', columns=dumpspec, + db=self.src, fh=self.fh) + self.fh.seek(0) + deltadump.load_table(table='test', columns=dumpspec, db=self.dst, + fh=self.fh) + self.compare_tables(self.src, self.dst) + + def test_3_deltas_3(self): + self.fill_deltas(self.src) + dumpspec = (('id', deltadump.INTEGER, -1),) + deltadump.dump_table(table='test', order='id', columns=dumpspec, + db=self.src, fh=self.fh) + self.fh.seek(0) + deltadump.load_table(table='test', columns=dumpspec, db=self.dst, + fh=self.fh) + self.compare_tables(self.src, self.dst) + + def test_4_time(self): + self.fill_vals(self.src) + + t1 = 0.5 * time.time() + t2 = 2 * time.time() + for (id_,) in self.src.query('SELECT id FROM test'): + val = random.uniform(t1, t2) + self.src.execute('UPDATE test SET buf=? WHERE id=?', (val, id_)) + + dumpspec = (('id', deltadump.INTEGER), + ('buf', deltadump.TIME)) + + deltadump.dump_table(table='test', order='id', columns=dumpspec, + db=self.src, fh=self.fh) + self.fh.seek(0) + deltadump.load_table(table='test', columns=dumpspec, db=self.dst, + fh=self.fh) + + self.compare_tables(self.src, self.dst) + + + def test_5_multi(self): + self.fill_vals(self.src) + dumpspec = (('id', deltadump.INTEGER, 0),) + deltadump.dump_table(table='test', order='id', columns=dumpspec, + db=self.src, fh=self.fh) + deltadump.dump_table(table='test', order='id', columns=dumpspec, + db=self.src, fh=self.fh) + self.fh.seek(0) + deltadump.load_table(table='test', columns=dumpspec, db=self.dst, + fh=self.fh) + self.dst.execute('DELETE FROM test') + deltadump.load_table(table='test', columns=dumpspec, db=self.dst, + fh=self.fh) + self.compare_tables(self.src, self.dst) + + + def compare_tables(self, db1, db2): + i1 = db1.query('SELECT id, buf FROM test ORDER BY id') + i2 = db2.query('SELECT id, buf FROM test ORDER BY id') + + for (id1, buf1) in i1: + (id2, buf2) = next(i2) + + self.assertEqual(id1, id2) + if isinstance(buf1, float): + self.assertAlmostEqual(buf1, buf2, places=9) + else: + self.assertEqual(buf1, buf2) + + self.assertRaises(StopIteration, i2.__next__) + + def fill_buf(self, db, len_=None): + with open('/dev/urandom', 'rb') as rfh: + first = True + for (id_,) in db.query('SELECT id FROM test'): + if len_ is None and first: + val = b'' # We always want to check this case + first = False + elif len_ is None: + val = rfh.read(random.randint(0, 140)) + else: + val = rfh.read(len_) + + db.execute('UPDATE test SET buf=? WHERE id=?', (val, id_)) + + def fill_vals(self, db): + vals = [] + for exp in [7, 8, 9, 15, 16, 17, 31, 32, 33, 62]: + vals += list(range(2 ** exp - 5, 2 ** exp + 6)) + vals += list(range(2 ** 63 - 5, 2 ** 63)) + vals += [ -v for v in vals ] + vals.append(-(2 ** 63)) + + for val in vals: + db.execute('INSERT INTO test (id) VALUES(?)', (val,)) + + def fill_deltas(self, db): + deltas = [] + for exp in [7, 8, 9, 15, 16, 17, 31, 32, 33]: + deltas += list(range(2 ** exp - 5, 2 ** exp + 6)) + deltas += [ -v for v in deltas ] + + last = 0 + for delta in deltas: + val = last + delta + last = val + db.execute('INSERT INTO test (id) VALUES(?)', (val,)) + + def create_table(self, db): + db.execute('''CREATE TABLE test ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + buf BLOB)''') diff --git a/tests/t1_retry.py b/tests/t1_retry.py new file mode 100755 index 0000000..4a4ed6e --- /dev/null +++ b/tests/t1_retry.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +''' +t1_retry.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +from s3ql.backends.common import retry, retry_generator +from pytest_checklogs import assert_logs +import logging + +class TemporaryProblem(Exception): + pass + +class NthAttempt: + def __init__(self, succeed_on=3): + self.count = 0 + self.succeed_on = succeed_on + + @staticmethod + def is_temp_failure(exc): + return isinstance(exc, TemporaryProblem) + + @retry + def do_stuff(self): + if self.count == self.succeed_on: + return True + self.count += 1 + raise TemporaryProblem() + + @retry_generator + def list_stuff(self, upto=10, start_after=-1): + for i in range(upto): + if i <= start_after: + continue + if i == 2 and self.count < 1: + self.count += 1 + raise TemporaryProblem + + if i == 7 and self.count < 4: + self.count += 1 + raise TemporaryProblem + + yield i + + @retry + def test_is_retry(self, is_retry=False): + assert is_retry == (self.count != 0) + if self.count == self.succeed_on: + return True + self.count += 1 + raise TemporaryProblem() + +def test_retry(): + inst = NthAttempt(3) + + assert inst.do_stuff() + +def test_retry_generator(): + inst = NthAttempt(3) + assert list(inst.list_stuff(10)) == list(range(10)) + +def test_is_retry(): + inst = NthAttempt(3) + assert inst.test_is_retry() + +def test_logging(): + inst = NthAttempt(6) + with assert_logs(r'^Encountered %s \(%s\), retrying ', + count=2, level=logging.WARNING): + inst.do_stuff() diff --git a/tests/t1_serialization.py b/tests/t1_serialization.py new file mode 100755 index 0000000..7258c93 --- /dev/null +++ b/tests/t1_serialization.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +''' +t1_serialization.py - this file is part of S3QL. + +Copyright © 2014 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +from s3ql.common import ThawError, freeze_basic_mapping, thaw_basic_mapping +from s3ql.backends.common import checksum_basic_mapping +import pytest +from collections import OrderedDict + + +def test_simple(): + d = { 'hello': 42, + 'world': True, + 'data': b'fooxyz', + 'str': 'nice' } + + assert thaw_basic_mapping(freeze_basic_mapping(d)) == d + +def test_wrong_key(): + d = { 'hello': 42, + True: False } + + with pytest.raises(ValueError): + freeze_basic_mapping(d) + +def test_cmplx_value(): + d = { 'hello': [1,2] } + + with pytest.raises(ValueError): + freeze_basic_mapping(d) + +def test_thaw_errors(): + buf = freeze_basic_mapping({ 'hello': 'world' }) + + for s in (buf[1:], buf[:-1], + b'"foo"[2]', b'open("/dev/null", "r")', + b'"foo".__class__'): + with pytest.raises(ThawError): + thaw_basic_mapping(s) + +def test_checksum(): + d1 = OrderedDict() + d2 = OrderedDict() + + d1['foo'] = 1 + d1['bar'] = None + + d2['bar'] = None + d2['foo'] = 1 + + assert list(d1.keys()) != list(d2.keys()) + assert checksum_basic_mapping(d1) == checksum_basic_mapping(d2) + + d2['foo'] += 1 + + assert checksum_basic_mapping(d1) != checksum_basic_mapping(d2) diff --git a/tests/t2_block_cache.py b/tests/t2_block_cache.py new file mode 100755 index 0000000..bede753 --- /dev/null +++ b/tests/t2_block_cache.py @@ -0,0 +1,584 @@ +#!/usr/bin/env python3 +''' +t2_block_cache.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +from contextlib import contextmanager +from s3ql.backends import local +from s3ql.backends.common import AbstractBackend +from s3ql.backends.pool import BackendPool +from s3ql.block_cache import BlockCache, QuitSentinel +from s3ql.mkfs import init_tables +from s3ql.metadata import create_tables +from s3ql.database import Connection +from s3ql.common import AsyncFn, time_ns +from common import safe_sleep +from pytest_checklogs import assert_logs +import llfuse +import errno +import os +import logging +import shutil +import stat +import tempfile +import threading +import unittest +import queue +import pytest + +# A dummy removal queue to monkeypatch around the need for removal and upload +# threads +class DummyQueue: + def __init__(self, cache): + self.obj = None + self.cache = cache + + def get_nowait(self): + return self.get(block=False) + + def put(self, obj, timeout=None): + self.obj = obj + self.cache._removal_loop() + return True + + def get(self, block=True): + if self.obj is None: + raise queue.Empty() + elif self.obj is QuitSentinel: + self.obj = None + return QuitSentinel + else: + tmp = self.obj + self.obj = QuitSentinel + return tmp + + def qsize(self): + return 0 + +class cache_tests(unittest.TestCase): + + def setUp(self): + + self.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') + self.backend_pool = BackendPool(lambda: local.Backend('local://' + self.backend_dir, + None, None)) + + self.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-') + self.max_obj_size = 1024 + + # Destructors are not guaranteed to run, and we can't unlink + # the file immediately because apsw refers to it by name. + # Therefore, we unlink the file manually in tearDown() + self.dbfile = tempfile.NamedTemporaryFile(delete=False) + self.db = Connection(self.dbfile.name) + create_tables(self.db) + init_tables(self.db) + + # Create an inode we can work with + self.inode = 42 + now_ns = time_ns() + self.db.execute("INSERT INTO inodes (id,mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?,?)", + (self.inode, stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR + | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, + os.getuid(), os.getgid(), now_ns, now_ns, now_ns, 1, 32)) + + cache = BlockCache(self.backend_pool, self.db, self.cachedir + "/cache", + self.max_obj_size * 100) + self.cache = cache + + # Monkeypatch around the need for removal and upload threads + cache.to_remove = DummyQueue(cache) + + class DummyDistributor: + def put(self, arg, timeout=None): + cache._do_upload(*arg) + return True + cache.to_upload = DummyDistributor() + + # Tested methods assume that they are called from + # file system request handler + llfuse.lock.acquire() + + def tearDown(self): + llfuse.lock.release() + self.cache.backend_pool = self.backend_pool + self.cache.destroy() + shutil.rmtree(self.cachedir) + shutil.rmtree(self.backend_dir) + self.dbfile.close() + os.unlink(self.dbfile.name) + + def test_thread_hang(self): + # Make sure that we don't deadlock if uploads threads or removal + # threads have died and we try to expire or terminate + + # Monkeypatch to avoid error messages about uncaught exceptions + # in other threads + upload_exc = False + removal_exc = False + def _upload_loop(*a, fn=self.cache._upload_loop): + try: + return fn(*a) + except NotADirectoryError: + nonlocal upload_exc + upload_exc = True + def _removal_loop(*a, fn=self.cache._removal_loop): + try: + return fn(*a) + except NotADirectoryError: + nonlocal removal_exc + removal_exc = True + self.cache._upload_loop = _upload_loop + self.cache._removal_loop = _removal_loop + + # Start threads + self.cache.init(threads=3) + + # Create first object (we'll try to remove that) + with self.cache.get(self.inode, 0) as fh: + fh.write(b'bar wurfz!') + self.cache.commit() + self.cache.wait() + + # Make sure that upload and removal will fail + os.rename(self.backend_dir, self.backend_dir + '-tmp') + open(self.backend_dir, 'w').close() + + # Create second object (we'll try to upload that) + with self.cache.get(self.inode, 1) as fh: + fh.write(b'bar wurfz number two!') + + # Schedule a removal + self.cache.remove(self.inode, 0) + + try: + # Try to clean-up (implicitly calls expire) + with llfuse.lock_released, \ + assert_logs('Unable to flush cache, no upload threads left alive', + level=logging.ERROR, count=1): + with pytest.raises(OSError) as exc_info: + self.cache.destroy() + assert exc_info.value.errno == errno.ENOTEMPTY + assert upload_exc + assert removal_exc + finally: + # Fix backend dir + os.unlink(self.backend_dir) + os.rename(self.backend_dir + '-tmp', self.backend_dir) + + # Remove objects from cache and make final destroy + # call into no-op. + self.cache.remove(self.inode, 1) + self.cache.destroy = lambda: None + + + @staticmethod + def random_data(len_): + with open("/dev/urandom", "rb") as fh: + return fh.read(len_) + + def test_get(self): + inode = self.inode + blockno = 11 + data = self.random_data(int(0.5 * self.max_obj_size)) + + # Case 1: Object does not exist yet + with self.cache.get(inode, blockno) as fh: + fh.seek(0) + fh.write(data) + + # Case 2: Object is in cache + with self.cache.get(inode, blockno) as fh: + fh.seek(0) + self.assertEqual(data, fh.read(len(data))) + + # Case 3: Object needs to be downloaded + self.cache.clear() + with self.cache.get(inode, blockno) as fh: + fh.seek(0) + self.assertEqual(data, fh.read(len(data))) + + def test_expire(self): + inode = self.inode + + # Define the 4 most recently accessed ones + most_recent = [7, 11, 10, 8] + for i in most_recent: + safe_sleep(0.2) + with self.cache.get(inode, i) as fh: + fh.write(('%d' % i).encode()) + + # And some others + for i in range(20): + if i in most_recent: + continue + with self.cache.get(inode, i) as fh: + fh.write(('%d' % i).encode()) + + # Flush the 2 most recently accessed ones + commit(self.cache, inode, most_recent[-2]) + commit(self.cache, inode, most_recent[-3]) + + # We want to expire 4 entries, 2 of which are already flushed + self.cache.cache.max_entries = 16 + self.cache.backend_pool = TestBackendPool(self.backend_pool, no_write=2) + self.cache.expire() + self.cache.backend_pool.verify() + self.assertEqual(len(self.cache.cache), 16) + + for i in range(20): + if i in most_recent: + self.assertTrue((inode, i) not in self.cache.cache) + else: + self.assertTrue((inode, i) in self.cache.cache) + + def test_upload(self): + inode = self.inode + datalen = int(0.1 * self.cache.cache.max_size) + blockno1 = 21 + blockno2 = 25 + blockno3 = 7 + + data1 = self.random_data(datalen) + data2 = self.random_data(datalen) + data3 = self.random_data(datalen) + + # Case 1: create new object + self.cache.backend_pool = TestBackendPool(self.backend_pool, no_write=1) + with self.cache.get(inode, blockno1) as fh: + fh.seek(0) + fh.write(data1) + el1 = fh + self.cache.upload(el1) + self.cache.backend_pool.verify() + + # Case 2: Link new object + self.cache.backend_pool = TestBackendPool(self.backend_pool) + with self.cache.get(inode, blockno2) as fh: + fh.seek(0) + fh.write(data1) + el2 = fh + self.cache.upload(el2) + self.cache.backend_pool.verify() + + # Case 3: Upload old object, still has references + self.cache.backend_pool = TestBackendPool(self.backend_pool, no_write=1) + with self.cache.get(inode, blockno1) as fh: + fh.seek(0) + fh.write(data2) + self.cache.upload(el1) + self.cache.backend_pool.verify() + + # Case 4: Upload old object, no references left + self.cache.backend_pool = TestBackendPool(self.backend_pool, no_del=1, no_write=1) + with self.cache.get(inode, blockno2) as fh: + fh.seek(0) + fh.write(data3) + self.cache.upload(el2) + self.cache.backend_pool.verify() + + # Case 5: Link old object, no references left + self.cache.backend_pool = TestBackendPool(self.backend_pool, no_del=1) + with self.cache.get(inode, blockno2) as fh: + fh.seek(0) + fh.write(data2) + self.cache.upload(el2) + self.cache.backend_pool.verify() + + # Case 6: Link old object, still has references + # (Need to create another object first) + self.cache.backend_pool = TestBackendPool(self.backend_pool, no_write=1) + with self.cache.get(inode, blockno3) as fh: + fh.seek(0) + fh.write(data1) + el3 = fh + self.cache.upload(el3) + self.cache.backend_pool.verify() + + self.cache.backend_pool = TestBackendPool(self.backend_pool) + with self.cache.get(inode, blockno1) as fh: + fh.seek(0) + fh.write(data1) + self.cache.upload(el1) + self.cache.clear() + self.cache.backend_pool.verify() + + def test_remove_referenced(self): + inode = self.inode + datalen = int(0.1 * self.cache.cache.max_size) + blockno1 = 21 + blockno2 = 24 + data = self.random_data(datalen) + + self.cache.backend_pool = TestBackendPool(self.backend_pool, no_write=1) + with self.cache.get(inode, blockno1) as fh: + fh.seek(0) + fh.write(data) + with self.cache.get(inode, blockno2) as fh: + fh.seek(0) + fh.write(data) + self.cache.clear() + self.cache.backend_pool.verify() + + self.cache.backend_pool = TestBackendPool(self.backend_pool) + self.cache.remove(inode, blockno1) + self.cache.backend_pool.verify() + + def test_remove_cache(self): + inode = self.inode + data1 = self.random_data(int(0.4 * self.max_obj_size)) + + # Case 1: Elements only in cache + with self.cache.get(inode, 1) as fh: + fh.seek(0) + fh.write(data1) + self.cache.remove(inode, 1) + with self.cache.get(inode, 1) as fh: + fh.seek(0) + self.assertEqual(fh.read(42), b'') + + def test_upload_race(self): + inode = self.inode + blockno = 1 + data1 = self.random_data(int(0.4 * self.max_obj_size)) + with self.cache.get(inode, blockno) as fh: + fh.seek(0) + fh.write(data1) + + # Remove it + self.cache.remove(inode, blockno) + + # Try to upload it, may happen if CommitThread is interrupted + self.cache.upload(fh) + + def test_expire_race(self): + # Create element + inode = self.inode + blockno = 1 + data1 = self.random_data(int(0.4 * self.max_obj_size)) + with self.cache.get(inode, blockno) as fh: + fh.seek(0) + fh.write(data1) + self.cache.upload(fh) + + # Make sure entry will be expired + self.cache.cache.max_entries = 0 + def e_w_l(): + with llfuse.lock: + self.cache.expire() + + # Lock it + self.cache._lock_entry(inode, blockno, release_global=True) + + try: + # Start expiration, will block on lock + t1 = AsyncFn(e_w_l) + t1.start() + + # Start second expiration, will block + t2 = AsyncFn(e_w_l) + t2.start() + + # Release lock + with llfuse.lock_released: + safe_sleep(0.1) + self.cache._unlock_entry(inode, blockno) + t1.join_and_raise() + t2.join_and_raise() + + assert len(self.cache.cache) == 0 + finally: + self.cache._unlock_entry(inode, blockno, release_global=True, + noerror=True) + + + def test_parallel_expire(self): + # Create elements + inode = self.inode + for i in range(5): + data1 = self.random_data(int(0.4 * self.max_obj_size)) + with self.cache.get(inode, i) as fh: + fh.write(data1) + + # We want to expire just one element, but have + # several threads running expire() simultaneously + self.cache.cache.max_entries = 4 + def e_w_l(): + with llfuse.lock: + self.cache.expire() + + # Lock first element so that we have time to start threads + self.cache._lock_entry(inode, 0, release_global=True) + + try: + # Start expiration, will block on lock + t1 = AsyncFn(e_w_l) + t1.start() + + # Start second expiration, will block + t2 = AsyncFn(e_w_l) + t2.start() + + # Release lock + with llfuse.lock_released: + safe_sleep(0.1) + self.cache._unlock_entry(inode, 0) + t1.join_and_raise() + t2.join_and_raise() + + assert len(self.cache.cache) == 4 + finally: + self.cache._unlock_entry(inode, 0, release_global=True, + noerror=True) + + + def test_remove_cache_db(self): + inode = self.inode + data1 = self.random_data(int(0.4 * self.max_obj_size)) + + # Case 2: Element in cache and db + with self.cache.get(inode, 1) as fh: + fh.seek(0) + fh.write(data1) + self.cache.backend_pool = TestBackendPool(self.backend_pool, no_write=1) + commit(self.cache, inode) + self.cache.backend_pool.verify() + self.cache.backend_pool = TestBackendPool(self.backend_pool, no_del=1) + self.cache.remove(inode, 1) + self.cache.backend_pool.verify() + + with self.cache.get(inode, 1) as fh: + fh.seek(0) + self.assertEqual(fh.read(42), b'') + + + def test_remove_db(self): + inode = self.inode + data1 = self.random_data(int(0.4 * self.max_obj_size)) + + # Case 3: Element only in DB + with self.cache.get(inode, 1) as fh: + fh.seek(0) + fh.write(data1) + self.cache.backend_pool = TestBackendPool(self.backend_pool, no_write=1) + self.cache.clear() + self.cache.backend_pool.verify() + self.cache.backend_pool = TestBackendPool(self.backend_pool, no_del=1) + self.cache.remove(inode, 1) + self.cache.backend_pool.verify() + with self.cache.get(inode, 1) as fh: + fh.seek(0) + self.assertEqual(fh.read(42), b'') + + +class TestBackendPool(AbstractBackend): + has_native_rename = False + + def __init__(self, backend_pool, no_read=0, no_write=0, no_del=0): + super().__init__() + self.no_read = no_read + self.no_write = no_write + self.no_del = no_del + self.backend_pool = backend_pool + self.backend = backend_pool.pop_conn() + self.lock = threading.Lock() + + def __del__(self): + self.backend_pool.push_conn(self.backend) + + def verify(self): + if self.no_read != 0: + raise RuntimeError('Got too few open_read calls') + if self.no_write != 0: + raise RuntimeError('Got too few open_write calls') + if self.no_del != 0: + raise RuntimeError('Got too few delete calls') + + @contextmanager + def __call__(self): + '''Provide connection from pool (context manager)''' + + with self.lock: + yield self + + def lookup(self, key): + return self.backend.lookup(key) + + def open_read(self, key): + self.no_read -= 1 + if self.no_read < 0: + raise RuntimeError('Got too many open_read calls') + + return self.backend.open_read(key) + + def open_write(self, key, metadata=None, is_compressed=False): + self.no_write -= 1 + if self.no_write < 0: + raise RuntimeError('Got too many open_write calls') + + return self.backend.open_write(key, metadata, is_compressed) + + def is_temp_failure(self, exc): + return self.backend.is_temp_failure(exc) + + def clear(self): + return self.backend.clear() + + def contains(self, key): + return self.backend.contains(key) + + def delete(self, key, force=False): + self.no_del -= 1 + if self.no_del < 0: + raise RuntimeError('Got too many delete calls') + + return self.backend.delete(key, force) + + def list(self, prefix=''): + '''List keys in backend + + Returns an iterator over all keys in the backend. + ''' + return self.backend.list(prefix) + + def copy(self, src, dest, metadata=None): + return self.backend.copy(src, dest, metadata) + + def rename(self, src, dest, metadata=None): + return self.backend.rename(src, dest, metadata) + + def update_meta(self, key, metadata): + return self.backend.update_meta(key, metadata) + + def get_size(self, key): + '''Return size of object stored under *key*''' + + return self.backend.get_size(key) + +def commit(cache, inode, block=None): + """Upload data for `inode` + + This is only for testing purposes, since the method blocks until all current + uploads have been completed. + """ + + for el in cache.cache.values(): + if el.inode != inode: + continue + if not el.dirty: + continue + + if block is not None and el.blockno != block: + continue + + cache.upload(el) diff --git a/tests/t3_fs_api.py b/tests/t3_fs_api.py new file mode 100755 index 0000000..1218d82 --- /dev/null +++ b/tests/t3_fs_api.py @@ -0,0 +1,1003 @@ +#!/usr/bin/env python3 +''' +t3_fs_api.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +from llfuse import FUSEError +from random import randint +from s3ql import fs +from s3ql.backends import local +from s3ql.backends.pool import BackendPool +from s3ql.backends.comprenc import ComprencBackend +from s3ql.block_cache import BlockCache +from s3ql import ROOT_INODE +from s3ql.mkfs import init_tables +from s3ql.metadata import create_tables +from s3ql.database import Connection +from s3ql.fsck import Fsck +from s3ql.inode_cache import InodeCache +from t2_block_cache import DummyQueue +from common import CLOCK_GRANULARITY, safe_sleep +from pytest_checklogs import assert_logs +import errno +import llfuse +import os +import shutil +import stat +import logging +import tempfile +import unittest + +# We need to access to protected members +#pylint: disable=W0212 + +# The classes provided by llfuse have read-only attributes, +# so we duck-type our own. +class Ctx: + def __init__(self): + self.uid = randint(0, 2 ** 32) + self.gid = randint(0, 2 ** 32) + self.pid = randint(0, 2 ** 32) + self.umask = 0 + +class SetattrFields: + def __init__(self, **kw): + self.update_atime = False + self.update_mtime = False + self.update_mode = False + self.update_uid = False + self.update_gid = False + self.update_size = False + self.__dict__.update(kw) + +some_ctx = Ctx() + +class fs_api_tests(unittest.TestCase): + + def setUp(self): + self.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') + plain_backend = local.Backend('local://' + self.backend_dir, None, None) + self.backend_pool = BackendPool(lambda: ComprencBackend(b'schwubl', ('zlib', 6), + plain_backend)) + self.backend = self.backend_pool.pop_conn() + self.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-') + self.max_obj_size = 1024 + + # Destructors are not guaranteed to run, and we can't unlink + # the file immediately because apsw refers to it by name. + # Therefore, we unlink the file manually in tearDown() + self.dbfile = tempfile.NamedTemporaryFile(delete=False) + + self.db = Connection(self.dbfile.name) + create_tables(self.db) + init_tables(self.db) + + # Tested methods assume that they are called from + # file system request handler + llfuse.lock.acquire() + + cache = BlockCache(self.backend_pool, self.db, self.cachedir + "/cache", + self.max_obj_size * 5) + self.block_cache = cache + self.server = fs.Operations(cache, self.db, self.max_obj_size, + InodeCache(self.db, 0)) + self.server.init() + + # Monkeypatch around the need for removal and upload threads + cache.to_remove = DummyQueue(cache) + + class DummyDistributor: + def put(self, arg, timeout=None): + cache._do_upload(*arg) + return True + cache.to_upload = DummyDistributor() + + # Keep track of unused filenames + self.name_cnt = 0 + + def tearDown(self): + self.server.inodes.destroy() + llfuse.lock.release() + self.block_cache.destroy() + shutil.rmtree(self.cachedir) + shutil.rmtree(self.backend_dir) + os.unlink(self.dbfile.name) + self.dbfile.close() + + @staticmethod + def random_data(len_): + with open("/dev/urandom", "rb") as fd: + return fd.read(len_) + + def fsck(self): + self.block_cache.clear() + self.server.inodes.flush() + fsck = Fsck(self.cachedir + '/cache', self.backend, + { 'max_obj_size': self.max_obj_size }, self.db) + fsck.check() + self.assertFalse(fsck.found_errors) + + def newname(self): + self.name_cnt += 1 + return ("s3ql_%d" % self.name_cnt).encode() + + def test_getattr_root(self): + self.assertTrue(stat.S_ISDIR(self.server.getattr(ROOT_INODE, some_ctx).st_mode)) + self.fsck() + + def test_create(self): + ctx = Ctx() + mode = self.dir_mode() + name = self.newname() + + inode_p_old = self.server.getattr(ROOT_INODE, some_ctx) + safe_sleep(CLOCK_GRANULARITY) + self.server._create(ROOT_INODE, name, mode, ctx) + + id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON name_id = names.id ' + 'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)) + + inode = self.server.getattr(id_, some_ctx) + + self.assertEqual(inode.st_mode, mode) + self.assertEqual(inode.st_uid, ctx.uid) + self.assertEqual(inode.st_gid, ctx.gid) + self.assertEqual(inode.st_nlink, 1) + self.assertEqual(inode.st_size, 0) + + inode_p_new = self.server.getattr(ROOT_INODE, some_ctx) + + self.assertGreater(inode_p_new.st_mtime_ns, inode_p_old.st_mtime_ns) + self.assertGreater(inode_p_new.st_ctime_ns, inode_p_old.st_ctime_ns) + + self.server.forget([(id_, 1)]) + self.fsck() + + def test_extstat(self): + # Test with zero contents + self.server.extstat() + + # Test with empty file + (fh, inode) = self.server.create(ROOT_INODE, self.newname(), + self.file_mode(), os.O_RDWR, some_ctx) + self.server.release(fh) + self.server.extstat() + + # Test with data in file + fh = self.server.open(inode.st_ino, os.O_RDWR, some_ctx) + self.server.write(fh, 0, b'foobar') + self.server.release(fh) + + self.server.extstat() + self.server.forget([(inode.st_ino, 1)]) + self.fsck() + + @staticmethod + def dir_mode(): + return (randint(0, 0o7777) & ~stat.S_IFDIR) | stat.S_IFDIR + + @staticmethod + def file_mode(): + return (randint(0, 0o7777) & ~stat.S_IFREG) | stat.S_IFREG + + def test_getxattr(self): + (fh, inode) = self.server.create(ROOT_INODE, self.newname(), + self.file_mode(), os.O_RDWR, some_ctx) + self.server.release(fh) + + self.assertRaises(FUSEError, self.server.getxattr, inode.st_ino, + b'nonexistant-attr', some_ctx) + + self.server.setxattr(inode.st_ino, b'my-attr', b'strabumm!', some_ctx) + self.assertEqual(self.server.getxattr(inode.st_ino, b'my-attr', some_ctx), + b'strabumm!') + + self.server.forget([(inode.st_ino, 1)]) + self.fsck() + + def test_link(self): + name = self.newname() + + inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(), + self.dir_mode(), some_ctx) + inode_p_new_before = self.server.getattr(inode_p_new.st_ino, some_ctx) + + (fh, inode) = self.server.create(ROOT_INODE, self.newname(), + self.file_mode(), os.O_RDWR, some_ctx) + self.server.release(fh) + safe_sleep(CLOCK_GRANULARITY) + + inode_before = self.server.getattr(inode.st_ino, some_ctx) + self.server.link(inode.st_ino, inode_p_new.st_ino, name, some_ctx) + + inode_after = self.server.lookup(inode_p_new.st_ino, name, some_ctx) + inode_p_new_after = self.server.getattr(inode_p_new.st_ino, some_ctx) + + id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (name, inode_p_new.st_ino)) + + self.assertEqual(inode_before.st_ino, id_) + self.assertEqual(inode_after.st_nlink, 2) + self.assertGreater(inode_after.st_ctime_ns, inode_before.st_ctime_ns) + self.assertLess(inode_p_new_before.st_mtime_ns, inode_p_new_after.st_mtime_ns) + self.assertLess(inode_p_new_before.st_ctime_ns, inode_p_new_after.st_ctime_ns) + self.server.forget([(inode.st_ino, 1), (inode_p_new.st_ino, 1), (inode_after.st_ino, 1)]) + self.fsck() + + def test_listxattr(self): + (fh, inode) = self.server.create(ROOT_INODE, self.newname(), + self.file_mode(), os.O_RDWR, some_ctx) + self.server.release(fh) + + self.assertListEqual([], self.server.listxattr(inode.st_ino, some_ctx)) + + self.server.setxattr(inode.st_ino, b'key1', b'blub', some_ctx) + self.assertListEqual([b'key1'], self.server.listxattr(inode.st_ino, some_ctx)) + + self.server.setxattr(inode.st_ino, b'key2', b'blub', some_ctx) + self.assertListEqual(sorted([b'key1', b'key2']), + sorted(self.server.listxattr(inode.st_ino, some_ctx))) + self.server.forget([(inode.st_ino, 1)]) + self.fsck() + + def test_read(self): + + len_ = self.max_obj_size + data = self.random_data(len_) + off = self.max_obj_size // 2 + (fh, inode) = self.server.create(ROOT_INODE, self.newname(), + self.file_mode(), os.O_RDWR, some_ctx) + + self.server.write(fh, off, data) + inode_before = self.server.getattr(inode.st_ino, some_ctx) + safe_sleep(CLOCK_GRANULARITY) + self.assertTrue(self.server.read(fh, off, len_) == data) + inode_after = self.server.getattr(inode.st_ino, some_ctx) + self.assertGreater(inode_after.st_atime_ns, inode_before.st_atime_ns) + self.assertTrue(self.server.read(fh, 0, len_) == b"\0" * off + data[:off]) + self.assertTrue(self.server.read(fh, self.max_obj_size, len_) == data[off:]) + self.server.release(fh) + self.server.forget([(inode.st_ino, 1)]) + self.fsck() + + def test_readdir(self): + + # Create a few entries + names = [ ('entry_%2d' % i).encode() for i in range(20) ] + for name in names: + (fh, inode) = self.server.create(ROOT_INODE, name, + self.file_mode(), os.O_RDWR, some_ctx) + self.server.release(fh) + self.server.forget([(inode.st_ino, 1)]) + + # Delete some to make sure that we don't have continous rowids + remove_no = [0, 2, 3, 5, 9] + for i in remove_no: + self.server.unlink(ROOT_INODE, names[i], some_ctx) + del names[i] + + # Read all + fh = self.server.opendir(ROOT_INODE, some_ctx) + self.assertListEqual(sorted(names + [b'lost+found']) , + sorted(x[0] for x in self.server.readdir(fh, 0))) + self.server.releasedir(fh) + + # Read in parts + fh = self.server.opendir(ROOT_INODE, some_ctx) + entries = list() + try: + next_ = 0 + while True: + gen = self.server.readdir(fh, next_) + for _ in range(3): + (name, _, next_) = next(gen) + entries.append(name) + + except StopIteration: + pass + + self.assertListEqual(sorted(names + [b'lost+found']) , + sorted(entries)) + self.server.releasedir(fh) + + self.fsck() + + def test_forget(self): + name = self.newname() + + # Test that entries are deleted when they're no longer referenced + (fh, inode) = self.server.create(ROOT_INODE, name, + self.file_mode(), os.O_RDWR, some_ctx) + self.server.write(fh, 0, b'foobar') + self.server.unlink(ROOT_INODE, name, some_ctx) + self.assertFalse(self.db.has_val('SELECT 1 FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE))) + self.assertTrue(self.server.getattr(inode.st_ino, some_ctx).st_ino) + self.server.release(fh) + self.server.forget([(inode.st_ino, 1)]) + + self.assertFalse(self.db.has_val('SELECT 1 FROM inodes WHERE id=?', (inode.st_ino,))) + + self.fsck() + + def test_removexattr(self): + (fh, inode) = self.server.create(ROOT_INODE, self.newname(), + self.file_mode(), os.O_RDWR, some_ctx) + self.server.release(fh) + + self.assertRaises(FUSEError, self.server.removexattr, inode.st_ino, b'some name', + some_ctx) + self.server.setxattr(inode.st_ino, b'key1', b'blub', some_ctx) + self.server.removexattr(inode.st_ino, b'key1', some_ctx) + self.assertListEqual([], self.server.listxattr(inode.st_ino, some_ctx)) + self.server.forget([(inode.st_ino, 1)]) + self.fsck() + + def test_rename(self): + oldname = self.newname() + newname = self.newname() + + inode = self.server.mkdir(ROOT_INODE, oldname, self.dir_mode(), some_ctx) + + inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(), self.dir_mode(), some_ctx) + inode_p_new_before = self.server.getattr(inode_p_new.st_ino, some_ctx) + inode_p_old_before = self.server.getattr(ROOT_INODE, some_ctx) + safe_sleep(CLOCK_GRANULARITY) + + self.server.rename(ROOT_INODE, oldname, inode_p_new.st_ino, newname, some_ctx) + + inode_p_old_after = self.server.getattr(ROOT_INODE, some_ctx) + inode_p_new_after = self.server.getattr(inode_p_new.st_ino, some_ctx) + + self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (oldname, ROOT_INODE))) + id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id == name_id ' + 'WHERE name=? AND parent_inode = ?', (newname, inode_p_new.st_ino)) + self.assertEqual(inode.st_ino, id_) + + assert inode_p_new_before.st_mtime_ns < inode_p_new_after.st_mtime_ns + assert inode_p_new_before.st_ctime_ns < inode_p_new_after.st_ctime_ns + assert inode_p_old_before.st_mtime_ns < inode_p_old_after.st_mtime_ns + assert inode_p_old_before.st_ctime_ns < inode_p_old_after.st_ctime_ns + + self.server.forget([(inode.st_ino, 1), (inode_p_new.st_ino, 1)]) + self.fsck() + + def test_replace_file(self): + oldname = self.newname() + newname = self.newname() + + (fh, inode) = self.server.create(ROOT_INODE, oldname, self.file_mode(), + os.O_RDWR, some_ctx) + self.server.write(fh, 0, b'some data to deal with') + self.server.release(fh) + self.server.setxattr(inode.st_ino, b'test_xattr', b'42*8', some_ctx) + + inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(), self.dir_mode(), some_ctx) + inode_p_new_before = self.server.getattr(inode_p_new.st_ino, some_ctx) + inode_p_old_before = self.server.getattr(ROOT_INODE, some_ctx) + + (fh, inode2) = self.server.create(inode_p_new.st_ino, newname, self.file_mode(), + os.O_RDWR, some_ctx) + self.server.write(fh, 0, b'even more data to deal with') + self.server.release(fh) + self.server.setxattr(inode2.st_ino, b'test_xattr', b'42*8', some_ctx) + self.server.forget([(inode2.st_ino, 1)]) + + safe_sleep(CLOCK_GRANULARITY) + self.server.rename(ROOT_INODE, oldname, inode_p_new.st_ino, newname, some_ctx) + + inode_p_old_after = self.server.getattr(ROOT_INODE, some_ctx) + inode_p_new_after = self.server.getattr(inode_p_new.st_ino, some_ctx) + + self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (oldname, ROOT_INODE))) + id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (newname, inode_p_new.st_ino)) + self.assertEqual(inode.st_ino, id_) + + self.assertLess(inode_p_new_before.st_mtime_ns, inode_p_new_after.st_mtime_ns) + self.assertLess(inode_p_new_before.st_ctime_ns, inode_p_new_after.st_ctime_ns) + self.assertLess(inode_p_old_before.st_mtime_ns, inode_p_old_after.st_mtime_ns) + self.assertLess(inode_p_old_before.st_ctime_ns, inode_p_old_after.st_ctime_ns) + + self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode2.st_ino,))) + self.server.forget([(inode.st_ino, 1), (inode_p_new.st_ino, 1)]) + self.fsck() + + def test_replace_dir(self): + oldname = self.newname() + newname = self.newname() + + inode = self.server.mkdir(ROOT_INODE, oldname, self.dir_mode(), some_ctx) + + inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(), self.dir_mode(), some_ctx) + inode_p_new_before = self.server.getattr(inode_p_new.st_ino, some_ctx) + inode_p_old_before = self.server.getattr(ROOT_INODE, some_ctx) + + inode2 = self.server.mkdir(inode_p_new.st_ino, newname, self.dir_mode(), some_ctx) + self.server.forget([(inode2.st_ino, 1)]) + + safe_sleep(CLOCK_GRANULARITY) + self.server.rename(ROOT_INODE, oldname, inode_p_new.st_ino, newname, some_ctx) + + inode_p_old_after = self.server.getattr(ROOT_INODE, some_ctx) + inode_p_new_after = self.server.getattr(inode_p_new.st_ino, some_ctx) + + self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (oldname, ROOT_INODE))) + id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (newname, inode_p_new.st_ino)) + self.assertEqual(inode.st_ino, id_) + + self.assertLess(inode_p_new_before.st_mtime_ns, inode_p_new_after.st_mtime_ns) + self.assertLess(inode_p_new_before.st_ctime_ns, inode_p_new_after.st_ctime_ns) + self.assertLess(inode_p_old_before.st_mtime_ns, inode_p_old_after.st_mtime_ns) + self.assertLess(inode_p_old_before.st_ctime_ns, inode_p_old_after.st_ctime_ns) + + self.server.forget([(inode.st_ino, 1), (inode_p_new.st_ino, 1)]) + self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode2.st_ino,))) + self.fsck() + + def test_setattr_one(self): + (fh, inode) = self.server.create(ROOT_INODE, self.newname(), self.file_mode(), + os.O_RDWR, some_ctx) + self.server.release(fh) + inode_old = self.server.getattr(inode.st_ino, some_ctx) + + attr = self.server.getattr(inode.st_ino, some_ctx) # this is a fresh instance + attr.st_mode = self.file_mode() + attr.st_uid = randint(0, 2 ** 32) + attr.st_gid = randint(0, 2 ** 32) # should be ignored + attr.st_atime_ns = randint(0, 2 ** 50) + attr.st_mtime_ns = randint(0, 2 ** 50) + + safe_sleep(CLOCK_GRANULARITY) + sf = SetattrFields(update_mode=True, update_uid=True, + update_atime=True, update_mtime=True) + self.server.setattr(inode.st_ino, attr, sf, None, some_ctx) + inode_new = self.server.getattr(inode.st_ino, some_ctx) + + for name in ('st_mode', 'st_uid', 'st_atime_ns', 'st_mtime_ns'): + assert getattr(attr, name) == getattr(inode_new, name) + for name in ('st_gid', 'st_size', 'st_nlink', 'st_rdev', + 'st_blocks', 'st_blksize'): + assert getattr(inode_old, name) == getattr(inode_new, name) + assert inode_old.st_ctime_ns < inode_new.st_ctime_ns + + self.server.forget([(inode.st_ino, 1)]) + self.fsck() + + def test_setattr_two(self): + (fh, inode_old) = self.server.create(ROOT_INODE, self.newname(), self.file_mode(), + os.O_RDWR, some_ctx) + + attr = self.server.getattr(inode_old.st_ino, some_ctx) + attr.st_mode = self.file_mode() + attr.st_uid = randint(0, 2 ** 32) + attr.st_gid = randint(0, 2 ** 32) + attr.st_mtime_ns = randint(0, 2 ** 50) + attr.st_ctime_ns = 5e9 + + safe_sleep(CLOCK_GRANULARITY) + sf = SetattrFields(update_gid=True, update_mtime=True) + + self.server.setattr(inode_old.st_ino, attr, sf, None, some_ctx) + inode_new = self.server.getattr(inode_old.st_ino, some_ctx) + + for name in ('st_gid', 'st_mtime_ns'): + assert getattr(attr, name) == getattr(inode_new, name) + for name in ('st_uid', 'st_size', 'st_nlink', 'st_rdev', + 'st_blocks', 'st_blksize', 'st_mode', 'st_atime_ns'): + assert getattr(inode_old, name) == getattr(inode_new, name) + assert inode_old.st_ctime_ns < inode_new.st_ctime_ns + + self.server.release(fh) + self.server.forget([(inode_old.st_ino, 1)]) + self.fsck() + + def test_truncate(self): + len_ = int(2.7 * self.max_obj_size) + data = self.random_data(len_) + + (fh, inode) = self.server.create(ROOT_INODE, self.newname(), self.file_mode(), + os.O_RDWR, some_ctx) + self.server.write(fh, 0, data) + + attr = self.server.getattr(inode.st_ino, some_ctx) + attr.st_size = len_ // 2 + sf = SetattrFields(update_size=True) + self.server.setattr(inode.st_ino, attr, sf, None, some_ctx) + self.assertTrue(self.server.read(fh, 0, len_) == data[:len_ // 2]) + attr.st_size = len_ + self.server.setattr(inode.st_ino, attr, sf, None, some_ctx) + self.assertTrue(self.server.read(fh, 0, len_) + == data[:len_ // 2] + b'\0' * (len_ // 2)) + self.server.release(fh) + self.server.forget([(inode.st_ino, 1)]) + self.fsck() + + def test_truncate_0(self): + len1 = 158 + len2 = 133 + + (fh, inode) = self.server.create(ROOT_INODE, self.newname(), + self.file_mode(), os.O_RDWR, some_ctx) + self.server.write(fh, 0, self.random_data(len1)) + self.server.release(fh) + self.server.inodes.flush() + + attr = self.server.getattr(inode.st_ino, some_ctx) + fh = self.server.open(inode.st_ino, os.O_RDWR, some_ctx) + attr.st_size = 0 + self.server.setattr(inode.st_ino, attr, SetattrFields(update_size=True), + fh, some_ctx) + self.server.write(fh, 0, self.random_data(len2)) + self.server.release(fh) + self.server.forget([(inode.st_ino, 1)]) + self.fsck() + + def test_setxattr(self): + (fh, inode) = self.server.create(ROOT_INODE, self.newname(), + self.file_mode(), os.O_RDWR, some_ctx) + self.server.release(fh) + + self.server.setxattr(inode.st_ino, b'my-attr', b'strabumm!', some_ctx) + self.assertEqual(self.server.getxattr(inode.st_ino, b'my-attr', some_ctx), + b'strabumm!') + self.server.forget([(inode.st_ino, 1)]) + self.fsck() + + def test_names(self): + name1 = self.newname() + name2 = self.newname() + + (fh, inode) = self.server.create(ROOT_INODE, name1, self.file_mode(), + os.O_RDWR, some_ctx) + self.server.release(fh) + self.server.forget([(inode.st_ino, 1)]) + + (fh, inode) = self.server.create(ROOT_INODE, name2, self.file_mode(), + os.O_RDWR, some_ctx) + self.server.release(fh) + + self.server.setxattr(inode.st_ino, name1, b'strabumm!', some_ctx) + self.fsck() + + self.server.removexattr(inode.st_ino, name1, some_ctx) + self.fsck() + + self.server.setxattr(inode.st_ino, name1, b'strabumm karacho!!', + some_ctx) + self.server.unlink(ROOT_INODE, name1, some_ctx) + self.server.forget([(inode.st_ino, 1)]) + self.fsck() + + + def test_statfs(self): + # Test with zero contents + self.server.statfs(some_ctx) + + # Test with empty file + (fh, inode) = self.server.create(ROOT_INODE, self.newname(), + self.file_mode(), os.O_RDWR, some_ctx) + self.server.release(fh) + self.server.statfs(some_ctx) + + # Test with data in file + fh = self.server.open(inode.st_ino, os.O_RDWR, some_ctx) + self.server.write(fh, 0, b'foobar') + self.server.release(fh) + self.server.forget([(inode.st_ino, 1)]) + self.server.statfs(some_ctx) + + def test_symlink(self): + target = self.newname() + name = self.newname() + + inode_p_before = self.server.getattr(ROOT_INODE, some_ctx) + safe_sleep(CLOCK_GRANULARITY) + inode = self.server.symlink(ROOT_INODE, name, target, some_ctx) + inode_p_after = self.server.getattr(ROOT_INODE, some_ctx) + + self.assertEqual(target, self.server.readlink(inode.st_ino, some_ctx)) + + id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)) + + self.assertEqual(inode.st_ino, id_) + self.assertLess(inode_p_before.st_mtime_ns, inode_p_after.st_mtime_ns) + self.assertLess(inode_p_before.st_ctime_ns, inode_p_after.st_ctime_ns) + + self.server.forget([(inode.st_ino, 1)]) + self.fsck() + + def test_unlink(self): + name = self.newname() + + (fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(), os.O_RDWR, some_ctx) + self.server.write(fh, 0, b'some data to deal with') + self.server.release(fh) + + # Add extended attributes + self.server.setxattr(inode.st_ino, b'test_xattr', b'42*8', some_ctx) + self.server.forget([(inode.st_ino, 1)]) + + inode_p_before = self.server.getattr(ROOT_INODE, some_ctx) + safe_sleep(CLOCK_GRANULARITY) + self.server.unlink(ROOT_INODE, name, some_ctx) + inode_p_after = self.server.getattr(ROOT_INODE, some_ctx) + + self.assertLess(inode_p_before.st_mtime_ns, inode_p_after.st_mtime_ns) + self.assertLess(inode_p_before.st_ctime_ns, inode_p_after.st_ctime_ns) + + self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE))) + self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode.st_ino,))) + + self.fsck() + + def test_rmdir(self): + name = self.newname() + inode = self.server.mkdir(ROOT_INODE, name, self.dir_mode(), some_ctx) + self.server.forget([(inode.st_ino, 1)]) + inode_p_before = self.server.getattr(ROOT_INODE, some_ctx) + safe_sleep(CLOCK_GRANULARITY) + self.server.rmdir(ROOT_INODE, name, some_ctx) + inode_p_after = self.server.getattr(ROOT_INODE, some_ctx) + + self.assertLess(inode_p_before.st_mtime_ns, inode_p_after.st_mtime_ns) + self.assertLess(inode_p_before.st_ctime_ns, inode_p_after.st_ctime_ns) + self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE))) + self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode.st_ino,))) + + self.fsck() + + def test_relink(self): + name = self.newname() + name2 = self.newname() + data = b'some data to deal with' + + (fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(), os.O_RDWR, some_ctx) + self.server.write(fh, 0, data) + self.server.release(fh) + self.server.unlink(ROOT_INODE, name, some_ctx) + self.server.inodes.flush() + self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE))) + self.assertTrue(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode.st_ino,))) + self.server.link(inode.st_ino, ROOT_INODE, name2, some_ctx) + self.server.forget([(inode.st_ino, 2)]) + + inode = self.server.lookup(ROOT_INODE, name2, some_ctx) + fh = self.server.open(inode.st_ino, os.O_RDONLY, some_ctx) + self.assertTrue(self.server.read(fh, 0, len(data)) == data) + self.server.release(fh) + self.server.forget([(inode.st_ino, 1)]) + self.fsck() + + def test_write(self): + len_ = self.max_obj_size + data = self.random_data(len_) + off = self.max_obj_size // 2 + (fh, inode) = self.server.create(ROOT_INODE, self.newname(), + self.file_mode(), os.O_RDWR, some_ctx) + inode_before = self.server.getattr(inode.st_ino, some_ctx) + safe_sleep(CLOCK_GRANULARITY) + self.server.write(fh, off, data) + inode_after = self.server.getattr(inode.st_ino, some_ctx) + + self.assertGreater(inode_after.st_mtime_ns, inode_before.st_mtime_ns) + self.assertGreater(inode_after.st_ctime_ns, inode_before.st_ctime_ns) + self.assertEqual(inode_after.st_size, off + len_) + + self.server.write(fh, 0, data) + inode_after = self.server.getattr(inode.st_ino, some_ctx) + self.assertEqual(inode_after.st_size, off + len_) + + self.server.release(fh) + self.server.forget([(inode.st_ino, 1)]) + self.fsck() + + def test_failsafe(self): + len_ = self.max_obj_size + data = self.random_data(len_) + (fh, inode) = self.server.create(ROOT_INODE, self.newname(), + self.file_mode(), os.O_RDWR, some_ctx) + self.server.write(fh, 0, data) + self.server.cache.clear() + self.assertTrue(self.server.failsafe is False) + + datafile = os.path.join(self.backend_dir, 's3ql_data_', 's3ql_data_1') + shutil.copy(datafile, datafile + '.bak') + + # Modify contents + with open(datafile, 'rb+') as rfh: + rfh.seek(560) + rfh.write(b'blrub!') + with self.assertRaises(FUSEError) as cm: + with assert_logs('^Backend returned malformed data for', + count=1, level=logging.ERROR): + self.server.read(fh, 0, len_) + self.assertEqual(cm.exception.errno, errno.EIO) + self.assertTrue(self.server.failsafe) + + # Restore contents, but should be marked as damaged now + os.rename(datafile + '.bak', datafile) + with self.assertRaises(FUSEError) as cm: + self.server.read(fh, 0, len_) + self.assertEqual(cm.exception.errno, errno.EIO) + + # Release and re-open, now we should be able to access again + self.server.release(fh) + self.server.forget([(inode.st_ino, 1)]) + + # ..but not write access since we are in failsafe mode + with self.assertRaises(FUSEError) as cm: + self.server.open(inode.st_ino, os.O_RDWR, some_ctx) + self.assertEqual(cm.exception.errno, errno.EPERM) + + # ..ready only is fine. + fh = self.server.open(inode.st_ino, os.O_RDONLY, some_ctx) + self.server.read(fh, 0, len_) + + # Remove completely, should give error after cache flush + os.unlink(datafile) + self.server.read(fh, 3, len_//2) + self.server.cache.clear() + with self.assertRaises(FUSEError) as cm: + with assert_logs('^Backend lost block', + count=1, level=logging.ERROR): + self.server.read(fh, 5, len_//2) + self.assertEqual(cm.exception.errno, errno.EIO) + + + # Don't call fsck, we're missing a block + + def test_create_open(self): + name = self.newname() + # Create a new file + (fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(), + os.O_RDWR, some_ctx) + self.server.release(fh) + self.server.forget([(inode, 1)]) + + # Open it atomically + (fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(), + os.O_RDWR, some_ctx) + self.server.release(fh) + self.server.forget([(inode, 1)]) + + self.fsck() + + def test_edit(self): + len_ = self.max_obj_size + data = self.random_data(len_) + (fh, inode) = self.server.create(ROOT_INODE, self.newname(), + self.file_mode(), os.O_RDWR, some_ctx) + self.server.write(fh, 0, data) + self.server.release(fh) + + self.block_cache.clear() + + fh = self.server.open(inode.st_ino, os.O_RDWR, some_ctx) + attr = self.server.getattr(inode.st_ino, some_ctx) + attr.st_size = 0 + self.server.setattr(inode.st_ino, attr, SetattrFields(update_size=True), + fh, some_ctx) + self.server.write(fh, 0, data[50:]) + self.server.release(fh) + self.server.forget([(inode.st_ino, 1)]) + self.fsck() + + def test_copy_tree(self): + ext_attr_name = b'system.foo.brazl' + ext_attr_val = b'schulla dku woumm bramp' + + src_inode = self.server.mkdir(ROOT_INODE, b'source', self.dir_mode(), some_ctx) + dst_inode = self.server.mkdir(ROOT_INODE, b'dest', self.dir_mode(), some_ctx) + + # Create file + (fh, f1_inode) = self.server.create(src_inode.st_ino, b'file1', + self.file_mode(), os.O_RDWR, some_ctx) + self.server.write(fh, 0, b'file1 contents') + self.server.release(fh) + self.server.setxattr(f1_inode.st_ino, ext_attr_name, ext_attr_val, some_ctx) + + # Create hardlink + (fh, f2_inode) = self.server.create(src_inode.st_ino, b'file2', + self.file_mode(), os.O_RDWR, some_ctx) + self.server.write(fh, 0, b'file2 contents') + self.server.release(fh) + f2_inode = self.server.link(f2_inode.st_ino, src_inode.st_ino, b'file2_hardlink', + some_ctx) + + # Create subdirectory + d1_inode = self.server.mkdir(src_inode.st_ino, b'dir1', self.dir_mode(), some_ctx) + d2_inode = self.server.mkdir(d1_inode.st_ino, b'dir2', self.dir_mode(), some_ctx) + + # ..with a 3rd hardlink + f2_inode = self.server.link(f2_inode.st_ino, d1_inode.st_ino, b'file2_hardlink', + some_ctx) + + # Replicate + self.server.copy_tree(src_inode.st_ino, dst_inode.st_ino) + + # Change files + fh = self.server.open(f1_inode.st_ino, os.O_RDWR, some_ctx) + self.server.write(fh, 0, b'new file1 contents') + self.server.release(fh) + + fh = self.server.open(f2_inode.st_ino, os.O_RDWR, some_ctx) + self.server.write(fh, 0, b'new file2 contents') + self.server.release(fh) + + # Get copy properties + f1_inode_c = self.server.lookup(dst_inode.st_ino, b'file1', some_ctx) + f2_inode_c = self.server.lookup(dst_inode.st_ino, b'file2', some_ctx) + f2h_inode_c = self.server.lookup(dst_inode.st_ino, b'file2_hardlink', some_ctx) + d1_inode_c = self.server.lookup(dst_inode.st_ino, b'dir1', some_ctx) + d2_inode_c = self.server.lookup(d1_inode_c.st_ino, b'dir2', some_ctx) + f2_h_inode_c = self.server.lookup(d1_inode_c.st_ino, b'file2_hardlink', some_ctx) + + # Check file1 + fh = self.server.open(f1_inode_c.st_ino, os.O_RDWR, some_ctx) + self.assertEqual(self.server.read(fh, 0, 42), b'file1 contents') + self.server.release(fh) + self.assertNotEqual(f1_inode.st_ino, f1_inode_c.st_ino) + self.assertEqual(self.server.getxattr(f1_inode_c.st_ino, ext_attr_name, + some_ctx), ext_attr_val) + + # Check file2 + fh = self.server.open(f2_inode_c.st_ino, os.O_RDWR, some_ctx) + self.assertTrue(self.server.read(fh, 0, 42) == b'file2 contents') + self.server.release(fh) + self.assertEqual(f2_inode_c.st_ino, f2h_inode_c.st_ino) + self.assertEqual(f2_inode_c.st_nlink, 3) + self.assertNotEqual(f2_inode.st_ino, f2_inode_c.st_ino) + self.assertEqual(f2_h_inode_c.st_ino, f2_inode_c.st_ino) + + # Check subdir1 + self.assertNotEqual(d1_inode.st_ino, d1_inode_c.st_ino) + self.assertNotEqual(d2_inode.st_ino, d2_inode_c.st_ino) + self.server.forget(list(self.server.open_inodes.items())) + self.fsck() + + def test_copy_tree_2(self): + src_inode = self.server.mkdir(ROOT_INODE, b'source', self.dir_mode(), some_ctx) + dst_inode = self.server.mkdir(ROOT_INODE, b'dest', self.dir_mode(), some_ctx) + + # Create file + (fh, inode) = self.server.create(src_inode.st_ino, b'file1', + self.file_mode(), os.O_RDWR, some_ctx) + self.server.write(fh, 0, b'block 1 contents') + self.server.write(fh, self.max_obj_size, b'block 1 contents') + self.server.release(fh) + self.server.forget([(inode.st_ino, 1)]) + + self.server.copy_tree(src_inode.st_ino, dst_inode.st_ino) + + self.server.forget([(src_inode.st_ino, 1), (dst_inode.st_ino, 1)]) + self.fsck() + + def test_lock_tree(self): + + inode1 = self.server.mkdir(ROOT_INODE, b'source', self.dir_mode(), some_ctx) + + # Create file + (fh, inode1a) = self.server.create(inode1.st_ino, b'file1', + self.file_mode(), os.O_RDWR, some_ctx) + self.server.write(fh, 0, b'file1 contents') + self.server.release(fh) + + # Create subdirectory + inode2 = self.server.mkdir(inode1.st_ino, b'dir1', self.dir_mode(), some_ctx) + (fh, inode2a) = self.server.create(inode2.st_ino, b'file2', + self.file_mode(), os.O_RDWR, some_ctx) + self.server.write(fh, 0, b'file2 contents') + self.server.release(fh) + + # Another file + (fh, inode3) = self.server.create(ROOT_INODE, b'file1', + self.file_mode(), os.O_RDWR, some_ctx) + self.server.release(fh) + + # Lock + self.server.lock_tree(inode1.st_ino) + + for i in (inode1.st_ino, inode1a.st_ino, inode2.st_ino, inode2a.st_ino): + self.assertTrue(self.server.inodes[i].locked) + + # Remove + with self.assertRaises(FUSEError) as cm: + self.server._remove(inode1.st_ino, b'file1', inode1a.st_ino) + self.assertEqual(cm.exception.errno, errno.EPERM) + + # Rename / Replace + with self.assertRaises(FUSEError) as cm: + self.server.rename(ROOT_INODE, b'file1', inode1.st_ino, b'file2', some_ctx) + self.assertEqual(cm.exception.errno, errno.EPERM) + with self.assertRaises(FUSEError) as cm: + self.server.rename(inode1.st_ino, b'file1', ROOT_INODE, b'file2', some_ctx) + self.assertEqual(cm.exception.errno, errno.EPERM) + + # Open + with self.assertRaises(FUSEError) as cm: + self.server.open(inode2a.st_ino, os.O_RDWR, some_ctx) + self.assertEqual(cm.exception.errno, errno.EPERM) + with self.assertRaises(FUSEError) as cm: + self.server.open(inode2a.st_ino, os.O_WRONLY, some_ctx) + self.assertEqual(cm.exception.errno, errno.EPERM) + self.server.release(self.server.open(inode3.st_ino, os.O_WRONLY, some_ctx)) + + # Write + fh = self.server.open(inode2a.st_ino, os.O_RDONLY, some_ctx) + with self.assertRaises(FUSEError) as cm: + self.server.write(fh, 0, b'foo') + self.assertEqual(cm.exception.errno, errno.EPERM) + self.server.release(fh) + + # Create + with self.assertRaises(FUSEError) as cm: + self.server._create(inode2.st_ino, b'dir1', self.dir_mode(), os.O_RDWR, some_ctx) + self.assertEqual(cm.exception.errno, errno.EPERM) + + # Setattr + attr = self.server.getattr(inode2a.st_ino, some_ctx) + with self.assertRaises(FUSEError) as cm: + self.server.setattr(inode2a.st_ino, attr, SetattrFields(update_mtime=True), + None, some_ctx) + self.assertEqual(cm.exception.errno, errno.EPERM) + + # xattr + with self.assertRaises(FUSEError) as cm: + self.server.setxattr(inode2.st_ino, b'name', b'value', some_ctx) + self.assertEqual(cm.exception.errno, errno.EPERM) + with self.assertRaises(FUSEError) as cm: + self.server.removexattr(inode2.st_ino, b'name', some_ctx) + self.assertEqual(cm.exception.errno, errno.EPERM) + self.server.forget(list(self.server.open_inodes.items())) + self.fsck() + + def test_remove_tree(self): + + inode1 = self.server.mkdir(ROOT_INODE, b'source', self.dir_mode(), some_ctx) + + # Create file + (fh, inode1a) = self.server.create(inode1.st_ino, b'file1', + self.file_mode(), os.O_RDWR, some_ctx) + self.server.write(fh, 0, b'file1 contents') + self.server.release(fh) + + # Create subdirectory + inode2 = self.server.mkdir(inode1.st_ino, b'dir1', self.dir_mode(), some_ctx) + (fh, inode2a) = self.server.create(inode2.st_ino, b'file2', + self.file_mode(), os.O_RDWR, some_ctx) + self.server.write(fh, 0, b'file2 contents') + self.server.release(fh) + + # Remove + self.server.forget(list(self.server.open_inodes.items())) + self.server.remove_tree(ROOT_INODE, b'source') + + for (id_p, name) in ((ROOT_INODE, b'source'), + (inode1.st_ino, b'file1'), + (inode1.st_ino, b'dir1'), + (inode2.st_ino, b'file2')): + self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id ' + 'WHERE name=? AND parent_inode = ?', (name, id_p))) + + for id_ in (inode1.st_ino, inode1a.st_ino, inode2.st_ino, inode2a.st_ino): + self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (id_,))) + + self.fsck() diff --git a/tests/t3_fsck.py b/tests/t3_fsck.py new file mode 100755 index 0000000..4e72e85 --- /dev/null +++ b/tests/t3_fsck.py @@ -0,0 +1,629 @@ +#!/usr/bin/env python3 +''' +t3_fsck.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +from s3ql.backends import local +from s3ql import ROOT_INODE +from s3ql.mkfs import init_tables +from s3ql.metadata import create_tables +from s3ql.database import Connection, NoSuchRowError +from s3ql.fsck import Fsck +from s3ql.common import time_ns +import os +import shutil +import hashlib +import stat +import tempfile +import _thread +import unittest + + +def sha256(s): + return hashlib.sha256(s).digest() + +class fsck_tests(unittest.TestCase): + + def setUp(self): + self.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') + self.backend = local.Backend('local://' + self.backend_dir, None, None) + self.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-') + self.max_obj_size = 1024 + + self.dbfile = tempfile.NamedTemporaryFile() + self.db = Connection(self.dbfile.name) + create_tables(self.db) + init_tables(self.db) + + self.fsck = Fsck(self.cachedir, self.backend, + { 'max_obj_size': self.max_obj_size }, self.db) + self.fsck.expect_errors = True + + def tearDown(self): + shutil.rmtree(self.cachedir) + shutil.rmtree(self.backend_dir) + self.dbfile.close() + + def assert_fsck(self, fn): + '''Check that fn detects and corrects an error''' + + self.fsck.found_errors = False + fn() + self.assertTrue(self.fsck.found_errors) + self.fsck.found_errors = False + self.fsck.check() + self.assertFalse(self.fsck.found_errors) + + def test_cache(self): + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR + | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, + os.getuid(), os.getgid(), time_ns(), time_ns(), time_ns(), + 1, 8)) + self._link(b'test-entry', inode) + + # Create new block + fh = open(self.cachedir + '/%d-0' % inode, 'wb') + fh.write(b'somedata') + fh.close() + self.assert_fsck(self.fsck.check_cache) + self.assertEqual(self.backend['s3ql_data_1'], b'somedata') + + # Existing block + self.db.execute('UPDATE inodes SET size=? WHERE id=?', + (self.max_obj_size + 8, inode)) + with open(self.cachedir + '/%d-1' % inode, 'wb') as fh: + fh.write(b'somedata') + self.assert_fsck(self.fsck.check_cache) + + # Old block preserved + with open(self.cachedir + '/%d-0' % inode, 'wb') as fh: + fh.write(b'somedat2') + self.assert_fsck(self.fsck.check_cache) + + # Old block removed + with open(self.cachedir + '/%d-1' % inode, 'wb') as fh: + fh.write(b'somedat3') + self.assert_fsck(self.fsck.check_cache) + + + def test_lof1(self): + + # Make lost+found a file + inode = self.db.get_val("SELECT inode FROM contents_v WHERE name=? AND parent_inode=?", + (b"lost+found", ROOT_INODE)) + self.db.execute('DELETE FROM contents WHERE parent_inode=?', (inode,)) + self.db.execute('UPDATE inodes SET mode=?, size=? WHERE id=?', + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, 0, inode)) + + def check(): + self.fsck.check_lof() + self.fsck.check_inodes_refcount() + + self.assert_fsck(check) + + def test_lof2(self): + # Remove lost+found + name_id = self.db.get_val('SELECT id FROM names WHERE name=?', (b'lost+found',)) + inode = self.db.get_val('SELECT inode FROM contents WHERE name_id=? AND ' + 'parent_inode=?', (name_id, ROOT_INODE)) + self.db.execute('DELETE FROM inodes WHERE id=?', (inode,)) + self.db.execute('DELETE FROM contents WHERE name_id=? and parent_inode=?', + (name_id, ROOT_INODE)) + self.db.execute('UPDATE names SET refcount = refcount-1 WHERE id=?', (name_id,)) + + self.assert_fsck(self.fsck.check_lof) + + def test_wrong_inode_refcount(self): + + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time_ns(), time_ns(), time_ns(), 1, 0)) + self._link(b'name1', inode) + self._link(b'name2', inode) + self.assert_fsck(self.fsck.check_inodes_refcount) + + def test_orphaned_inode(self): + + self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time_ns(), time_ns(), time_ns(), 1, 0)) + self.assert_fsck(self.fsck.check_inodes_refcount) + + def test_name_refcount(self): + + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time_ns(), time_ns(), time_ns(), 2, 0)) + self._link(b'name1', inode) + self._link(b'name2', inode) + + self.db.execute('UPDATE names SET refcount=refcount+1 WHERE name=?', (b'name1',)) + + self.assert_fsck(self.fsck.check_names_refcount) + + def test_orphaned_name(self): + + self._add_name(b'zupbrazl') + self.assert_fsck(self.fsck.check_names_refcount) + + def test_contents_inode(self): + + self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)', + (self._add_name(b'foobar'), 124, ROOT_INODE)) + + self.assert_fsck(self.fsck.check_contents_inode) + + def test_contents_inode_p(self): + + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time_ns(), time_ns(), time_ns(), 1, 0)) + self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)', + (self._add_name(b'foobar'), inode, 123)) + + self.assert_fsck(self.fsck.check_contents_parent_inode) + + def test_contents_name(self): + + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time_ns(), time_ns(), time_ns(), 1, 0)) + self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)', + (42, inode, ROOT_INODE)) + + self.assert_fsck(self.fsck.check_contents_name) + + def _add_name(self, name): + '''Get id for *name* and increase refcount + + Name is inserted in table if it does not yet exist. + ''' + + try: + name_id = self.db.get_val('SELECT id FROM names WHERE name=?', (name,)) + except NoSuchRowError: + name_id = self.db.rowid('INSERT INTO names (name, refcount) VALUES(?,?)', + (name, 1)) + else: + self.db.execute('UPDATE names SET refcount=refcount+1 WHERE id=?', (name_id,)) + return name_id + + def _link(self, name, inode, parent_inode=ROOT_INODE): + '''Link /*name* to *inode*''' + + self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)', + (self._add_name(name), inode, parent_inode)) + + def test_inodes_size(self): + + id_ = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time_ns(), time_ns(), time_ns(), 1, 128)) + self._link(b'test-entry', id_) + + obj_id = self.db.rowid('INSERT INTO objects (refcount,size) VALUES(?,?)', (1, 36)) + block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size, hash) ' + 'VALUES(?,?,?,?)', (1, obj_id, 512, sha256(b'foo'))) + self.backend['s3ql_data_%d' % obj_id] = b'foo' + + # Case 1 + self.db.execute('UPDATE inodes SET size=? WHERE id=?', (self.max_obj_size + 120, id_)) + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?, ?, ?)', + (id_, 1, block_id)) + self.assert_fsck(self.fsck.check_inodes_size) + + # Case 2 + self.db.execute('DELETE FROM inode_blocks WHERE inode=?', (id_,)) + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?, ?, ?)', + (id_, 0, block_id)) + self.db.execute('UPDATE inodes SET size=? WHERE id=?', (129, id_)) + self.assert_fsck(self.fsck.check_inodes_size) + + # Case 3 + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?, ?, ?)', + (id_, 1, block_id)) + self.db.execute('UPDATE inodes SET size=? WHERE id=?', + (self.max_obj_size + 120, id_)) + self.db.execute('UPDATE blocks SET refcount = refcount + 1 WHERE id = ?', + (block_id,)) + self.assert_fsck(self.fsck.check_inodes_size) + + + def test_objects_id(self): + # Create an object that only exists in the backend + self.backend['s3ql_data_4364'] = b'Testdata' + self.assert_fsck(self.fsck.check_objects_id) + + # Create an object that does not exist in the backend + self.db.execute('INSERT INTO objects (id, refcount, size) VALUES(?, ?, ?)', (34, 1, 27)) + self.assert_fsck(self.fsck.check_objects_id) + + def test_blocks_checksum(self): + id_ = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time_ns(), time_ns(), time_ns(), 1, 8)) + self._link(b'test-entry', id_) + + # Assume that due to a crash we did not write the hash for the block + self.backend['s3ql_data_4364'] = b'Testdata' + self.db.execute('INSERT INTO objects (id, refcount, size) VALUES(?, ?, ?)', + (4364, 1, 8)) + block_id = self.db.execute('INSERT INTO blocks (obj_id, refcount, size) VALUES(?, ?, ?)', + (4364, 1, 8)) + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?, ?, ?)', + (id_, 0, block_id)) + + + # Should pick up wrong hash and delete objects + self.fsck.found_errors = False + self.fsck.check_blocks_checksum() + assert self.fsck.found_errors + self.fsck.found_errors = False + self.fsck.check_blocks_checksum() + assert not self.fsck.found_errors + + # Should save files in lost+found + self.fsck.found_errors = False + self.fsck.check() + assert self.fsck.found_errors + + # Now everything should be good + self.fsck.found_errors = False + self.fsck.check() + assert not self.fsck.found_errors + + assert not self.db.has_val('SELECT block_id FROM inode_blocks WHERE inode=?', + (id_,)) + inode_p = self.db.get_val('SELECT parent_inode FROM contents_v WHERE inode=?', (id_,)) + lof_id = self.db.get_val("SELECT inode FROM contents_v WHERE name=? AND parent_inode=?", + (b"lost+found", ROOT_INODE)) + assert inode_p == lof_id + + def test_blocks_obj_id(self): + + block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)', + (1, 48, 128)) + + id_ = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time_ns(), time_ns(), time_ns(), 1, 128)) + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)', + (id_, 0, block_id)) + + self._link(b'test-entry', id_) + self.assert_fsck(self.fsck.check_blocks_obj_id) + + def test_missing_obj(self): + + obj_id = self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, 32)') + block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)', + (1, obj_id, 128)) + + id_ = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time_ns(), time_ns(), time_ns(), 1, 128)) + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)', + (id_, 0, block_id)) + + self._link(b'test-entry', id_) + self.assert_fsck(self.fsck.check_objects_id) + + + def test_inode_blocks_inode(self): + + obj_id = self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, 42)') + self.backend['s3ql_data_%d' % obj_id] = b'foo' + + block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)', + (1, obj_id, 34)) + + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)', + (27, 0, block_id)) + + self.assert_fsck(self.fsck.check_inode_blocks_inode) + + def test_inode_blocks_block_id(self): + + id_ = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time_ns(), time_ns(), time_ns(), 1, 128)) + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)', + (id_, 0, 35)) + + self._link(b'test-entry', id_) + self.assert_fsck(self.fsck.check_inode_blocks_block_id) + + def test_symlinks_inode(self): + + self.db.execute('INSERT INTO symlink_targets (inode, target) VALUES(?,?)', + (42, b'somewhere else')) + + self.assert_fsck(self.fsck.check_symlinks_inode) + + def test_ext_attrs_inode(self): + + self.db.execute('INSERT INTO ext_attributes (name_id, inode, value) VALUES(?,?,?)', + (self._add_name(b'some name'), 34, b'some value')) + + self.assert_fsck(self.fsck.check_ext_attributes_inode) + + def test_ext_attrs_name(self): + + id_ = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time_ns(), time_ns(), time_ns(), 1, 128)) + self._link(b'test-entry', id_) + + self.db.execute('INSERT INTO ext_attributes (name_id, inode, value) VALUES(?,?,?)', + (34, id_, b'some value')) + + self.assert_fsck(self.fsck.check_ext_attributes_name) + + @staticmethod + def random_data(len_): + with open("/dev/urandom", "rb") as fd: + return fd.read(len_) + + def test_loops(self): + + # Create some directory inodes + inodes = [ self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) " + "VALUES (?,?,?,?,?,?,?)", + (stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR, + 0, 0, time_ns(), time_ns(), time_ns(), 1)) + for dummy in range(3) ] + + inodes.append(inodes[0]) + last = inodes[0] + for inode in inodes[1:]: + self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?, ?, ?)', + (self._add_name(str(inode).encode()), inode, last)) + last = inode + + self.assert_fsck(self.fsck.check_loops) + + def test_tmpfile(self): + # Ensure that path exists + objname = 's3ql_data_38375' + self.backend[objname] = b'bla' + del self.backend[objname] + path = self.backend._key_to_path(objname) + tmpname = '%s#%d-%d.tmp' % (path, os.getpid(), _thread.get_ident()) + with open(tmpname, 'wb') as fh: + fh.write(b'Hello, world') + + self.assert_fsck(self.fsck.check_objects_temp) + + def test_obj_refcounts(self): + + obj_id = self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, 42)') + block_id_1 = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size, hash) ' + 'VALUES(?,?,?,?)', (1, obj_id, 0, sha256(b'foo'))) + block_id_2 = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size, hash) ' + 'VALUES(?,?,?,?)', (1, obj_id, 0, sha256(b'bar'))) + self.backend['s3ql_data_%d' % obj_id] = b'foo and bar' + + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + os.getuid(), os.getgid(), time_ns(), time_ns(), time_ns(), + 1, 2048)) + self._link(b'test-entry', inode) + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)', + (inode, 1, block_id_1)) + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)', + (inode, 2, block_id_2)) + + self.assert_fsck(self.fsck.check_objects_refcount) + + def test_orphaned_obj(self): + + self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, 33)') + self.assert_fsck(self.fsck.check_objects_refcount) + + def test_wrong_block_refcount(self): + + obj_id = self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, 23)') + self.backend['s3ql_data_%d' % obj_id] = b'foo' + block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size, hash) ' + 'VALUES(?,?,?,?)', (1, obj_id, 0, sha256(b''))) + + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, os.getuid(), os.getgid(), + time_ns(), time_ns(), time_ns(), 1, self.max_obj_size)) + self._link(b'test-entry', inode) + + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)', + (inode, 0, block_id)) + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)', + (inode, 1, block_id)) + + self.assert_fsck(self.fsck.check_blocks_refcount) + + def test_orphaned_block(self): + + obj_id = self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, 24)') + self.backend['s3ql_data_%d' % obj_id] = b'foo' + self.db.rowid('INSERT INTO blocks (refcount, obj_id, size, hash) VALUES(?,?,?,?)', + (1, obj_id, 3, sha256(b'xyz'))) + self.assert_fsck(self.fsck.check_blocks_refcount) + + def test_unix_size(self): + + inode = 42 + self.db.execute("INSERT INTO inodes (id, mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?,?)", + (inode, stat.S_IFIFO | stat.S_IRUSR | stat.S_IWUSR, + os.getuid(), os.getgid(), time_ns(), time_ns(), time_ns(), 1, 0)) + self._link(b'test-entry', inode) + + self.fsck.found_errors = False + self.fsck.check_unix() + self.assertFalse(self.fsck.found_errors) + + self.db.execute('UPDATE inodes SET size = 1 WHERE id=?', (inode,)) + self.fsck.check_unix() + self.assertTrue(self.fsck.found_errors) + + + def test_unix_size_symlink(self): + + inode = 42 + target = b'some funny random string' + self.db.execute("INSERT INTO inodes (id, mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) " + "VALUES (?,?,?,?,?,?,?,?,?)", + (inode, stat.S_IFLNK | stat.S_IRUSR | stat.S_IWUSR, + os.getuid(), os.getgid(), time_ns(), time_ns(), time_ns(), 1, + len(target))) + self.db.execute('INSERT INTO symlink_targets (inode, target) VALUES(?,?)', (inode, target)) + self._link(b'test-entry', inode) + + self.fsck.found_errors = False + self.fsck.check_unix() + self.assertFalse(self.fsck.found_errors) + + self.db.execute('UPDATE inodes SET size = 0 WHERE id=?', (inode,)) + self.fsck.check_unix() + self.assertTrue(self.fsck.found_errors) + + def test_unix_target(self): + + inode = 42 + self.db.execute("INSERT INTO inodes (id, mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) " + "VALUES (?,?,?,?,?,?,?,?)", + (inode, stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR, + os.getuid(), os.getgid(), time_ns(), time_ns(), time_ns(), 1)) + self._link(b'test-entry', inode) + + self.fsck.found_errors = False + self.fsck.check_unix() + self.assertFalse(self.fsck.found_errors) + + self.db.execute('INSERT INTO symlink_targets (inode, target) VALUES(?,?)', (inode, 'foo')) + self.fsck.check_unix() + self.assertTrue(self.fsck.found_errors) + + def test_unix_nomode_reg(self): + + perms = stat.S_IRUSR | stat.S_IWUSR | stat.S_IROTH | stat.S_IRGRP + stamp = time_ns() + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) " + "VALUES (?,?,?,?,?,?,?)", + (perms, os.getuid(), os.getgid(), stamp, stamp, stamp, 1)) + self._link(b'test-entry', inode) + + self.assert_fsck(self.fsck.check_unix) + + newmode = self.db.get_val('SELECT mode FROM inodes WHERE id=?', (inode,)) + self.assertEqual(stat.S_IMODE(newmode), perms) + self.assertEqual(stat.S_IFMT(newmode), stat.S_IFREG) + + def test_unix_nomode_dir(self): + + perms = stat.S_IRUSR | stat.S_IWUSR | stat.S_IROTH | stat.S_IRGRP + stamp = time_ns() + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) " + "VALUES (?,?,?,?,?,?,?)", + (perms, os.getuid(), os.getgid(), stamp, stamp, stamp, 1)) + inode2 = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) " + "VALUES (?,?,?,?,?,?,?)", + (perms | stat.S_IFREG, os.getuid(), os.getgid(), stamp, + stamp, stamp, 1)) + + self._link(b'test-entry', inode) + self._link(b'subentry', inode2, inode) + + self.assert_fsck(self.fsck.check_unix) + + newmode = self.db.get_val('SELECT mode FROM inodes WHERE id=?', (inode,)) + self.assertEqual(stat.S_IMODE(newmode), perms) + self.assertEqual(stat.S_IFMT(newmode), stat.S_IFDIR) + + + def test_unix_symlink_no_target(self): + + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) " + "VALUES (?,?,?,?,?,?,?)", + (stat.S_IFLNK | stat.S_IRUSR | stat.S_IWUSR, + os.getuid(), os.getgid(), time_ns(), time_ns(), time_ns(), 1)) + self._link(b'test-entry', inode) + self.fsck.check_unix() + self.assertTrue(self.fsck.found_errors) + + def test_unix_rdev(self): + + inode = 42 + self.db.execute("INSERT INTO inodes (id, mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) " + "VALUES (?,?,?,?,?,?,?,?)", + (inode, stat.S_IFIFO | stat.S_IRUSR | stat.S_IWUSR, + os.getuid(), os.getgid(), time_ns(), time_ns(), time_ns(), 1)) + self._link(b'test-entry', inode) + + self.fsck.found_errors = False + self.fsck.check_unix() + self.assertFalse(self.fsck.found_errors) + + self.db.execute('UPDATE inodes SET rdev=? WHERE id=?', (42, inode)) + self.fsck.check_unix() + self.assertTrue(self.fsck.found_errors) + + def test_unix_child(self): + + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) " + "VALUES (?,?,?,?,?,?,?)", + (stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, + os.getuid(), os.getgid(), time_ns(), time_ns(), time_ns(), 1)) + self._link(b'test-entry', inode) + + self.fsck.found_errors = False + self.fsck.check_unix() + self.assertFalse(self.fsck.found_errors) + self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)', + (self._add_name(b'foo'), ROOT_INODE, inode)) + self.fsck.check_unix() + self.assertTrue(self.fsck.found_errors) + + def test_unix_blocks(self): + + inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) " + "VALUES (?,?,?,?,?,?,?)", + (stat.S_IFSOCK | stat.S_IRUSR | stat.S_IWUSR, + os.getuid(), os.getgid(), time_ns(), time_ns(), time_ns(), 1)) + self._link(b'test-entry', inode) + + self.fsck.found_errors = False + self.fsck.check_unix() + self.assertFalse(self.fsck.found_errors) + + obj_id = self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, 32)') + block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)', + (1, obj_id, 0)) + + self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)', + (inode, 1, block_id)) + + self.fsck.check_unix() + self.assertTrue(self.fsck.found_errors) diff --git a/tests/t3_inode_cache.py b/tests/t3_inode_cache.py new file mode 100755 index 0000000..950e48b --- /dev/null +++ b/tests/t3_inode_cache.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +''' +t2_inode_cache.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +from s3ql import inode_cache +from s3ql.mkfs import init_tables +from s3ql.common import time_ns +from s3ql.metadata import create_tables +from s3ql.database import Connection +import unittest +import tempfile +import os + +class cache_tests(unittest.TestCase): + + def setUp(self): + # Destructors are not guaranteed to run, and we can't unlink + # the file immediately because apsw refers to it by name. + # Therefore, we unlink the file manually in tearDown() + self.dbfile = tempfile.NamedTemporaryFile(delete=False) + + self.db = Connection(self.dbfile.name) + create_tables(self.db) + init_tables(self.db) + self.cache = inode_cache.InodeCache(self.db, 0) + + def tearDown(self): + self.cache.destroy() + os.unlink(self.dbfile.name) + + def test_create(self): + attrs = {'mode': 784, + 'refcount': 3, + 'uid': 7, + 'gid': 2, + 'size': 34674, + 'rdev': 11, + 'atime_ns': time_ns(), + 'ctime_ns': time_ns(), + 'mtime_ns': time_ns() } + + inode = self.cache.create_inode(**attrs) + + for key in list(attrs.keys()): + self.assertEqual(attrs[key], getattr(inode, key)) + + self.assertTrue(self.db.has_val('SELECT 1 FROM inodes WHERE id=?', (inode.id,))) + + + def test_del(self): + attrs = {'mode': 784, + 'refcount': 3, + 'uid': 7, + 'gid': 2, + 'size': 34674, + 'rdev': 11, + 'atime_ns': time_ns(), + 'ctime_ns': time_ns(), + 'mtime_ns': time_ns() } + inode = self.cache.create_inode(**attrs) + del self.cache[inode.id] + self.assertFalse(self.db.has_val('SELECT 1 FROM inodes WHERE id=?', (inode.id,))) + self.assertRaises(KeyError, self.cache.__delitem__, inode.id) + + def test_get(self): + attrs = {'mode': 784, + 'refcount': 3, + 'uid': 7, + 'gid': 2, + 'size': 34674, + 'rdev': 11, + 'atime_ns': time_ns(), + 'ctime_ns': time_ns(), + 'mtime_ns': time_ns() } + + inode = self.cache.create_inode(**attrs) + for (key, val) in attrs.items(): + self.assertEqual(getattr(inode, key), val) + + # Create another inode + self.cache.create_inode(**attrs) + + self.db.execute('DELETE FROM inodes WHERE id=?', (inode.id,)) + # Entry should still be in cache + self.assertEqual(inode, self.cache[inode.id]) + + # Now it should be out of the cache + for _ in range(inode_cache.CACHE_SIZE + 1): + self.cache.create_inode(**attrs) + + self.assertRaises(KeyError, self.cache.__getitem__, inode.id) diff --git a/tests/t3_verify.py b/tests/t3_verify.py new file mode 100755 index 0000000..c2548cc --- /dev/null +++ b/tests/t3_verify.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +''' +t3_verify.py - this file is part of S3QL. + +Copyright © 2014 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +from s3ql.backends import local +from s3ql.backends.comprenc import ComprencBackend +from s3ql.mkfs import init_tables +from s3ql.metadata import create_tables +from s3ql.database import Connection +from s3ql import verify +from pytest_checklogs import assert_logs +import io +import logging +import shutil +import tempfile +import pytest + +@pytest.yield_fixture() +def backend(): + backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') + be = local.Backend('local://' + backend_dir, None, None) + try: + yield be + finally: + be.close() + shutil.rmtree(backend_dir) + +@pytest.yield_fixture() +def db(): + dbfile = tempfile.NamedTemporaryFile() + db = Connection(dbfile.name) + create_tables(db) + init_tables(db) + try: + yield db + finally: + db.close() + dbfile.close() + +def test_retrieve(backend, db): + plain_backend = backend + backend = ComprencBackend(b'schnorz', ('zlib', 6), plain_backend) + + # Create a few objects in db + obj_ids = (22, 25, 30, 31) + for id_ in obj_ids: + db.execute('INSERT INTO objects (id, refcount, size) VALUES(?, ?, ?)', + (id_, 1, 27 * id_)) + + # Object one will be missing in backend + + # Object two will have a checksum error in the data + key = 's3ql_data_%d' % obj_ids[1] + backend[key] = b'some data that will be broken on a data check' + (raw, meta) = plain_backend.fetch(key) + raw = bytearray(raw) + assert len(raw) > 20 + raw[-10:-6] = b'forg' + plain_backend.store(key, raw, meta) + + # Object three will have a checksum error in the metadata + key = 's3ql_data_%d' % obj_ids[2] + backend.store(key, b'some data that will be broken on a metadata check', + { 'meta-key1': 'some textual data that just increases', + 'meta-key2': 'the metadata size so that we can tamper with it' }) + meta = plain_backend.lookup(key) + raw = bytearray(meta['data']) + assert len(raw) > 20 + raw[-10:-6] = b'forg' + meta['data'] = raw + plain_backend.update_meta(key, meta) + + # Object four will be ok + backend['s3ql_data_%d' % obj_ids[3]] = b'some data that is well' + + # When using a single thread, we can fake the backend factory + def backend_factory(): + return backend + + missing_fh = io.StringIO() + corrupted_fh = io.StringIO() + + with assert_logs('^Backend seems to have lost', count=1, level=logging.WARNING), \ + assert_logs('^Object %d is corrupted', count=1, level=logging.WARNING): + verify.retrieve_objects(db, backend_factory, corrupted_fh, missing_fh, + thread_count=1, full=False) + assert missing_fh.getvalue() == 's3ql_data_%d\n' % obj_ids[0] + assert corrupted_fh.getvalue() == 's3ql_data_%d\n' % obj_ids[2] + + missing_fh = io.StringIO() + corrupted_fh = io.StringIO() + with assert_logs('^Backend seems to have lost', count=1, level=logging.WARNING), \ + assert_logs('^Object %d is corrupted', count=2, level=logging.WARNING): + verify.retrieve_objects(db, backend_factory, corrupted_fh, missing_fh, + thread_count=1, full=True) + assert missing_fh.getvalue() == 's3ql_data_%d\n' % obj_ids[0] + assert corrupted_fh.getvalue() == ('s3ql_data_%d\n'*2) % obj_ids[1:3] diff --git a/tests/t4_adm.py b/tests/t4_adm.py new file mode 100755 index 0000000..a17267e --- /dev/null +++ b/tests/t4_adm.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +''' +t4_adm.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +from s3ql.backends import local +from s3ql.backends.comprenc import ComprencBackend +import shutil +import tempfile +import unittest +import subprocess +import pytest + +@pytest.mark.usefixtures('s3ql_cmd_argv', 'pass_reg_output') +class AdmTests(unittest.TestCase): + + def setUp(self): + self.cache_dir = tempfile.mkdtemp(prefix='s3ql-cache-') + self.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') + + self.storage_url = 'local://' + self.backend_dir + self.passphrase = 'oeut3d' + + def tearDown(self): + shutil.rmtree(self.cache_dir) + shutil.rmtree(self.backend_dir) + + def mkfs(self): + proc = subprocess.Popen(self.s3ql_cmd_argv('mkfs.s3ql') + + ['-L', 'test fs', '--max-obj-size', '500', + '--authfile', '/dev/null', '--cachedir', self.cache_dir, + '--quiet', self.storage_url ], + stdin=subprocess.PIPE, universal_newlines=True) + + print(self.passphrase, file=proc.stdin) + print(self.passphrase, file=proc.stdin) + proc.stdin.close() + + self.assertEqual(proc.wait(), 0) + self.reg_output(r'^WARNING: Maximum object sizes less than ' + '1 MiB will degrade performance\.$', count=1) + + def test_passphrase(self): + self.mkfs() + + passphrase_new = 'sd982jhd' + + proc = subprocess.Popen(self.s3ql_cmd_argv('s3qladm') + + [ '--quiet', '--log', 'none', '--authfile', + '/dev/null', 'passphrase', self.storage_url ], + stdin=subprocess.PIPE, universal_newlines=True) + + print(self.passphrase, file=proc.stdin) + print(passphrase_new, file=proc.stdin) + print(passphrase_new, file=proc.stdin) + proc.stdin.close() + + self.assertEqual(proc.wait(), 0) + + plain_backend = local.Backend(self.storage_url, None, None) + backend = ComprencBackend(passphrase_new.encode(), ('zlib', 6), plain_backend) + + backend.fetch('s3ql_passphrase') # will fail with wrong pw + + + def test_authinfo(self): + self.mkfs() + + with tempfile.NamedTemporaryFile('wt') as fh: + print('[entry1]', + 'storage-url: local://', + 'fs-passphrase: clearly wrong', + '', + '[entry2]', + 'storage-url: %s' % self.storage_url, + 'fs-passphrase: %s' % self.passphrase, + file=fh, sep='\n') + fh.flush() + + proc = subprocess.Popen(self.s3ql_cmd_argv('fsck.s3ql') + + [ '--quiet', '--authfile', fh.name, + '--cachedir', self.cache_dir, '--log', 'none', self.storage_url ], + stdin=subprocess.PIPE, universal_newlines=True) + + proc.stdin.close() + self.assertEqual(proc.wait(), 0) diff --git a/tests/t4_fuse.py b/tests/t4_fuse.py new file mode 100755 index 0000000..d529c77 --- /dev/null +++ b/tests/t4_fuse.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python3 +''' +t4_fuse.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +from os.path import basename +from s3ql import CTRL_NAME +from s3ql.common import path2bytes +from common import retry, skip_if_no_fusermount +import filecmp +import llfuse +import os.path +import shutil +import platform +import stat +import subprocess +import tempfile +import pytest +from pytest import raises as assert_raises + +# For debugging +USE_VALGRIND = False + +@pytest.mark.usefixtures('s3ql_cmd_argv', 'pass_reg_output') +class TestFuse: + + def setup_method(self, method): + if platform.system() != 'Darwin': + skip_if_no_fusermount() + + # We need this to test multi block operations + self.src = __file__ + if os.path.getsize(self.src) < 1048: + raise RuntimeError("test file %s should be bigger than 1 KiB" % self.src) + + self.mnt_dir = tempfile.mkdtemp(prefix='s3ql-mnt-') + self.cache_dir = tempfile.mkdtemp(prefix='s3ql-cache-') + self.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') + + self.storage_url = 'local://%s/' % (self.backend_dir,) + self.passphrase = 'oeut3d' + self.backend_login = None + self.backend_passphrase = None + + self.mount_process = None + self.name_cnt = 0 + + def mkfs(self, max_obj_size=500): + argv = (self.s3ql_cmd_argv('mkfs.s3ql') + + [ '-L', 'test fs', '--max-obj-size', str(max_obj_size), + '--cachedir', self.cache_dir, '--quiet', + '--authfile', '/dev/null', self.storage_url ]) + if self.passphrase is None: + argv.append('--plain') + + proc = subprocess.Popen(argv, stdin=subprocess.PIPE, universal_newlines=True) + + if self.backend_login is not None: + print(self.backend_login, file=proc.stdin) + print(self.backend_passphrase, file=proc.stdin) + if self.passphrase is not None: + print(self.passphrase, file=proc.stdin) + print(self.passphrase, file=proc.stdin) + proc.stdin.close() + + assert proc.wait() == 0 + self.reg_output(r'^WARNING: Maximum object sizes less than ' + '1 MiB will degrade performance\.$', count=1) + + def mount(self, expect_fail=None): + cmd = (self.s3ql_cmd_argv('mount.s3ql') + + ["--fg", '--cachedir', self.cache_dir, '--log', 'none', + '--compress', 'zlib', '--quiet', self.storage_url, self.mnt_dir, + '--authfile', '/dev/null' ]) + self.mount_process = subprocess.Popen(cmd, stdin=subprocess.PIPE, + universal_newlines=True) + if self.backend_login is not None: + print(self.backend_login, file=self.mount_process.stdin) + print(self.backend_passphrase, file=self.mount_process.stdin) + if self.passphrase is not None: + print(self.passphrase, file=self.mount_process.stdin) + self.mount_process.stdin.close() + + if expect_fail: + retry(30, self.mount_process.poll) + assert self.mount_process.returncode == expect_fail + else: + def poll(): + if os.path.ismount(self.mnt_dir): + return True + assert self.mount_process.poll() is None + retry(30, poll) + + def umount(self): + with open('/dev/null', 'wb') as devnull: + retry(5, lambda: subprocess.call(['fuser', '-m', self.mnt_dir], + stdout=devnull, stderr=devnull) == 1) + + proc = subprocess.Popen(self.s3ql_cmd_argv('umount.s3ql') + + ['--quiet', self.mnt_dir]) + retry(90, lambda : proc.poll() is not None) + assert proc.wait() == 0 + + assert self.mount_process.poll() == 0 + assert not os.path.ismount(self.mnt_dir) + + def fsck(self): + # Use fsck to test authinfo reading + with tempfile.NamedTemporaryFile('wt') as authinfo_fh: + print('[entry1]', + 'storage-url: %s' % self.storage_url[:6], + 'fs-passphrase: clearly wrong', + 'backend-login: bla', + 'backend-password: not much better', + '', + '[entry2]', + 'storage-url: %s' % self.storage_url, + 'fs-passphrase: %s' % self.passphrase, + 'backend-login: %s' % self.backend_login, + 'backend-password:%s' % self.backend_passphrase, + file=authinfo_fh, sep='\n') + authinfo_fh.flush() + + proc = subprocess.Popen(self.s3ql_cmd_argv('fsck.s3ql') + + [ '--force', '--quiet', '--log', 'none', '--cachedir', + self.cache_dir, '--authfile', + authinfo_fh.name, self.storage_url ], + stdin=subprocess.PIPE, universal_newlines=True) + proc.stdin.close() + assert proc.wait() == 0 + + def teardown_method(self, method): + with open('/dev/null', 'wb') as devnull: + if platform.system() == 'Darwin': + subprocess.call(['umount', '-l', self.mnt_dir], stderr=devnull) + else: + subprocess.call(['fusermount', '-z', '-u', self.mnt_dir], + stderr=devnull) + os.rmdir(self.mnt_dir) + + # Give mount process a little while to terminate + if self.mount_process is not None: + try: + retry(90, lambda : self.mount_process.poll() is not None) + except TimeoutError: + # Ignore errors during teardown + pass + + shutil.rmtree(self.cache_dir) + shutil.rmtree(self.backend_dir) + + def test(self): + # Run all tests in same environment, mounting and umounting + # just takes too long otherwise + + self.mkfs() + self.mount() + self.tst_chown() + self.tst_link() + self.tst_mkdir() + self.tst_mknod() + self.tst_readdir() + self.tst_statvfs() + self.tst_symlink() + self.tst_truncate() + self.tst_truncate_nocache() + self.tst_write() + self.tst_bug382() + self.umount() + self.fsck() + + def newname(self): + self.name_cnt += 1 + return "s3ql_%d" % self.name_cnt + + def tst_mkdir(self): + dirname = self.newname() + fullname = self.mnt_dir + "/" + dirname + os.mkdir(fullname) + fstat = os.stat(fullname) + assert stat.S_ISDIR(fstat.st_mode) + assert llfuse.listdir(fullname) == [] + assert fstat.st_nlink == 1 + assert dirname in llfuse.listdir(self.mnt_dir) + os.rmdir(fullname) + assert_raises(FileNotFoundError, os.stat, fullname) + assert dirname not in llfuse.listdir(self.mnt_dir) + + def tst_symlink(self): + linkname = self.newname() + fullname = self.mnt_dir + "/" + linkname + os.symlink("/imaginary/dest", fullname) + fstat = os.lstat(fullname) + assert stat.S_ISLNK(fstat.st_mode) + assert os.readlink(fullname) == "/imaginary/dest" + assert fstat.st_nlink == 1 + assert linkname in llfuse.listdir(self.mnt_dir) + os.unlink(fullname) + assert_raises(FileNotFoundError, os.lstat, fullname) + assert linkname not in llfuse.listdir(self.mnt_dir) + + def tst_mknod(self): + filename = os.path.join(self.mnt_dir, self.newname()) + src = self.src + shutil.copyfile(src, filename) + fstat = os.lstat(filename) + assert stat.S_ISREG(fstat.st_mode) + assert fstat.st_nlink == 1 + assert basename(filename) in llfuse.listdir(self.mnt_dir) + assert filecmp.cmp(src, filename, False) + os.unlink(filename) + assert_raises(FileNotFoundError, os.stat, filename) + assert basename(filename) not in llfuse.listdir(self.mnt_dir) + + def tst_chown(self): + filename = os.path.join(self.mnt_dir, self.newname()) + os.mkdir(filename) + fstat = os.lstat(filename) + uid = fstat.st_uid + gid = fstat.st_gid + + uid_new = uid + 1 + os.chown(filename, uid_new, -1) + fstat = os.lstat(filename) + assert fstat.st_uid == uid_new + assert fstat.st_gid == gid + + gid_new = gid + 1 + os.chown(filename, -1, gid_new) + fstat = os.lstat(filename) + assert fstat.st_uid == uid_new + assert fstat.st_gid == gid_new + + os.rmdir(filename) + assert_raises(FileNotFoundError, os.stat, filename) + assert basename(filename) not in llfuse.listdir(self.mnt_dir) + + def tst_write(self): + name = os.path.join(self.mnt_dir, self.newname()) + src = self.src + shutil.copyfile(src, name) + assert filecmp.cmp(name, src, False) + + # Don't unlink file, we want to see if cache flushing + # works + + def tst_statvfs(self): + os.statvfs(self.mnt_dir) + + def tst_link(self): + name1 = os.path.join(self.mnt_dir, self.newname()) + name2 = os.path.join(self.mnt_dir, self.newname()) + src = self.src + shutil.copyfile(src, name1) + assert filecmp.cmp(name1, src, False) + os.link(name1, name2) + + fstat1 = os.lstat(name1) + fstat2 = os.lstat(name2) + + assert fstat1 == fstat2 + assert fstat1.st_nlink == 2 + + assert basename(name2) in llfuse.listdir(self.mnt_dir) + assert filecmp.cmp(name1, name2, False) + os.unlink(name2) + fstat1 = os.lstat(name1) + assert fstat1.st_nlink == 1 + os.unlink(name1) + + def tst_readdir(self): + dir_ = os.path.join(self.mnt_dir, self.newname()) + file_ = dir_ + "/" + self.newname() + subdir = dir_ + "/" + self.newname() + subfile = subdir + "/" + self.newname() + src = self.src + + os.mkdir(dir_) + shutil.copyfile(src, file_) + os.mkdir(subdir) + shutil.copyfile(src, subfile) + + listdir_is = llfuse.listdir(dir_) + listdir_is.sort() + listdir_should = [ basename(file_), basename(subdir) ] + listdir_should.sort() + assert listdir_is == listdir_should + + os.unlink(file_) + os.unlink(subfile) + os.rmdir(subdir) + os.rmdir(dir_) + + def tst_truncate(self): + filename = os.path.join(self.mnt_dir, self.newname()) + src = self.src + shutil.copyfile(src, filename) + assert filecmp.cmp(filename, src, False) + fstat = os.stat(filename) + size = fstat.st_size + fd = os.open(filename, os.O_RDWR) + + os.ftruncate(fd, size + 1024) # add > 1 block + assert os.stat(filename).st_size == size + 1024 + + os.ftruncate(fd, size - 1024) # Truncate > 1 block + assert os.stat(filename).st_size == size - 1024 + + os.close(fd) + os.unlink(filename) + + def tst_truncate_nocache(self): + filename = os.path.join(self.mnt_dir, self.newname()) + src = self.src + shutil.copyfile(src, filename) + assert filecmp.cmp(filename, src, False) + fstat = os.stat(filename) + size = fstat.st_size + + subprocess.check_call(self.s3ql_cmd_argv('s3qlctrl') + + [ '--quiet', 'flushcache', self.mnt_dir ]) + + fd = os.open(filename, os.O_RDWR) + + os.ftruncate(fd, size + 1024) # add > 1 block + assert os.stat(filename).st_size == size + 1024 + + os.ftruncate(fd, size - 1024) # Truncate > 1 block + assert os.stat(filename).st_size == size - 1024 + + os.close(fd) + os.unlink(filename) + + def tst_bug382(self): + dirname = self.newname() + fullname = self.mnt_dir + "/" + dirname + os.mkdir(fullname) + assert stat.S_ISDIR(os.stat(fullname).st_mode) + assert dirname in llfuse.listdir(self.mnt_dir) + cmd = ('(%d, %r)' % (llfuse.ROOT_INODE, path2bytes(dirname))).encode() + llfuse.setxattr('%s/%s' % (self.mnt_dir, CTRL_NAME), 'rmtree', cmd) + assert_raises(FileNotFoundError, os.stat, fullname) + assert dirname not in llfuse.listdir(self.mnt_dir) diff --git a/tests/t5_cp.py b/tests/t5_cp.py new file mode 100755 index 0000000..d2c8681 --- /dev/null +++ b/tests/t5_cp.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +''' +t5_cp.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +from common import populate_dir, skip_without_rsync +import os.path +import shutil +import subprocess +from subprocess import check_output, CalledProcessError +import t4_fuse +import tempfile +import pytest + +class TestCp(t4_fuse.TestFuse): + + def test(self): + skip_without_rsync() + self.mkfs() + self.mount() + self.tst_cp() + self.umount() + self.fsck() + + def tst_cp(self): + + tempdir = tempfile.mkdtemp(prefix='s3ql-cp-') + try: + populate_dir(tempdir) + + # Rsync + subprocess.check_call(['rsync', '-aHAX', tempdir + '/', + os.path.join(self.mnt_dir, 'orig') + '/']) + + # copy + subprocess.check_call(self.s3ql_cmd_argv('s3qlcp') + + [ '--quiet', os.path.join(self.mnt_dir, 'orig'), + os.path.join(self.mnt_dir, 'copy')]) + + # compare + try: + out = check_output(['rsync', '-anciHAX', '--delete', tempdir + '/', + os.path.join(self.mnt_dir, 'copy') + '/'], + universal_newlines=True, stderr=subprocess.STDOUT) + except CalledProcessError as exc: + pytest.fail('rsync failed with ' + exc.output) + + if out: + pytest.fail('Copy not equal to original, rsync says:\n' + out) + + finally: + shutil.rmtree(tempdir) diff --git a/tests/t5_ctrl.py b/tests/t5_ctrl.py new file mode 100755 index 0000000..29834ce --- /dev/null +++ b/tests/t5_ctrl.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +''' +t5_ctrl.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +import s3ql.ctrl +import sys +import t4_fuse + +class TestCtrl(t4_fuse.TestFuse): + + def test(self): + self.mkfs() + self.mount() + self.tst_ctrl_flush() + self.tst_ctrl_log() + self.tst_ctrl_cachesize() + self.umount() + self.fsck() + + def tst_ctrl_flush(self): + try: + s3ql.ctrl.main(['flushcache', self.mnt_dir]) + except: + sys.excepthook(*sys.exc_info()) + pytest.fail("s3qlctrl raised exception") + + def tst_ctrl_log(self): + try: + s3ql.ctrl.main(['log', self.mnt_dir, 'warn']) + s3ql.ctrl.main(['log', self.mnt_dir, 'debug', 's3ql', 'dugong']) + s3ql.ctrl.main(['log', self.mnt_dir, 'info']) + except: + sys.excepthook(*sys.exc_info()) + pytest.fail("s3qlctrl raised exception") + + def tst_ctrl_cachesize(self): + try: + s3ql.ctrl.main(['cachesize', self.mnt_dir, '10240']) + except: + sys.excepthook(*sys.exc_info()) + pytest.fail("s3qlctrl raised exception") diff --git a/tests/t5_failsafe.py b/tests/t5_failsafe.py new file mode 100755 index 0000000..6559497 --- /dev/null +++ b/tests/t5_failsafe.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +''' +t5_failsafe.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +import os.path +import t4_fuse +import s3ql.ctrl +import pytest +import errno +import time +from common import get_remote_test_info, NoTestSection +from s3ql.backends import gs +from s3ql.backends.local import Backend as LocalBackend +from s3ql.common import get_seq_no +from s3ql import BUFSIZE + +@pytest.mark.usefixtures('pass_reg_output') +class TestFailsafe(t4_fuse.TestFuse): + ''' + Test behavior with corrupted backend. In contrast to the tests + in t3_fs_api, here we also make sure that remote connections + are properly reset. + + We use Google Storage, so that we don't have to worry about + propagation delays. + ''' + + def setup_method(self, method): + super().setup_method(method) + try: + (backend_login, backend_pw, + self.storage_url) = get_remote_test_info('gs-test') + except NoTestSection as exc: + pytest.skip(exc.reason) + + self.backend_login = backend_login + self.backend_passphrase = backend_pw + + self.backend = gs.Backend(self.storage_url, backend_login, backend_pw, {}) + + def test(self): + self.mkfs(max_obj_size=10*1024**2) + self.mount() + + fname1 = os.path.join(self.mnt_dir, 'file1') + fname2 = os.path.join(self.mnt_dir, 'file2') + + # We need lots of data to keep the connection alive + # and reproduce issue 424 + with open(fname1, 'wb') as fh: + with open('/dev/urandom', 'rb') as src: + for _ in range(5): + fh.write(src.read(BUFSIZE)) + s3ql.ctrl.main(['flushcache', self.mnt_dir]) + + with open(fname2, 'w') as fh: + fh.write('Hello, second world') + s3ql.ctrl.main(['flushcache', self.mnt_dir]) + + # Unmount required to avoid reading from kernel cache + self.umount() + + # Modify + (val, meta) = self.backend.fetch('s3ql_data_1') + self.backend.store('s3ql_data_1', val[:500] + b'oops' + val[500:], meta) + + # Try to read + + self.mount() + with pytest.raises(IOError) as exc_info: + with open(fname1, 'rb') as fh: + fh.read() + assert exc_info.value.errno == errno.EIO + self.reg_output(r'^ERROR: Backend returned malformed data for ' + 'block 0 of inode \d+ .+$', count=1) + + # This should still work + with open(fname2, 'rb') as fh: + fh.read() + + # But this should not + with pytest.raises(PermissionError): + open(fname2, 'wb') + self.reg_output(r'^ERROR: Backend returned malformed data for ' + 'block 0 of inode \d+ .+$', count=1) + + # Printed during umount + self.reg_output(r'^WARNING: File system errors encountered, ' + 'marking for fsck.$', count=1) + + +@pytest.mark.usefixtures('pass_reg_output') +class TestNewerMetadata(t4_fuse.TestFuse): + ''' + Make sure that we turn on failsafe mode and don't overwrite + remote metadata if it suddenly becomes newer than local. + ''' + + def test(self): + self.mkfs() + + # Get backend instance + plain_backend = LocalBackend(self.storage_url, None, None) + + # Save metadata + meta = plain_backend['s3ql_metadata'] + + # Mount file system + self.mount() + + # Increase sequence number + seq_no = get_seq_no(plain_backend) + plain_backend['s3ql_seq_no_%d' % (seq_no+1)] = b'Empty' + + # Create a file, so that there's metadata to flush + fname = os.path.join(self.mnt_dir, 'file1') + with open(fname, 'w') as fh: + fh.write('hello, world') + + # Try to upload metadata + s3ql.ctrl.main(['upload-meta', self.mnt_dir]) + + # Try to write. We repeat a few times, since the metadata upload + # happens asynchronously. + with pytest.raises(PermissionError): + for _ in range(10): + with open(fname + 'barz', 'w') as fh: + fh.write('foobar') + time.sleep(1) + self.reg_output(r'^ERROR: Remote metadata is newer than local ' + '\(\d+ vs \d+\), refusing to overwrite(?: and switching ' + 'to failsafe mode)?!$', count=2) + self.reg_output(r'^WARNING: File system errors encountered, marking for ' + 'fsck\.$', count=1) + self.reg_output(r'^ERROR: The locally cached metadata will be ' + '\*lost\* the next .+$', count=1) + self.umount() + + # Assert that remote metadata has not been overwritten + assert meta == plain_backend['s3ql_metadata'] + + plain_backend.close() diff --git a/tests/t5_fsck.py b/tests/t5_fsck.py new file mode 100755 index 0000000..3d54754 --- /dev/null +++ b/tests/t5_fsck.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +''' +t5_fsck.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +from s3ql.common import get_backend_cachedir +from s3ql.database import Connection +from common import populate_dir, skip_without_rsync +import shutil +import subprocess +from subprocess import check_output, CalledProcessError +import t4_fuse +import tempfile + +class TestFsck(t4_fuse.TestFuse): + + def test(self): + skip_without_rsync() + ref_dir = tempfile.mkdtemp(prefix='s3ql-ref-') + try: + populate_dir(ref_dir) + + # Make file system and fake high inode number + self.mkfs() + db = Connection(get_backend_cachedir(self.storage_url, self.cache_dir) + '.db') + db.execute('UPDATE sqlite_sequence SET seq=? WHERE name=?', + (2 ** 31 + 10, 'inodes')) + db.close() + + # Copy source data + self.mount() + subprocess.check_call(['rsync', '-aHAX', ref_dir + '/', + self.mnt_dir + '/']) + self.umount() + + # Check that inode watermark is high + db = Connection(get_backend_cachedir(self.storage_url, self.cache_dir) + '.db') + assert db.get_val('SELECT seq FROM sqlite_sequence WHERE name=?', + ('inodes',)) > 2 ** 31 + 10 + assert db.get_val('SELECT MAX(id) FROM inodes') > 2 ** 31 + 10 + db.close() + + # Renumber inodes + self.fsck() + + # Check if renumbering was done + db = Connection(get_backend_cachedir(self.storage_url, self.cache_dir) + '.db') + assert db.get_val('SELECT seq FROM sqlite_sequence WHERE name=?', + ('inodes',)) < 2 ** 31 + assert db.get_val('SELECT MAX(id) FROM inodes') < 2 ** 31 + db.close() + + # Compare + self.mount() + try: + out = check_output(['rsync', '-anciHAX', '--delete', '--exclude', '/lost+found', + ref_dir + '/', self.mnt_dir + '/'], universal_newlines=True, + stderr=subprocess.STDOUT) + except CalledProcessError as exc: + pytest.fail('rsync failed with ' + exc.output) + if out: + pytest.fail('Copy not equal to original, rsync says:\n' + out) + + self.umount() + finally: + shutil.rmtree(ref_dir) diff --git a/tests/t5_full.py b/tests/t5_full.py new file mode 100755 index 0000000..e3011e2 --- /dev/null +++ b/tests/t5_full.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +''' +t5_full.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +from common import populate_dir, skip_without_rsync, get_remote_test_info, NoTestSection +from s3ql import backends +import shutil +import subprocess +from subprocess import check_output, CalledProcessError +import t4_fuse +import tempfile +import pytest + +class TestFull(t4_fuse.TestFuse): + + def populate_dir(self, path): + populate_dir(path) + + def test(self): + skip_without_rsync() + + ref_dir = tempfile.mkdtemp(prefix='s3ql-ref-') + try: + self.populate_dir(ref_dir) + + # Copy source data + self.mkfs() + self.mount() + subprocess.check_call(['rsync', '-aHAX', ref_dir + '/', + self.mnt_dir + '/']) + self.umount() + self.fsck() + + # Delete cache, run fsck and compare + shutil.rmtree(self.cache_dir) + self.cache_dir = tempfile.mkdtemp('s3ql-cache-') + self.fsck() + self.mount() + try: + out = check_output(['rsync', '-anciHAX', '--delete', '--exclude', '/lost+found', + ref_dir + '/', self.mnt_dir + '/'], universal_newlines=True, + stderr=subprocess.STDOUT) + except CalledProcessError as exc: + pytest.fail('rsync failed with ' + exc.output) + if out: + pytest.fail('Copy not equal to original, rsync says:\n' + out) + + self.umount() + + # Delete cache and mount + shutil.rmtree(self.cache_dir) + self.cache_dir = tempfile.mkdtemp(prefix='s3ql-cache-') + self.mount() + self.umount() + + finally: + shutil.rmtree(ref_dir) + + +class RemoteTest: + def setup_method(self, method, name): + super().setup_method(method) + try: + (backend_login, backend_pw, + self.storage_url) = get_remote_test_info(name) + except NoTestSection as exc: + pytest.skip(exc.reason) + self.backend_login = backend_login + self.backend_passphrase = backend_pw + + def populate_dir(self, path): + populate_dir(path, entries=50, size=5*1024*1024) + + def teardown_method(self, method): + super().teardown_method(method) + + proc = subprocess.Popen(self.s3ql_cmd_argv('s3qladm') + + [ '--quiet', '--authfile', '/dev/null', + 'clear', self.storage_url ], + stdin=subprocess.PIPE, universal_newlines=True) + if self.backend_login is not None: + print(self.backend_login, file=proc.stdin) + print(self.backend_passphrase, file=proc.stdin) + print('yes', file=proc.stdin) + proc.stdin.close() + + assert proc.wait() == 0 + + +# Dynamically generate tests for other backends +for backend_name in backends.prefix_map: + if backend_name == 'local': + continue + def setup_method(self, method, backend_name=backend_name): + RemoteTest.setup_method(self, method, backend_name + '-test') + test_class_name = 'TestFull' + backend_name + globals()[test_class_name] = type(test_class_name, + (RemoteTest, TestFull), + { 'setup_method': setup_method }) diff --git a/tests/t5_lock_rm.py b/tests/t5_lock_rm.py new file mode 100755 index 0000000..eabc3c1 --- /dev/null +++ b/tests/t5_lock_rm.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +''' +t5_lock_rm.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +import llfuse +import os.path +import s3ql.lock +import s3ql.remove +import sys +from pytest import raises as assert_raises +import pytest +import t4_fuse + +class TestLockRemove(t4_fuse.TestFuse): + + def test(self): + self.mkfs() + self.mount() + self.tst_lock_rm() + self.umount() + self.fsck() + + def tst_lock_rm(self): + + # Extract tar + tempdir = os.path.join(self.mnt_dir, 'lock_dir') + filename = os.path.join(tempdir, 'myfile') + os.mkdir(tempdir) + with open(filename, 'w') as fh: + fh.write('Hello, world') + + # copy + try: + s3ql.lock.main([tempdir]) + except: + sys.excepthook(*sys.exc_info()) + pytest.fail("s3qllock raised exception") + + # Try to delete + assert_raises(PermissionError, os.unlink, filename) + + # Try to write + with pytest.raises(PermissionError): + open(filename, 'w+').write('Hello') + + # delete properly + try: + s3ql.remove.main([tempdir]) + except: + sys.excepthook(*sys.exc_info()) + pytest.fail("s3qlrm raised exception") + + assert 'lock_dir' not in llfuse.listdir(self.mnt_dir) diff --git a/tests/t6_upgrade.py b/tests/t6_upgrade.py new file mode 100755 index 0000000..62074d1 --- /dev/null +++ b/tests/t6_upgrade.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +''' +t6_upgrade.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +from common import populate_dir, skip_without_rsync, retry +from t1_backends import get_remote_test_info, NoTestSection +from s3ql import backends +import shutil +import subprocess +from subprocess import check_output, CalledProcessError +import t4_fuse +import tempfile +import os +import pytest + +@pytest.mark.usefixtures('pass_reg_output') +class TestUpgrade(t4_fuse.TestFuse): + + def setup_method(self, method): + skip_without_rsync() + + basedir_old = os.path.abspath(os.path.join(os.path.dirname(__file__), + '..', 's3ql.old')) + if not os.path.exists(os.path.join(basedir_old, 'bin', 'mkfs.s3ql')): + pytest.skip('no previous S3QL version found') + + super().setup_method(method) + self.ref_dir = tempfile.mkdtemp(prefix='s3ql-ref-') + self.bak_dir = tempfile.mkdtemp(prefix='s3ql-bak-') + self.basedir_old = basedir_old + + def teardown_method(self, method): + super().teardown_method(method) + shutil.rmtree(self.ref_dir) + shutil.rmtree(self.bak_dir) + + def mkfs_old(self, force=False, max_obj_size=500): + argv = [ os.path.join(self.basedir_old, 'bin', 'mkfs.s3ql'), + '-L', 'test fs', '--max-obj-size', str(max_obj_size), + '--cachedir', self.cache_dir, '--quiet', + '--authfile', '/dev/null', self.storage_url ] + if force: + argv.append('--force') + if self.passphrase is None: + argv.append('--plain') + proc = subprocess.Popen(argv, stdin=subprocess.PIPE, universal_newlines=True) + + if self.backend_login is not None: + print(self.backend_login, file=proc.stdin) + print(self.backend_passphrase, file=proc.stdin) + if self.passphrase is not None: + print(self.passphrase, file=proc.stdin) + print(self.passphrase, file=proc.stdin) + proc.stdin.close() + assert proc.wait() == 0 + self.reg_output(r'^Warning: maximum object sizes less than 1 MiB ' + 'will seriously degrade performance\.$', count=1) + + def mount_old(self): + self.mount_process = subprocess.Popen([os.path.join(self.basedir_old, 'bin', 'mount.s3ql'), + "--fg", '--cachedir', self.cache_dir, '--log', + 'none', '--quiet', '--authfile', '/dev/null', + '--compress', 'zlib', self.storage_url, self.mnt_dir], + stdin=subprocess.PIPE, universal_newlines=True) + if self.backend_login is not None: + print(self.backend_login, file=self.mount_process.stdin) + print(self.backend_passphrase, file=self.mount_process.stdin) + if self.passphrase is not None: + print(self.passphrase, file=self.mount_process.stdin) + self.mount_process.stdin.close() + def poll(): + if os.path.ismount(self.mnt_dir): + return True + assert self.mount_process.poll() is None + retry(30, poll) + + def umount_old(self): + with open('/dev/null', 'wb') as devnull: + retry(5, lambda: subprocess.call(['fuser', '-m', self.mnt_dir], + stdout=devnull, stderr=devnull) == 1) + + proc = subprocess.Popen([os.path.join(self.basedir_old, 'bin', 'umount.s3ql'), + '--quiet', self.mnt_dir]) + retry(90, lambda : proc.poll() is not None) + assert proc.wait() == 0 + + assert self.mount_process.poll() == 0 + assert not os.path.ismount(self.mnt_dir) + + def upgrade(self): + proc = subprocess.Popen(self.s3ql_cmd_argv('s3qladm') + + [ '--cachedir', self.cache_dir, '--authfile', + '/dev/null', '--quiet', 'upgrade', self.storage_url ], + stdin=subprocess.PIPE, universal_newlines=True) + + if self.backend_login is not None: + print(self.backend_login, file=proc.stdin) + print(self.backend_passphrase, file=proc.stdin) + if self.passphrase is not None: + print(self.passphrase, file=proc.stdin) + print('yes', file=proc.stdin) + proc.stdin.close() + + assert proc.wait() == 0 + + def compare(self): + try: + out = check_output(['rsync', '-anciHAX', '--delete', '--exclude', '/lost+found', + self.ref_dir + '/', self.mnt_dir + '/'], universal_newlines=True, + stderr=subprocess.STDOUT) + except CalledProcessError as exc: + pytest.fail('rsync failed with ' + exc.output) + if out: + pytest.fail('Copy not equal to original, rsync says:\n' + out) + + def populate(self): + populate_dir(self.ref_dir) + + @pytest.mark.parametrize("with_cache", (True, False)) + def test(self, with_cache): + self.populate() + + # Create and mount using previous S3QL version + self.mkfs_old() + self.mount_old() + subprocess.check_call(['rsync', '-aHAX', self.ref_dir + '/', self.mnt_dir + '/']) + self.umount_old() + + # Try to access with new version (should fail) + if not with_cache: + shutil.rmtree(self.cache_dir) + self.cache_dir = tempfile.mkdtemp(prefix='s3ql-cache-') + self.mount(expect_fail=32) + self.reg_output(r'^ERROR: File system revision too old, please ' + 'run `s3qladm upgrade` first\.$', count=1) + + # Upgrade + if not with_cache: + shutil.rmtree(self.cache_dir) + self.cache_dir = tempfile.mkdtemp(prefix='s3ql-cache-') + self.upgrade() + + # ...and test + if not with_cache: + shutil.rmtree(self.cache_dir) + self.cache_dir = tempfile.mkdtemp(prefix='s3ql-cache-') + self.fsck() + self.mount() + self.compare() + + # Try if we can still write (we messed this up in the upgrade + # from 2.16 to 2.17). + with open('%s/some_new_file' % (self.mnt_dir,), 'w') as fh: + fh.write('hello, world') + + self.umount() + +class TestPlainUpgrade(TestUpgrade): + def setup_method(self, method): + super().setup_method(method) + self.passphrase = None + +class RemoteUpgradeTest: + def setup_method(self, method, name): + super().setup_method(method) + try: + (backend_login, backend_pw, + self.storage_url) = get_remote_test_info(name) + except NoTestSection as exc: + pytest.skip(exc.reason) + self.backend_login = backend_login + self.backend_passphrase = backend_pw + + def populate(self): + populate_dir(self.ref_dir, entries=50, size=5*1024*1024) + + def teardown_method(self, method): + super().teardown_method(method) + + proc = subprocess.Popen(self.s3ql_cmd_argv('s3qladm') + + [ '--quiet', '--authfile', '/dev/null', + 'clear', self.storage_url ], + stdin=subprocess.PIPE, universal_newlines=True) + if self.backend_login is not None: + print(self.backend_login, file=proc.stdin) + print(self.backend_passphrase, file=proc.stdin) + print('yes', file=proc.stdin) + proc.stdin.close() + + assert proc.wait() == 0 + +# Dynamically generate tests for other backends +for backend_name in backends.prefix_map: + if backend_name == 'local': + continue + def setup_method(self, method, backend_name=backend_name): + RemoteUpgradeTest.setup_method(self, method, backend_name + '-test') + test_class_name = 'Test' + backend_name + 'Upgrade' + globals()[test_class_name] = type(test_class_name, + (RemoteUpgradeTest, TestUpgrade), + { 'setup_method': setup_method }) diff --git a/util/cmdline_lexer.py b/util/cmdline_lexer.py new file mode 100644 index 0000000..419c1ef --- /dev/null +++ b/util/cmdline_lexer.py @@ -0,0 +1,31 @@ +''' +cmdline_lexer.py - this file is part of S3QL. + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from pygments.token import Comment, Name, Generic, Literal +from pygments.lexer import RegexLexer + +__all__ = [ 'CommandLineLexer' ] + +class CommandLineLexer(RegexLexer): + """ + A lexer that highlights a command line with variable parts + """ + + name = 'CommandLine' + aliases = ['commandline'] + mimetypes = [] + + tokens = { + 'root': [ + (r'#.*\n', Comment), + (r'[^[<]+', Literal), + (r'\[[^[\]]+\]', Generic.Emph), + (r'<[^>]+>', Name.Variable), + ], + + } diff --git a/util/sphinx_pipeinclude.py b/util/sphinx_pipeinclude.py new file mode 100644 index 0000000..4253641 --- /dev/null +++ b/util/sphinx_pipeinclude.py @@ -0,0 +1,54 @@ +''' +sphinx_pipe.py - this file is part of S3QL. + +Implements a Sphinx extension that provides a `pipeinclude` directive +to include the output of a program. + + +Copyright © 2008 Nikolaus Rath + +This work can be distributed under the terms of the GNU GPLv3. +''' + +from docutils.parsers.rst.directives.misc import Include +import subprocess +import shlex +from docutils.utils.error_reporting import SafeString +import tempfile +import os.path +import sys + +class PipeInclude(Include): + """ + Include program output as ReST source. + """ + + def run(self): + # To maximize code reuse, we just write the output in a temporary + # file and call the base class. Otherwise we'd have to copy & paste + # all the code to handle start-line, end-line etc options. + + source = self.state_machine.input_lines.source( + self.lineno - self.state_machine.input_offset - 1) + source_dir = os.path.dirname(os.path.abspath(source)) + + command = self.arguments[0] + command_list = shlex.split(command) + + if command_list[0] == 'python': + command_list[0] = sys.executable + + with tempfile.NamedTemporaryFile() as fh: + exitcode = subprocess.call(command_list, stdout=fh, + cwd=source_dir) + if exitcode != 0: + raise self.severe('Problems with "%s" directive:\n' + 'Command %s returned with exit code %d' % + (self.name, SafeString(command), exitcode)) + + fh.flush() + self.arguments[0] = fh.name + return super().run() + +def setup(app): + app.add_directive('pipeinclude', PipeInclude) -- cgit v1.2.3 From e7541d5f28330b0c85dc082d818a0f741f5279cb Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Skip tests requiring /proc Origin: debian Forwarded: no Patch-Name: proc_mount.diff Gbp-Pq: Name proc_mount.diff --- tests/t4_adm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/t4_adm.py b/tests/t4_adm.py index a17267e..36e7f6a 100755 --- a/tests/t4_adm.py +++ b/tests/t4_adm.py @@ -19,6 +19,7 @@ import tempfile import unittest import subprocess import pytest +import os @pytest.mark.usefixtures('s3ql_cmd_argv', 'pass_reg_output') class AdmTests(unittest.TestCase): @@ -49,6 +50,7 @@ class AdmTests(unittest.TestCase): self.reg_output(r'^WARNING: Maximum object sizes less than ' '1 MiB will degrade performance\.$', count=1) + @unittest.skipUnless(os.path.exists('/proc/mounts'), '/proc/mounts not available') def test_passphrase(self): self.mkfs() -- cgit v1.2.3 From 31bf6ca63ccb06e264475729e3642d3d8c30cea7 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Estimate system clock granularity more conservatively Origin: debian Forwarded: no Patch-Name: clock-granularity.diff Some unit tests check that file access time stamps are updated correctly. For such a test to succeed, it's necessarily that the time between two subsequent test-accesses can actually be resolved. The heuristic to determine this granularity doesn't seem to work well on e.g. the i386 buildd, so we make it much more conservative. This means the test suite runs longer, but otherwise has no ill effects. Gbp-Pq: Name clock-granularity.diff --- tests/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common.py b/tests/common.py index 9373654..af5ed3c 100644 --- a/tests/common.py +++ b/tests/common.py @@ -31,7 +31,7 @@ def get_clock_granularity(): stamp2 = time.time() resolution = min(resolution, stamp2 - stamp1) time.sleep(0.01) - return resolution + return max(1, 10 * resolution) CLOCK_GRANULARITY = get_clock_granularity() # When testing, we want to make sure that we don't sleep for too short a time -- cgit v1.2.3 From 66b0572889926c1e02663340e218c079e07ab0a6 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Ignore compiler warnings due to old Cython version Origin: debian Forwarded: not-needed Patch-Name: ignore_cython_warnings.diff Cython versions prior to 0.24 generate Gcc compiler warnings. Since Cython 0.24 isn't in Debian yet, we ignore these warnings. Gbp-Pq: Name ignore_cython_warnings.diff --- setup.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/setup.py b/setup.py index ae1c00a..e2c9a21 100755 --- a/setup.py +++ b/setup.py @@ -111,14 +111,6 @@ def main(): compile_args = ['-Wall', '-Wextra', '-Wconversion', '-Wsign-compare'] - # Value-changing conversions should always be explicit. - compile_args.append('-Werror=conversion') - - # Note that (i > -1) is false if i is unsigned (-1 will be converted to - # a large positive value). We certainly don't want to do this by - # accident. - compile_args.append('-Werror=sign-compare') - # Enable all fatal warnings only when compiling from Mercurial tip. # (otherwise we break forward compatibility because compilation with newer # compiler may fail if additional warnings are added) -- cgit v1.2.3 From 2e667ee91eeb0425f66096ed832b7f4ecd63ed18 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Allow upgrade of file systems created with jessie's s3ql Origin: debian Forwarded: not-needed Patch-Name: support_jessie_upgrade.diff Upstream has dropped support for upgrading file systems created with the S3QL version in jessie. This patch forward-ports this capability. It is based on upstream's Mercurial commit 773931c43368. Gbp-Pq: Name support_jessie_upgrade.diff --- src/s3ql/adm.py | 205 +++++++++++++++++++++-- src/s3ql/backends/comprenc.py | 371 ++++++++++++++++++++++++++++++++++++++---- src/s3ql/backends/local.py | 12 +- src/s3ql/backends/s3c.py | 29 +++- src/s3ql/upgrade_support.py | 75 +++++++++ 5 files changed, 647 insertions(+), 45 deletions(-) create mode 100644 src/s3ql/upgrade_support.py diff --git a/src/s3ql/adm.py b/src/s3ql/adm.py index d0e6757..3b5d980 100644 --- a/src/s3ql/adm.py +++ b/src/s3ql/adm.py @@ -7,19 +7,23 @@ This work can be distributed under the terms of the GNU GPLv3. ''' from .logging import logging, QuietError, setup_logging -from . import CURRENT_FS_REV, REV_VER_MAP +from . import CURRENT_FS_REV, REV_VER_MAP, BUFSIZE from .backends.comprenc import ComprencBackend from .database import Connection from .deltadump import TIME, INTEGER from .common import (get_backend_cachedir, get_seq_no, is_mounted, get_backend, - load_params, save_params) + load_params, save_params, AsyncFn, get_backend_factory, + pretty_print_size, split_by_n, handle_on_return) from .metadata import dump_and_upload_metadata, download_metadata from . import metadata from .parse_args import ArgumentParser from datetime import datetime as Datetime from getpass import getpass from contextlib import contextmanager +from base64 import b64encode +from queue import Queue, Full as QueueFull import os +import tempfile import shutil import functools import sys @@ -86,12 +90,11 @@ def main(args=None): if options.action == 'clear': with get_backend(options, raw=True) as backend: return clear(backend, options) + elif options.action == 'upgrade': + return upgrade(options) with get_backend(options) as backend: - if options.action == 'upgrade': - return upgrade(backend, get_backend_cachedir(options.storage_url, - options.cachedir)) - elif options.action == 'passphrase': + if options.action == 'passphrase': return change_passphrase(backend) elif options.action == 'download-metadata': @@ -206,11 +209,24 @@ def get_old_rev_msg(rev, prog): ''' % { 'version': REV_VER_MAP[rev], 'prog': prog }) -def upgrade(backend, cachepath): + +@handle_on_return +def upgrade(options, on_return): '''Upgrade file system to newest revision''' log.info('Getting file system parameters..') + from . import backends + backends.local.SUPPORT_LEGACY_FORMAT = True + backends.s3c.SUPPORT_LEGACY_FORMAT = True + backends.comprenc.SUPPORT_LEGACY_FORMAT = True + + cachepath = get_backend_cachedir(options.storage_url, options.cachedir) + + backend_factory = get_backend_factory(options.storage_url, options.backend_options, + options.authfile) + backend = on_return.enter_context(backend_factory()) + # Check for cached metadata db = None seq_no = get_seq_no(backend) @@ -249,7 +265,7 @@ def upgrade(backend, cachepath): raise QuietError() # Check revision - if param['revision'] < CURRENT_FS_REV-1: + if param['revision'] < 21: print(textwrap.dedent(''' File system revision too old to upgrade! @@ -285,13 +301,31 @@ def upgrade(backend, cachepath): with monkeypatch_metadata_retrieval(): db = download_metadata(backend, cachepath + '.db') - log.info('Upgrading from revision %d to %d...', param['revision'], CURRENT_FS_REV) - - param['revision'] = CURRENT_FS_REV param['last-modified'] = time.time() param['seq_no'] += 1 + + if param['revision'] == 21: + log.info('Upgrading from revision %d to %d...', 21, 22) + param['revision'] = 22 + + # Ensure that there are backups of the master key + if backend.passphrase is not None: + data_pw = backend.passphrase + backend.passphrase = backend.fs_passphrase + for i in range(1,4): + obj_id = 's3ql_passphrase_bak%d' % i + if obj_id not in backend: + backend[obj_id] = data_pw + backend.passphrase = data_pw + + # Upgrade to revision 22 + update_obj_metadata(backend, backend_factory, db, options.threads) + updated_from_21 = True + else: + updated_from_21 = False - # Upgrade + log.info('Upgrading from revision %d to %d...', 22, 23) + param['revision'] = 23 for name in ('atime', 'mtime', 'ctime'): db.execute('ALTER TABLE inodes ADD COLUMN {time}_ns ' 'INT NOT NULL DEFAULT 0'.format(time=name)) @@ -308,6 +342,14 @@ def upgrade(backend, cachepath): print('File system upgrade complete.') + if updated_from_21 and backend.passphrase is not None: + print('\nPlease store the following master key in a safe location. It allows ', + 'decryption of the S3QL file system in case the storage objects holding ', + 'this information get corrupted:', + '---BEGIN MASTER KEY---', + ' '.join(split_by_n(b64encode(backend.passphrase).decode(), 4)), + '---END MASTER KEY---', + sep='\n') @contextmanager def monkeypatch_metadata_retrieval(): @@ -351,5 +393,144 @@ def monkeypatch_metadata_retrieval(): metadata.DUMP_SPEC[2] = DUMP_SPEC_bak metadata.create_tables = create_tables_bak +def update_obj_metadata(backend, backend_factory, db, thread_count): + '''Upgrade metadata of storage objects''' + + plain_backend = backend.backend + + # No need to update sequence number, since we are going to + # write out a new one after the upgrade. + if backend.passphrase is None: + extra_objects = { 's3ql_metadata' } + else: + extra_objects = { 's3ql_metadata', + 's3ql_passphrase', 's3ql_passphrase_bak1', + 's3ql_passphrase_bak2', 's3ql_passphrase_bak3' } + + for i in range(30): + obj_id = 's3ql_metadata_bak_%d' % i + if obj_id in plain_backend: + extra_objects.add(obj_id) + + def yield_objects(): + for (id_,) in db.query('SELECT id FROM objects'): + yield 's3ql_data_%d' % id_ + for obj_id in extra_objects: + yield obj_id + total = db.get_val('SELECT COUNT(id) FROM objects') + len(extra_objects) + + queue = Queue(maxsize=thread_count) + threads = [] + for _ in range(thread_count): + t = AsyncFn(upgrade_loop, queue, backend_factory) + # Don't wait for worker threads, gives deadlock if main thread + # terminates with exception + t.daemon = True + t.start() + threads.append(t) + + # Updating this value is prone to race conditions. However, + # we don't care because this is for an approximate progress + # output only. + queue.rewrote_size = 0 + stamp = 0 + for (i, obj_id) in enumerate(yield_objects()): + stamp2 = time.time() + if stamp2 - stamp > 1: + sys.stdout.write('\r..processed %d/%d objects (%.1f%%, %s rewritten)..' + % (i, total, i/total*100, + pretty_print_size(queue.rewrote_size))) + sys.stdout.flush() + stamp = stamp2 + + # Terminate early if any thread failed with an exception + for t in threads: + if not t.is_alive(): + t.join_and_raise() + + # Avoid blocking if all threads terminated + while True: + try: + queue.put(obj_id, timeout=1) + except QueueFull: + pass + else: + break + for t in threads: + if not t.is_alive(): + t.join_and_raise() + + queue.maxsize += len(threads) + for t in threads: + queue.put(None) + + for t in threads: + t.join_and_raise() + + sys.stdout.write('\n') + +def upgrade_loop(queue, backend_factory): + + with backend_factory() as backend: + plain_backend = backend.backend + while True: + obj_id = queue.get() + if obj_id is None: + break + + meta = plain_backend.lookup(obj_id) + if meta.get('format_version', 0) == 2: + continue + + # For important objects, we make a copy first (just to be safe) + if not obj_id.startswith('s3ql_data'): + plain_backend.copy(obj_id, 's3ql_pre2.13' + obj_id[4:]) + + # When reading passphrase objects, we have to use the + # "outer" password + if obj_id.startswith('s3ql_passphrase'): + data_pw = backend.passphrase + backend.passphrase = backend.fs_passphrase + + meta = backend._convert_legacy_metadata(meta) + if meta['encryption'] == 'AES': + # Two statements to reduce likelihood of update races + size = rewrite_legacy_object(backend, obj_id) + queue.rewrote_size += size + else: + plain_backend.update_meta(obj_id, meta) + + if obj_id.startswith('s3ql_passphrase'): + backend.passphrase = data_pw + +def rewrite_legacy_object(backend, obj_id): + with tempfile.TemporaryFile() as tmpfh: + + # Read object + def do_read(fh): + tmpfh.seek(0) + tmpfh.truncate() + while True: + buf = fh.read(BUFSIZE) + if not buf: + break + tmpfh.write(buf) + return fh.metadata + + meta = backend.perform_read(do_read, obj_id) + + # Write object + def do_write(fh): + tmpfh.seek(0) + while True: + buf = tmpfh.read(BUFSIZE) + if not buf: + break + fh.write(buf) + return fh + out_fh = backend.perform_write(do_write, obj_id, meta) + + return out_fh.get_obj_size() + if __name__ == '__main__': main(sys.argv[1:]) diff --git a/src/s3ql/backends/comprenc.py b/src/s3ql/backends/comprenc.py index 58e90a6..285f56a 100644 --- a/src/s3ql/backends/comprenc.py +++ b/src/s3ql/backends/comprenc.py @@ -25,6 +25,11 @@ import zlib log = logging.getLogger(__name__) +from ..upgrade_support import safe_unpickle, pickle +from base64 import b64decode, b64encode +import binascii +SUPPORT_LEGACY_FORMAT=False + HMAC_SIZE = 32 def sha256(s): @@ -67,6 +72,8 @@ class ComprencBackend(AbstractBackend, metaclass=ABCDocstMeta): @copy_ancestor_docstring def lookup(self, key): meta_raw = self.backend.lookup(key) + if SUPPORT_LEGACY_FORMAT and meta_raw.get('format_version', 0) < 2: + meta_raw = self._convert_legacy_metadata(meta_raw) return self._verify_meta(key, meta_raw)[1] @prepend_ancestor_docstring @@ -147,43 +154,80 @@ class ComprencBackend(AbstractBackend, metaclass=ABCDocstMeta): """ fh = self.backend.open_read(key) + checksum_warning = False try: - meta_raw = fh.metadata - (nonce, meta) = self._verify_meta(key, meta_raw) - if nonce: - data_key = sha256(self.passphrase + nonce) - - # The `payload_offset` key only exists if the storage object was - # created with on old S3QL version. In order to avoid having to - # download and re-upload the entire object during the upgrade, the - # upgrade procedure adds this header to tell us how many bytes at - # the beginning of the object we have to skip to get to the payload. - if 'payload_offset' in meta_raw: - to_skip = meta_raw['payload_offset'] - while to_skip: - to_skip -= len(fh.read(to_skip)) - - encr_alg = meta_raw['encryption'] - if encr_alg == 'AES_v2': - fh = DecryptFilter(fh, data_key) - elif encr_alg != 'None': - raise RuntimeError('Unsupported encryption: %s' % encr_alg) - - compr_alg = meta_raw['compression'] - if compr_alg == 'BZIP2': - fh = DecompressFilter(fh, bz2.BZ2Decompressor()) - elif compr_alg == 'LZMA': - fh = DecompressFilter(fh, lzma.LZMADecompressor()) - elif compr_alg == 'ZLIB': - fh = DecompressFilter(fh,zlib.decompressobj()) - elif compr_alg != 'None': - raise RuntimeError('Unsupported compression: %s' % compr_alg) + if SUPPORT_LEGACY_FORMAT: + if fh.metadata.get('format_version', 0) < 2: + meta_raw = self._convert_legacy_metadata(fh.metadata) + else: + meta_raw = fh.metadata + (nonce, meta) = self._verify_meta(key, meta_raw) + if nonce: + data_key = sha256(self.passphrase + nonce) + compr_alg = meta_raw['compression'] + encr_alg = meta_raw['encryption'] + if compr_alg == 'BZIP2': + decompressor = bz2.BZ2Decompressor() + elif compr_alg == 'LZMA': + decompressor = lzma.LZMADecompressor() + elif compr_alg == 'ZLIB': + decompressor = zlib.decompressobj() + elif compr_alg == 'None': + decompressor = None + else: + raise RuntimeError('Unsupported compression: %s' % compr_alg) + if 'payload_offset' in meta_raw: + to_skip = meta_raw['payload_offset'] + while to_skip: + to_skip -= len(fh.read(to_skip)) + checksum_warning = True + if encr_alg == 'AES': + fh = LegacyDecryptDecompressFilter(fh, data_key, decompressor) + decompressor = None + elif encr_alg == 'AES_v2': + fh = DecryptFilter(fh, data_key) + elif encr_alg != 'None': + raise RuntimeError('Unsupported encryption: %s' % encr_alg) + if decompressor: + fh = DecompressFilter(fh, decompressor) + else: + meta_raw = fh.metadata + + (nonce, meta) = self._verify_meta(key, meta_raw) + if nonce: + data_key = sha256(self.passphrase + nonce) + + # The `payload_offset` key only exists if the storage object was + # created with on old S3QL version. In order to avoid having to + # download and re-upload the entire object during the upgrade, the + # upgrade procedure adds this header to tell us how many bytes at + # the beginning of the object we have to skip to get to the payload. + if 'payload_offset' in meta_raw: + to_skip = meta_raw['payload_offset'] + while to_skip: + to_skip -= len(fh.read(to_skip)) + + encr_alg = meta_raw['encryption'] + if encr_alg == 'AES_v2': + fh = DecryptFilter(fh, data_key) + elif encr_alg != 'None': + raise RuntimeError('Unsupported encryption: %s' % encr_alg) + + compr_alg = meta_raw['compression'] + if compr_alg == 'BZIP2': + fh = DecompressFilter(fh, bz2.BZ2Decompressor()) + elif compr_alg == 'LZMA': + fh = DecompressFilter(fh, lzma.LZMADecompressor()) + elif compr_alg == 'ZLIB': + fh = DecompressFilter(fh,zlib.decompressobj()) + elif compr_alg != 'None': + raise RuntimeError('Unsupported compression: %s' % compr_alg) fh.metadata = meta except: # Don't emit checksum warning, caller hasn't even # started reading anything. - fh.close(checksum_warning=False) + fh.close(checksum_warning=checksum_warning) raise return fh @@ -275,6 +319,8 @@ class ComprencBackend(AbstractBackend, metaclass=ABCDocstMeta): def _copy_or_rename(self, src, dest, rename, metadata=None): meta_raw = self.backend.lookup(src) + if SUPPORT_LEGACY_FORMAT and meta_raw.get('format_version', 0) < 2: + meta_raw = self._convert_legacy_metadata(meta_raw) (nonce, meta_old) = self._verify_meta(src, meta_raw) if nonce: @@ -303,6 +349,165 @@ class ComprencBackend(AbstractBackend, metaclass=ABCDocstMeta): def close(self): self.backend.close() + def _convert_legacy_metadata(self, meta): + '''Convert metadata to newest format + + This method ensures that we can read objects written + by older S3QL versions. + ''' + + format_version = meta.get('format_version', 0) + assert format_version in (0,1) + if format_version == 0: + meta = self._convert_legacy_metadata0(meta) + return self._convert_legacy_metadata1(meta) + + def _convert_legacy_metadata0(self, meta, + LEN_BYTES = struct.calcsize(b' 2 or opcode.name not in SAFE_UNPICKLE_OPCODES: + raise pickle.UnpicklingError('opcode %s is unsafe' % opcode.name) + except (ValueError, EOFError): + raise pickle.UnpicklingError('corrupted data') + + fh.seek(pos) + + # Then use a custom Unpickler to ensure that we only give access to + # specific, whitelisted globals. Note that with the above opcodes, there is + # no way to trigger attribute access, so "brachiating" from a white listed + # object to __builtins__ is not possible. + return SafeUnpickler(fh, fix_imports=fix_imports, + encoding=encoding, errors=errors).load() + +def safe_unpickle(buf, fix_imports=True, encoding="ASCII", + errors="strict"): + '''Safely unpickle untrusted data in *buf*''' + + return safe_unpickle_fh(io.BytesIO(buf), fix_imports=fix_imports, + encoding=encoding, errors=errors) -- cgit v1.2.3 From d601afc326245703745a29ae579399a08d907461 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Mon, 27 Jun 2016 11:25:34 -0700 Subject: Don't hide pdflatex output. Origin: debian Forwarded: not-needed Patch-Name: show_pdflatex_output.diff Upstream is not interested in this patch. Gbp-Pq: Name show_pdflatex_output.diff --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e2c9a21..375c76c 100755 --- a/setup.py +++ b/setup.py @@ -97,9 +97,8 @@ class build_docs(setuptools.Command): print('Running pdflatex...') for _ in range(3): - with open('/dev/null', 'wb') as null: - subprocess.check_call(['pdflatex', '-interaction', 'batchmode', 'manual.tex'], - cwd=os.path.join(dest_dir, 'latex'), stdout=null) + subprocess.check_call(['pdflatex', '-interaction', 'nonstopmode', 'manual.tex'], + cwd=os.path.join(dest_dir, 'latex')) os.rename(os.path.join(dest_dir, 'latex', 'manual.pdf'), os.path.join(dest_dir, 'manual.pdf')) -- cgit v1.2.3 From b65902f85113ff88330f31188b18ef2fac3dd5bd Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Skip tests requiring /proc Origin: debian Forwarded: no Patch-Name: proc_mount.diff Gbp-Pq: Name proc_mount.diff --- tests/t4_adm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/t4_adm.py b/tests/t4_adm.py index 005c91b..1d8ad7a 100755 --- a/tests/t4_adm.py +++ b/tests/t4_adm.py @@ -22,6 +22,7 @@ import tempfile import unittest import subprocess import pytest +import os @pytest.mark.usefixtures('s3ql_cmd_argv', 'pass_reg_output') class AdmTests(unittest.TestCase): @@ -56,6 +57,7 @@ class AdmTests(unittest.TestCase): return stdout + @unittest.skipUnless(os.path.exists('/proc/mounts'), '/proc/mounts not available') def test_passphrase(self): self.mkfs() -- cgit v1.2.3 From 41a30c3059626f8496962a7d0e203b37a36512d1 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Estimate system clock granularity more conservatively Origin: debian Forwarded: no Patch-Name: clock-granularity.diff Some unit tests check that file access time stamps are updated correctly. For such a test to succeed, it's necessarily that the time between two subsequent test-accesses can actually be resolved. The heuristic to determine this granularity doesn't seem to work well on e.g. the i386 buildd, so we make it much more conservative. This means the test suite runs longer, but otherwise has no ill effects. Gbp-Pq: Name clock-granularity.diff --- tests/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common.py b/tests/common.py index 8c3be59..bea2e5c 100644 --- a/tests/common.py +++ b/tests/common.py @@ -27,7 +27,7 @@ def get_clock_granularity(): stamp2 = time.time() resolution = min(resolution, stamp2 - stamp1) time.sleep(0.01) - return resolution + return max(1, 10 * resolution) CLOCK_GRANULARITY = get_clock_granularity() # When testing, we want to make sure that we don't sleep for too short a time -- cgit v1.2.3 From ebb646ccd7972fb28e7f1c7be9484d81b0dd394d Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Mon, 27 Jun 2016 11:25:34 -0700 Subject: Don't hide pdflatex output. Origin: debian Forwarded: not-needed Patch-Name: show_pdflatex_output.diff Upstream is not interested in this patch. Gbp-Pq: Name show_pdflatex_output.diff --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 93a8e52..3ea858f 100755 --- a/setup.py +++ b/setup.py @@ -98,9 +98,8 @@ class build_docs(setuptools.Command): print('Running pdflatex...') for _ in range(3): - with open('/dev/null', 'wb') as null: - subprocess.check_call(['pdflatex', '-interaction', 'batchmode', 'manual.tex'], - cwd=os.path.join(dest_dir, 'latex'), stdout=null) + subprocess.check_call(['pdflatex', '-interaction', 'nonstopmode', 'manual.tex'], + cwd=os.path.join(dest_dir, 'latex')) os.rename(os.path.join(dest_dir, 'latex', 'manual.pdf'), os.path.join(dest_dir, 'manual.pdf')) -- cgit v1.2.3 From f73ec80849455b7ac7606907714b6d786aa49987 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Skip tests requiring /proc Origin: debian Forwarded: no Patch-Name: proc_mount.diff Gbp-Pq: Name proc_mount.diff --- tests/t4_adm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/t4_adm.py b/tests/t4_adm.py index 005c91b..1d8ad7a 100755 --- a/tests/t4_adm.py +++ b/tests/t4_adm.py @@ -22,6 +22,7 @@ import tempfile import unittest import subprocess import pytest +import os @pytest.mark.usefixtures('s3ql_cmd_argv', 'pass_reg_output') class AdmTests(unittest.TestCase): @@ -56,6 +57,7 @@ class AdmTests(unittest.TestCase): return stdout + @unittest.skipUnless(os.path.exists('/proc/mounts'), '/proc/mounts not available') def test_passphrase(self): self.mkfs() -- cgit v1.2.3 From bb9b22c41eea784b2ffd67b32d5296f7885fe328 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Estimate system clock granularity more conservatively Origin: debian Forwarded: no Patch-Name: clock-granularity.diff Some unit tests check that file access time stamps are updated correctly. For such a test to succeed, it's necessarily that the time between two subsequent test-accesses can actually be resolved. The heuristic to determine this granularity doesn't seem to work well on e.g. the i386 buildd, so we make it much more conservative. This means the test suite runs longer, but otherwise has no ill effects. Gbp-Pq: Name clock-granularity.diff --- tests/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common.py b/tests/common.py index 8c3be59..bea2e5c 100644 --- a/tests/common.py +++ b/tests/common.py @@ -27,7 +27,7 @@ def get_clock_granularity(): stamp2 = time.time() resolution = min(resolution, stamp2 - stamp1) time.sleep(0.01) - return resolution + return max(1, 10 * resolution) CLOCK_GRANULARITY = get_clock_granularity() # When testing, we want to make sure that we don't sleep for too short a time -- cgit v1.2.3 From b7128a05f30b72224e0d61046d045b1221934858 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Mon, 27 Jun 2016 11:25:34 -0700 Subject: Don't hide pdflatex output. Origin: debian Forwarded: not-needed Patch-Name: show_pdflatex_output.diff Upstream is not interested in this patch. Gbp-Pq: Name show_pdflatex_output.diff --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 93a8e52..3ea858f 100755 --- a/setup.py +++ b/setup.py @@ -98,9 +98,8 @@ class build_docs(setuptools.Command): print('Running pdflatex...') for _ in range(3): - with open('/dev/null', 'wb') as null: - subprocess.check_call(['pdflatex', '-interaction', 'batchmode', 'manual.tex'], - cwd=os.path.join(dest_dir, 'latex'), stdout=null) + subprocess.check_call(['pdflatex', '-interaction', 'nonstopmode', 'manual.tex'], + cwd=os.path.join(dest_dir, 'latex')) os.rename(os.path.join(dest_dir, 'latex', 'manual.pdf'), os.path.join(dest_dir, 'manual.pdf')) -- cgit v1.2.3 From 154d1947f2e60588907170f232edc0fcde9ae769 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 6 Nov 2018 19:33:40 +0000 Subject: Don't drop table while referenced by a view. This does not work with the SQLite version in unstable. Gbp-Pq: Name 0004-Don-t-drop-table-while-referenced-by-a-view.patch --- src/s3ql/fsck.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/s3ql/fsck.py b/src/s3ql/fsck.py index a0a227a..b2c4a96 100644 --- a/src/s3ql/fsck.py +++ b/src/s3ql/fsck.py @@ -1315,8 +1315,8 @@ def renumber_inodes(db): create_tables(db) for table in ('names', 'blocks', 'objects'): - db.execute('DROP TABLE %s' % table) - db.execute('ALTER TABLE %s_old RENAME TO %s' % (table, table)) + db.execute('INSERT INTO %s SELECT * FROM %s_old' % (table, table)) + db.execute('DROP TABLE %s_old' % table) log.info('..mapping..') db.execute('CREATE TEMPORARY TABLE inode_map (rowid INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER UNIQUE)') -- cgit v1.2.3 From c2edae1f2556c10dd47b6920c29f8c9005f08820 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Skip tests requiring /proc Origin: debian Forwarded: no Patch-Name: proc_mount.diff Gbp-Pq: Name proc_mount.diff --- tests/t4_adm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/t4_adm.py b/tests/t4_adm.py index 005c91b..1d8ad7a 100755 --- a/tests/t4_adm.py +++ b/tests/t4_adm.py @@ -22,6 +22,7 @@ import tempfile import unittest import subprocess import pytest +import os @pytest.mark.usefixtures('s3ql_cmd_argv', 'pass_reg_output') class AdmTests(unittest.TestCase): @@ -56,6 +57,7 @@ class AdmTests(unittest.TestCase): return stdout + @unittest.skipUnless(os.path.exists('/proc/mounts'), '/proc/mounts not available') def test_passphrase(self): self.mkfs() -- cgit v1.2.3 From 856bceab7c18c595e6fd2fca1bfaa47bf9e6d070 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Estimate system clock granularity more conservatively Origin: debian Forwarded: no Patch-Name: clock-granularity.diff Some unit tests check that file access time stamps are updated correctly. For such a test to succeed, it's necessarily that the time between two subsequent test-accesses can actually be resolved. The heuristic to determine this granularity doesn't seem to work well on e.g. the i386 buildd, so we make it much more conservative. This means the test suite runs longer, but otherwise has no ill effects. Gbp-Pq: Name clock-granularity.diff --- tests/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common.py b/tests/common.py index 8c3be59..bea2e5c 100644 --- a/tests/common.py +++ b/tests/common.py @@ -27,7 +27,7 @@ def get_clock_granularity(): stamp2 = time.time() resolution = min(resolution, stamp2 - stamp1) time.sleep(0.01) - return resolution + return max(1, 10 * resolution) CLOCK_GRANULARITY = get_clock_granularity() # When testing, we want to make sure that we don't sleep for too short a time -- cgit v1.2.3 From f2b1be4cc38a0b508a43cbc115b0bda83128982e Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Mon, 27 Jun 2016 11:25:34 -0700 Subject: Don't hide pdflatex output. Origin: debian Forwarded: not-needed Patch-Name: show_pdflatex_output.diff Upstream is not interested in this patch. Gbp-Pq: Name show_pdflatex_output.diff --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 93a8e52..3ea858f 100755 --- a/setup.py +++ b/setup.py @@ -98,9 +98,8 @@ class build_docs(setuptools.Command): print('Running pdflatex...') for _ in range(3): - with open('/dev/null', 'wb') as null: - subprocess.check_call(['pdflatex', '-interaction', 'batchmode', 'manual.tex'], - cwd=os.path.join(dest_dir, 'latex'), stdout=null) + subprocess.check_call(['pdflatex', '-interaction', 'nonstopmode', 'manual.tex'], + cwd=os.path.join(dest_dir, 'latex')) os.rename(os.path.join(dest_dir, 'latex', 'manual.pdf'), os.path.join(dest_dir, 'manual.pdf')) -- cgit v1.2.3 From aba104337cb5f6da1433026ced9dba1be3f59803 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Skip tests requiring /proc Origin: debian Forwarded: no Patch-Name: proc_mount.diff Gbp-Pq: Name proc_mount.diff --- tests/t4_adm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/t4_adm.py b/tests/t4_adm.py index a38e0fa..4a3f509 100755 --- a/tests/t4_adm.py +++ b/tests/t4_adm.py @@ -22,6 +22,7 @@ import tempfile import unittest import subprocess import pytest +import os @pytest.mark.usefixtures('pass_s3ql_cmd_argv', 'pass_reg_output') class AdmTests(unittest.TestCase): @@ -56,6 +57,7 @@ class AdmTests(unittest.TestCase): return stdout + @unittest.skipUnless(os.path.exists('/proc/mounts'), '/proc/mounts not available') def test_passphrase(self): self.mkfs() -- cgit v1.2.3 From b564edb57c794b335b6f3c3541e2620a9e756c04 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Estimate system clock granularity more conservatively Origin: debian Forwarded: no Patch-Name: clock-granularity.diff Some unit tests check that file access time stamps are updated correctly. For such a test to succeed, it's necessarily that the time between two subsequent test-accesses can actually be resolved. The heuristic to determine this granularity doesn't seem to work well on e.g. the i386 buildd, so we make it much more conservative. This means the test suite runs longer, but otherwise has no ill effects. Gbp-Pq: Name clock-granularity.diff --- tests/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common.py b/tests/common.py index 8c3be59..bea2e5c 100644 --- a/tests/common.py +++ b/tests/common.py @@ -27,7 +27,7 @@ def get_clock_granularity(): stamp2 = time.time() resolution = min(resolution, stamp2 - stamp1) time.sleep(0.01) - return resolution + return max(1, 10 * resolution) CLOCK_GRANULARITY = get_clock_granularity() # When testing, we want to make sure that we don't sleep for too short a time -- cgit v1.2.3 From 33c3225c44f78475c43c995e91b9beb56c3ad70b Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Mon, 27 Jun 2016 11:25:34 -0700 Subject: Don't hide pdflatex output. Origin: debian Forwarded: not-needed Patch-Name: show_pdflatex_output.diff Upstream is not interested in this patch. Gbp-Pq: Name show_pdflatex_output.diff --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 4f28bd3..7d2dc48 100755 --- a/setup.py +++ b/setup.py @@ -98,9 +98,8 @@ class build_docs(setuptools.Command): print('Running pdflatex...') for _ in range(3): - with open('/dev/null', 'wb') as null: - subprocess.check_call(['pdflatex', '-interaction', 'batchmode', 'manual.tex'], - cwd=os.path.join(dest_dir, 'latex'), stdout=null) + subprocess.check_call(['pdflatex', '-interaction', 'nonstopmode', 'manual.tex'], + cwd=os.path.join(dest_dir, 'latex')) os.rename(os.path.join(dest_dir, 'latex', 'manual.pdf'), os.path.join(dest_dir, 'manual.pdf')) -- cgit v1.2.3 From d2a5f3e4f16dd228a4146aa202f21eeabac31821 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Skip tests requiring /proc Origin: debian Forwarded: no Patch-Name: proc_mount.diff Gbp-Pq: Name proc_mount.diff --- tests/t4_adm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/t4_adm.py b/tests/t4_adm.py index a38e0fa..4a3f509 100755 --- a/tests/t4_adm.py +++ b/tests/t4_adm.py @@ -22,6 +22,7 @@ import tempfile import unittest import subprocess import pytest +import os @pytest.mark.usefixtures('pass_s3ql_cmd_argv', 'pass_reg_output') class AdmTests(unittest.TestCase): @@ -56,6 +57,7 @@ class AdmTests(unittest.TestCase): return stdout + @unittest.skipUnless(os.path.exists('/proc/mounts'), '/proc/mounts not available') def test_passphrase(self): self.mkfs() -- cgit v1.2.3 From afcadf08c20284ed324375162080d33f45621ae1 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Estimate system clock granularity more conservatively Origin: debian Forwarded: no Patch-Name: clock-granularity.diff Some unit tests check that file access time stamps are updated correctly. For such a test to succeed, it's necessarily that the time between two subsequent test-accesses can actually be resolved. The heuristic to determine this granularity doesn't seem to work well on e.g. the i386 buildd, so we make it much more conservative. This means the test suite runs longer, but otherwise has no ill effects. Gbp-Pq: Name clock-granularity.diff --- tests/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common.py b/tests/common.py index 8c3be59..bea2e5c 100644 --- a/tests/common.py +++ b/tests/common.py @@ -27,7 +27,7 @@ def get_clock_granularity(): stamp2 = time.time() resolution = min(resolution, stamp2 - stamp1) time.sleep(0.01) - return resolution + return max(1, 10 * resolution) CLOCK_GRANULARITY = get_clock_granularity() # When testing, we want to make sure that we don't sleep for too short a time -- cgit v1.2.3 From 9b6534770359230a598a63484f060d95246650d7 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Mon, 27 Jun 2016 11:25:34 -0700 Subject: Don't hide pdflatex output. Origin: debian Forwarded: not-needed Patch-Name: show_pdflatex_output.diff Upstream is not interested in this patch. Gbp-Pq: Name show_pdflatex_output.diff --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 02964e1..ee7a257 100755 --- a/setup.py +++ b/setup.py @@ -99,9 +99,8 @@ class build_docs(setuptools.Command): print('Running pdflatex...') for _ in range(3): - with open('/dev/null', 'wb') as null: - subprocess.check_call(['pdflatex', '-interaction', 'batchmode', 'manual.tex'], - cwd=os.path.join(dest_dir, 'latex'), stdout=null) + subprocess.check_call(['pdflatex', '-interaction', 'nonstopmode', 'manual.tex'], + cwd=os.path.join(dest_dir, 'latex')) os.rename(os.path.join(dest_dir, 'latex', 'manual.pdf'), os.path.join(dest_dir, 'manual.pdf')) -- cgit v1.2.3 From 532af1ce4722de355b4cb53dea2cfe870faea49e Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Sat, 6 Apr 2019 16:28:05 +0100 Subject: Commit patch to update .gitignore [dgit (3.11~deb9u1) update-gitignore-quilt-fixup] --- debian/patches/auto-gitignore | 15 +++++++++++++++ debian/patches/series | 1 + 2 files changed, 16 insertions(+) create mode 100644 debian/patches/auto-gitignore diff --git a/debian/patches/auto-gitignore b/debian/patches/auto-gitignore new file mode 100644 index 0000000..a7a4674 --- /dev/null +++ b/debian/patches/auto-gitignore @@ -0,0 +1,15 @@ +Subject: Update .gitignore from Debian packaging branch + +The Debian packaging git branch contains these updates to the upstream +.gitignore file(s). This patch is autogenerated, to provide these +updates to users of the official Debian archive view of the package. + +[dgit (3.11~deb9u1) update-gitignore] +--- +diff --git a/.gitignore b/.gitignore +new file mode 100644 +index 0000000..26a7c92 +--- /dev/null ++++ b/.gitignore +@@ -0,0 +1 @@ ++/.pc/ diff --git a/debian/patches/series b/debian/patches/series index 05b8717..f349004 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -1,3 +1,4 @@ proc_mount.diff clock-granularity.diff show_pdflatex_output.diff +auto-gitignore -- cgit v1.2.3 From a8b78b4a15bc59228f326064e054eedaa0eb338d Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Skip tests requiring /proc Origin: debian Forwarded: no Patch-Name: proc_mount.diff Gbp-Pq: Name proc_mount.diff --- tests/t4_adm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/t4_adm.py b/tests/t4_adm.py index a38e0fa..4a3f509 100755 --- a/tests/t4_adm.py +++ b/tests/t4_adm.py @@ -22,6 +22,7 @@ import tempfile import unittest import subprocess import pytest +import os @pytest.mark.usefixtures('pass_s3ql_cmd_argv', 'pass_reg_output') class AdmTests(unittest.TestCase): @@ -56,6 +57,7 @@ class AdmTests(unittest.TestCase): return stdout + @unittest.skipUnless(os.path.exists('/proc/mounts'), '/proc/mounts not available') def test_passphrase(self): self.mkfs() -- cgit v1.2.3 From f5af5eba93fa9d65f73c3d24f666f7b4e06458c4 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Estimate system clock granularity more conservatively Origin: debian Forwarded: no Patch-Name: clock-granularity.diff Some unit tests check that file access time stamps are updated correctly. For such a test to succeed, it's necessarily that the time between two subsequent test-accesses can actually be resolved. The heuristic to determine this granularity doesn't seem to work well on e.g. the i386 buildd, so we make it much more conservative. This means the test suite runs longer, but otherwise has no ill effects. Gbp-Pq: Name clock-granularity.diff --- tests/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common.py b/tests/common.py index 8c3be59..bea2e5c 100644 --- a/tests/common.py +++ b/tests/common.py @@ -27,7 +27,7 @@ def get_clock_granularity(): stamp2 = time.time() resolution = min(resolution, stamp2 - stamp1) time.sleep(0.01) - return resolution + return max(1, 10 * resolution) CLOCK_GRANULARITY = get_clock_granularity() # When testing, we want to make sure that we don't sleep for too short a time -- cgit v1.2.3 From 2f09857a9696331d0085f744eee9da77149f58b3 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Mon, 27 Jun 2016 11:25:34 -0700 Subject: Don't hide pdflatex output. Origin: debian Forwarded: not-needed Patch-Name: show_pdflatex_output.diff Upstream is not interested in this patch. Gbp-Pq: Name show_pdflatex_output.diff --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 02964e1..ee7a257 100755 --- a/setup.py +++ b/setup.py @@ -99,9 +99,8 @@ class build_docs(setuptools.Command): print('Running pdflatex...') for _ in range(3): - with open('/dev/null', 'wb') as null: - subprocess.check_call(['pdflatex', '-interaction', 'batchmode', 'manual.tex'], - cwd=os.path.join(dest_dir, 'latex'), stdout=null) + subprocess.check_call(['pdflatex', '-interaction', 'nonstopmode', 'manual.tex'], + cwd=os.path.join(dest_dir, 'latex')) os.rename(os.path.join(dest_dir, 'latex', 'manual.pdf'), os.path.join(dest_dir, 'manual.pdf')) -- cgit v1.2.3 From ad2ed5c38cdb86b9fe44ed8ba75ab8c2a97f7cdb Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Wed, 10 Apr 2019 21:36:05 +0100 Subject: Drop dependency on google-auth-oauthlib This is optional and only necessary for some features. Origin: debian Forwarded: not-needed Gbp-Pq: Name drop_oauthlib_dep.diff --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index ee7a257..9dfa793 100755 --- a/setup.py +++ b/setup.py @@ -145,8 +145,7 @@ def main(): 'defusedxml', 'dugong >= 3.4, < 4.0', 'llfuse >= 1.0, < 2.0', - 'google-auth', - 'google-auth-oauthlib'] + 'google-auth'] setuptools.setup( name='s3ql', -- cgit v1.2.3 From 2206daf1fe04b9a6872bab5b4b88a104e49daf6d Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Wed, 10 Apr 2019 21:42:10 +0100 Subject: Commit patch to update .gitignore [dgit (3.11~deb9u1) update-gitignore-quilt-fixup] --- debian/patches/auto-gitignore | 15 +++++++++++++++ debian/patches/series | 1 + 2 files changed, 16 insertions(+) create mode 100644 debian/patches/auto-gitignore diff --git a/debian/patches/auto-gitignore b/debian/patches/auto-gitignore new file mode 100644 index 0000000..a7a4674 --- /dev/null +++ b/debian/patches/auto-gitignore @@ -0,0 +1,15 @@ +Subject: Update .gitignore from Debian packaging branch + +The Debian packaging git branch contains these updates to the upstream +.gitignore file(s). This patch is autogenerated, to provide these +updates to users of the official Debian archive view of the package. + +[dgit (3.11~deb9u1) update-gitignore] +--- +diff --git a/.gitignore b/.gitignore +new file mode 100644 +index 0000000..26a7c92 +--- /dev/null ++++ b/.gitignore +@@ -0,0 +1 @@ ++/.pc/ diff --git a/debian/patches/series b/debian/patches/series index 11921ef..8f9792a 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -2,3 +2,4 @@ proc_mount.diff clock-granularity.diff show_pdflatex_output.diff drop_oauthlib_dep.diff +auto-gitignore -- cgit v1.2.3