From 347f05d4ab3ac1d7ce1db9599278f6533477acf5 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Wed, 9 Mar 2016 10:09:11 -0800 Subject: Import s3ql_2.5.orig.tar.bz2 --- tests/t4_fuse.py | 265 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 156 insertions(+), 109 deletions(-) (limited to 'tests/t4_fuse.py') diff --git a/tests/t4_fuse.py b/tests/t4_fuse.py index f120637..e2cfe3b 100644 --- a/tests/t4_fuse.py +++ b/tests/t4_fuse.py @@ -6,10 +6,9 @@ Copyright (C) 2008-2009 Nikolaus Rath This program can be distributed under the terms of the GNU GPLv3. ''' -from __future__ import absolute_import, division, print_function from os.path import basename -from s3ql.common import CTRL_NAME -import cPickle as pickle +from s3ql.common import CTRL_NAME, PICKLE_PROTOCOL, path2bytes +import pickle import filecmp import llfuse import logging @@ -23,17 +22,17 @@ import tempfile import threading import time import traceback -import unittest2 as unittest +import unittest -log = logging.getLogger() +log = logging.getLogger(__name__) # For debugging USE_VALGRIND = False class ExceptionStoringThread(threading.Thread): def __init__(self): - super(ExceptionStoringThread, self).__init__() + super().__init__() self._exc_info = None self._joined = False @@ -75,7 +74,7 @@ class EmbeddedException(Exception): ''' def __init__(self, exc_info, threadname): - super(EmbeddedException, self).__init__() + super().__init__() self.exc_info = exc_info self.threadname = threadname @@ -89,7 +88,7 @@ class EmbeddedException(Exception): class AsyncFn(ExceptionStoringThread): def __init__(self, fn, *args, **kwargs): - super(AsyncFn, self).__init__() + super().__init__() self.target = fn self.args = args self.kwargs = kwargs @@ -102,7 +101,7 @@ def retry(timeout, fn, *a, **kw): If the return value of fn() returns something True, this value is returned. Otherwise, the function is called repeatedly for - `timeout` seconds. If the timeout is reached, `TimeoutError` is + `timeout` seconds. If the timeout is reached, `RetryTimeoutError` is raised. """ @@ -117,9 +116,9 @@ def retry(timeout, fn, *a, **kw): if step < waited / 30: step *= 2 - raise TimeoutError() + raise RetryTimeoutError() -class TimeoutError(Exception): +class RetryTimeoutError(Exception): '''Raised by `retry()` when a timeout is reached.''' pass @@ -127,10 +126,11 @@ class TimeoutError(Exception): def skip_if_no_fusermount(): '''Raise SkipTest if fusermount is not available''' - which = subprocess.Popen(['which', 'fusermount'], stdout=subprocess.PIPE) - fusermount_path = which.communicate()[0].strip() - - if not fusermount_path or which.wait() != 0: + with subprocess.Popen(['which', 'fusermount'], stdout=subprocess.PIPE, + universal_newlines=True) as which: + fusermount_path = which.communicate()[0].strip() + + if not fusermount_path or which.returncode != 0: raise unittest.SkipTest("Can't find fusermount executable") if not os.path.exists('/dev/fuse'): @@ -144,17 +144,27 @@ def skip_if_no_fusermount(): raise unittest.SkipTest('fusermount executable not setuid, and we are not root.') try: - subprocess.check_call([fusermount_path, '-V'], - stdout=open('/dev/null', 'wb')) + with open('/dev/null', 'wb') as null: + subprocess.check_call([fusermount_path, '-V'], stdout=null) except subprocess.CalledProcessError: - raise unittest.SkipTest('Unable to execute fusermount') + raise unittest.SkipTest('Unable to execute fusermount') from None +def skip_without_rsync(): + try: + with open('/dev/null', 'wb') as null: + subprocess.call(['rsync', '--version'], stdout=null, + stderr=subprocess.STDOUT,) + except FileNotFoundError: + raise unittest.SkipTest('rsync not installed') from None + + if __name__ == '__main__': mypath = sys.argv[0] else: mypath = __file__ BASEDIR = os.path.abspath(os.path.join(os.path.dirname(mypath), '..')) + class fuse_tests(unittest.TestCase): def setUp(self): @@ -165,35 +175,46 @@ class fuse_tests(unittest.TestCase): if os.path.getsize(self.src) < 1048: raise RuntimeError("test file %s should be bigger than 1 KiB" % self.src) - self.mnt_dir = tempfile.mkdtemp() - self.cache_dir = tempfile.mkdtemp() - self.backend_dir = tempfile.mkdtemp() + self.mnt_dir = tempfile.mkdtemp(prefix='s3ql-mnt-') + self.cache_dir = tempfile.mkdtemp(prefix='s3ql-cache-') + self.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-') self.storage_url = 'local://' + self.backend_dir self.passphrase = 'oeut3d' + self.backend_login = None + self.backend_passphrase = None self.mount_process = None self.name_cnt = 0 - def mkfs(self): + def mkfs(self, max_obj_size=500): proc = subprocess.Popen([sys.executable, os.path.join(BASEDIR, 'bin', 'mkfs.s3ql'), - '-L', 'test fs', '--max-obj-size', '500', - '--cachedir', self.cache_dir, '--quiet', - self.storage_url ], stdin=subprocess.PIPE) - + '-L', 'test fs', '--max-obj-size', str(max_obj_size), + '--fatal-warnings', '--cachedir', self.cache_dir, '--quiet', + '--authfile', '/dev/null', self.storage_url ], + stdin=subprocess.PIPE, universal_newlines=True) + + if self.backend_login is not None: + print(self.backend_login, file=proc.stdin) + print(self.backend_passphrase, file=proc.stdin) print(self.passphrase, file=proc.stdin) print(self.passphrase, file=proc.stdin) proc.stdin.close() self.assertEqual(proc.wait(), 0) - def mount(self): - self.mount_process = subprocess.Popen([sys.executable, - os.path.join(BASEDIR, 'bin', 'mount.s3ql'), - "--fg", '--cachedir', self.cache_dir, - '--log', 'none', '--quiet', - self.storage_url, self.mnt_dir], - stdin=subprocess.PIPE) + def mount(self, fatal_warnings=True): + cmd = [sys.executable, os.path.join(BASEDIR, 'bin', 'mount.s3ql'), + "--fg", '--cachedir', self.cache_dir, '--log', 'none', + '--compress', 'zlib', '--quiet', self.storage_url, self.mnt_dir, + '--authfile', '/dev/null' ] + if fatal_warnings: + cmd.append('--fatal-warnings') + self.mount_process = subprocess.Popen(cmd, stdin=subprocess.PIPE, + universal_newlines=True) + if self.backend_login is not None: + print(self.backend_login, file=self.mount_process.stdin) + print(self.backend_passphrase, file=self.mount_process.stdin) print(self.passphrase, file=self.mount_process.stdin) self.mount_process.stdin.close() def poll(): @@ -203,35 +224,56 @@ class fuse_tests(unittest.TestCase): retry(30, poll) def umount(self): - devnull = open('/dev/null', 'wb') - retry(5, lambda: subprocess.call(['fuser', '-m', self.mnt_dir], - stdout=devnull, stderr=devnull) == 1) + with open('/dev/null', 'wb') as devnull: + retry(5, lambda: subprocess.call(['fuser', '-m', self.mnt_dir], + stdout=devnull, stderr=devnull) == 1) - proc = subprocess.Popen([os.path.join(BASEDIR, 'bin', 'umount.s3ql'), + proc = subprocess.Popen([sys.executable, os.path.join(BASEDIR, 'bin', 'umount.s3ql'), '--quiet', self.mnt_dir]) retry(90, lambda : proc.poll() is not None) - self.assertEquals(proc.wait(), 0) + self.assertEqual(proc.wait(), 0) - self.assertEqual(self.mount_process.wait(), 0) + self.assertEqual(self.mount_process.poll(), 0) self.assertFalse(os.path.ismount(self.mnt_dir)) def fsck(self): - proc = subprocess.Popen([sys.executable, os.path.join(BASEDIR, 'bin', 'fsck.s3ql'), - '--force', '--quiet', '--log', 'none', - '--cachedir', self.cache_dir, - self.storage_url ], stdin=subprocess.PIPE) - print(self.passphrase, file=proc.stdin) - proc.stdin.close() - self.assertEqual(proc.wait(), 0) + # Use fsck to test authinfo reading + with tempfile.NamedTemporaryFile('wt') as authinfo_fh: + print('[entry1]', + 'storage-url: %s' % self.storage_url[:6], + 'fs-passphrase: clearly wrong', + 'backend-login: bla', + 'backend-password: not much better', + '', + '[entry2]', + 'storage-url: %s' % self.storage_url, + 'fs-passphrase: %s' % self.passphrase, + 'backend-login: %s' % self.backend_login, + 'backend-password:%s' % self.backend_passphrase, + file=authinfo_fh, sep='\n') + authinfo_fh.flush() + + proc = subprocess.Popen([sys.executable, os.path.join(BASEDIR, 'bin', 'fsck.s3ql'), + '--force', '--quiet', '--log', 'none', '--cachedir', + self.cache_dir, '--fatal-warnings', '--authfile', + authinfo_fh.name, self.storage_url ], + stdin=subprocess.PIPE, universal_newlines=True) + proc.stdin.close() + self.assertEqual(proc.wait(), 0) def tearDown(self): - subprocess.call(['fusermount', '-z', '-u', self.mnt_dir], - stderr=open('/dev/null', 'wb')) + with open('/dev/null', 'wb') as devnull: + subprocess.call(['fusermount', '-z', '-u', self.mnt_dir], + stderr=devnull) os.rmdir(self.mnt_dir) # Give mount process a little while to terminate if self.mount_process is not None: - retry(10, lambda : self.mount_process.poll() is not None) + try: + retry(90, lambda : self.mount_process.poll() is not None) + except TimeoutError: + # Ignore errors during teardown + pass shutil.rmtree(self.cache_dir) shutil.rmtree(self.backend_dir) @@ -271,11 +313,11 @@ class fuse_tests(unittest.TestCase): os.mkdir(fullname) fstat = os.stat(fullname) self.assertTrue(stat.S_ISDIR(fstat.st_mode)) - self.assertEquals(llfuse.listdir(fullname), []) - self.assertEquals(fstat.st_nlink, 1) + self.assertEqual(llfuse.listdir(fullname), []) + self.assertEqual(fstat.st_nlink, 1) self.assertTrue(dirname in llfuse.listdir(self.mnt_dir)) os.rmdir(fullname) - self.assertRaises(OSError, os.stat, fullname) + self.assertRaises(FileNotFoundError, os.stat, fullname) self.assertTrue(dirname not in llfuse.listdir(self.mnt_dir)) def tst_symlink(self): @@ -284,11 +326,11 @@ class fuse_tests(unittest.TestCase): os.symlink("/imaginary/dest", fullname) fstat = os.lstat(fullname) self.assertTrue(stat.S_ISLNK(fstat.st_mode)) - self.assertEquals(os.readlink(fullname), "/imaginary/dest") - self.assertEquals(fstat.st_nlink, 1) + self.assertEqual(os.readlink(fullname), "/imaginary/dest") + self.assertEqual(fstat.st_nlink, 1) self.assertTrue(linkname in llfuse.listdir(self.mnt_dir)) os.unlink(fullname) - self.assertRaises(OSError, os.lstat, fullname) + self.assertRaises(FileNotFoundError, os.lstat, fullname) self.assertTrue(linkname not in llfuse.listdir(self.mnt_dir)) def tst_mknod(self): @@ -297,11 +339,11 @@ class fuse_tests(unittest.TestCase): shutil.copyfile(src, filename) fstat = os.lstat(filename) self.assertTrue(stat.S_ISREG(fstat.st_mode)) - self.assertEquals(fstat.st_nlink, 1) + self.assertEqual(fstat.st_nlink, 1) self.assertTrue(basename(filename) in llfuse.listdir(self.mnt_dir)) self.assertTrue(filecmp.cmp(src, filename, False)) os.unlink(filename) - self.assertRaises(OSError, os.stat, filename) + self.assertRaises(FileNotFoundError, os.stat, filename) self.assertTrue(basename(filename) not in llfuse.listdir(self.mnt_dir)) def tst_chown(self): @@ -314,17 +356,17 @@ class fuse_tests(unittest.TestCase): uid_new = uid + 1 os.chown(filename, uid_new, -1) fstat = os.lstat(filename) - self.assertEquals(fstat.st_uid, uid_new) - self.assertEquals(fstat.st_gid, gid) + self.assertEqual(fstat.st_uid, uid_new) + self.assertEqual(fstat.st_gid, gid) gid_new = gid + 1 os.chown(filename, -1, gid_new) fstat = os.lstat(filename) - self.assertEquals(fstat.st_uid, uid_new) - self.assertEquals(fstat.st_gid, gid_new) + self.assertEqual(fstat.st_uid, uid_new) + self.assertEqual(fstat.st_gid, gid_new) os.rmdir(filename) - self.assertRaises(OSError, os.stat, filename) + self.assertRaises(FileNotFoundError, os.stat, filename) self.assertTrue(basename(filename) not in llfuse.listdir(self.mnt_dir)) @@ -351,14 +393,14 @@ class fuse_tests(unittest.TestCase): fstat1 = os.lstat(name1) fstat2 = os.lstat(name2) - self.assertEquals(fstat1, fstat2) - self.assertEquals(fstat1.st_nlink, 2) + self.assertEqual(fstat1, fstat2) + self.assertEqual(fstat1.st_nlink, 2) self.assertTrue(basename(name2) in llfuse.listdir(self.mnt_dir)) self.assertTrue(filecmp.cmp(name1, name2, False)) os.unlink(name2) fstat1 = os.lstat(name1) - self.assertEquals(fstat1.st_nlink, 1) + self.assertEqual(fstat1.st_nlink, 1) os.unlink(name1) def tst_readdir(self): @@ -377,7 +419,7 @@ class fuse_tests(unittest.TestCase): listdir_is.sort() listdir_should = [ basename(file_), basename(subdir) ] listdir_should.sort() - self.assertEquals(listdir_is, listdir_should) + self.assertEqual(listdir_is, listdir_should) os.unlink(file_) os.unlink(subfile) @@ -394,10 +436,10 @@ class fuse_tests(unittest.TestCase): fd = os.open(filename, os.O_RDWR) os.ftruncate(fd, size + 1024) # add > 1 block - self.assertEquals(os.stat(filename).st_size, size + 1024) + self.assertEqual(os.stat(filename).st_size, size + 1024) os.ftruncate(fd, size - 1024) # Truncate > 1 block - self.assertEquals(os.stat(filename).st_size, size - 1024) + self.assertEqual(os.stat(filename).st_size, size - 1024) os.close(fd) os.unlink(filename) @@ -416,10 +458,10 @@ class fuse_tests(unittest.TestCase): fd = os.open(filename, os.O_RDWR) os.ftruncate(fd, size + 1024) # add > 1 block - self.assertEquals(os.stat(filename).st_size, size + 1024) + self.assertEqual(os.stat(filename).st_size, size + 1024) os.ftruncate(fd, size - 1024) # Truncate > 1 block - self.assertEquals(os.stat(filename).st_size, size - 1024) + self.assertEqual(os.stat(filename).st_size, size - 1024) os.close(fd) os.unlink(filename) @@ -431,9 +473,9 @@ class fuse_tests(unittest.TestCase): self.assertTrue(stat.S_ISDIR(os.stat(fullname).st_mode)) self.assertTrue(dirname in llfuse.listdir(self.mnt_dir)) llfuse.setxattr('%s/%s' % (self.mnt_dir, CTRL_NAME), - 'rmtree', pickle.dumps((llfuse.ROOT_INODE, dirname), - pickle.HIGHEST_PROTOCOL)) - self.assertRaises(OSError, os.stat, fullname) + 'rmtree', pickle.dumps((llfuse.ROOT_INODE, path2bytes(dirname)), + PICKLE_PROTOCOL)) + self.assertRaises(FileNotFoundError, os.stat, fullname) self.assertTrue(dirname not in llfuse.listdir(self.mnt_dir)) @@ -441,25 +483,48 @@ class fuse_tests(unittest.TestCase): def suite(): return unittest.makeSuite(fuse_tests) -def populate_dir(path, entries=4096, max_size=10*1024*1024, +def populate_dir(path, entries=1000, size=20*1024*1024, pooldir='/usr/bin', seed=None): '''Populate directory with random data - *entires* specifies the total number of directory entries that - are created in the tree. *max_size* specifies the maximum size - occupied by all files. The files in *pooldir* are used as a - source of directory names and file contents. + *entries* specifies the total number of directory entries that are created + in the tree. *size* specifies the size occupied by all files together. The + files in *pooldir* are used as a source of directory names and file + contents. - *seed* is used to initalize the random number generator and - can be used to make the created structure reproducible - (provided that the contents of *pooldir* don't change). + *seed* is used to initalize the random number generator and can be used to + make the created structure reproducible (provided that the contents of + *pooldir* don't change). ''' poolnames = os.listdir(pooldir) if seed is None: + # We want tests to be reproducible on a given system, so users + # can report meaningful bugs seed = len(poolnames) random.seed(seed) - + + # Entries in percentages + subdir_cnt = random.randint(5, 10) + file_cnt = random.randint(60, 70) + fifo_cnt = random.randint(5, 10) + symlink_cnt = random.randint(10, 20) + hardlink_cnt = random.randint(5, 15) + + # Normalize to desired entry count + scale = entries / sum((subdir_cnt, file_cnt, fifo_cnt, symlink_cnt, hardlink_cnt)) + subdir_cnt = int(scale * subdir_cnt) + file_cnt = int(scale * file_cnt) + fifo_cnt = int(scale * fifo_cnt) + symlink_cnt = int(scale * symlink_cnt) + hardlink_cnt = int(scale * hardlink_cnt) + + # Sizes, make sure there is at least one big file + file_sizes = [ random.randint(0, 100) for _ in range(file_cnt-1) ] + scale = 0.5 * size / sum(file_sizes) + file_sizes = [ int(scale * x) for x in file_sizes ] + file_sizes.append(int(0.5 * size)) + # Special characters for use in filenames special_chars = [ chr(x) for x in range(256) if x not in (0, ord('/')) ] @@ -467,8 +532,7 @@ def populate_dir(path, entries=4096, max_size=10*1024*1024, def random_name(path): '''Get random, non-existing file name underneath *path* - Returns a fully qualified path with a filename chosen - from *poolnames*. + Returns a fully qualified path with a filename chosen from *poolnames*. ''' while True: name = poolnames[random.randrange(len(poolnames))] @@ -478,7 +542,7 @@ def populate_dir(path, entries=4096, max_size=10*1024*1024, if len_ > 0: pos = random.choice((-1,0,1)) # Prefix, Middle, Suffix s = ''.join(special_chars[random.randrange(len(special_chars))] - for _ in xrange(len_)) + for _ in range(len_)) if pos == -1: name = s + name elif pos == 1: @@ -495,10 +559,8 @@ def populate_dir(path, entries=4096, max_size=10*1024*1024, # # Step 1: create directory tree # - subdir_cnt = random.randint(0, int(0.1 * entries)) - entries -= subdir_cnt dirs = [ path ] - for _ in xrange(subdir_cnt): + for _ in range(subdir_cnt): idx = random.randrange(len(dirs)) name = random_name(dirs[idx]) os.mkdir(name) @@ -508,14 +570,10 @@ def populate_dir(path, entries=4096, max_size=10*1024*1024, # # Step 2: populate the tree with files # - file_cnt = random.randint(int(entries/3), int(3*entries/4)) - entries -= file_cnt files = [] - for _ in xrange(file_cnt): + for size in file_sizes: idx = random.randrange(len(dirs)) name = random_name(dirs[idx]) - size = random.randint(0, int(0.01 * max_size)) - max_size -= size with open(name, 'wb') as dst: while size > 0: idx = random.randrange(len(poolnames)) @@ -531,19 +589,15 @@ def populate_dir(path, entries=4096, max_size=10*1024*1024, # # Step 3: Special files # - fifo_cnt = random.randint(int(entries/3), int(2*entries/3)) - entries -= fifo_cnt - for _ in xrange(fifo_cnt): + for _ in range(fifo_cnt): name = random_name(dirs[random.randrange(len(dirs))]) os.mkfifo(name) files.append(name) - - # + + # # Step 4: populate tree with symlinks # - symlink_cnt = random.randint(int(entries/3), int(2*entries/3)) - entries -= symlink_cnt - for _ in xrange(symlink_cnt): + for _ in range(symlink_cnt): relative = random.choice((True, False)) existing = random.choice((True, False)) idx = random.randrange(len(dirs)) @@ -569,9 +623,7 @@ def populate_dir(path, entries=4096, max_size=10*1024*1024, # # Step 5: Create some hardlinks # - hardlink_cnt = random.randint(int(entries/3), int(2*entries/3)) - entries -= hardlink_cnt - for _ in xrange(hardlink_cnt): + for _ in range(hardlink_cnt): samedir = random.choice((True, False)) target = files[random.randrange(len(files))] @@ -582,8 +634,3 @@ def populate_dir(path, entries=4096, max_size=10*1024*1024, name = random_name(dir_) os.link(target, name) files.append(name) - - -# Allow calling from command line -if __name__ == "__main__": - unittest.main() -- cgit v1.2.3