Import s3ql_1.13.1.orig.tar.bz2

5 files changed, 85 insertions, 62 deletions

diff --git a/src/s3ql/backends/common.py b/src/s3ql/backends/common.py
index 5e22e70..9331879 100644
--- a/src/s3ql/backends/common.py
+++ b/src/s3ql/backends/common.py
@@ -19,17 +19,19 @@ from s3ql.common import ChecksumError
 import ConfigParser
 import bz2
 import cPickle as pickle
+import errno
 import hashlib
 import hmac
 import httplib
 import logging
 import lzma
+import math
 import os
 import re
+import socket
 import stat
 import struct
 import sys
-import math
 import threading
 import time
 import zlib
@@ -89,6 +91,31 @@ True.
 '''
     return wrapped
 
+def is_temp_network_error(exc):
+    '''Return true if *exc* represents a potentially temporary network problem'''
+
+    if isinstance(exc, (httplib.IncompleteRead, socket.timeout)):
+        return True
+     
+    # Server closed connection
+    elif (isinstance(exc, httplib.BadStatusLine)
+          and (not exc.line or exc.line == "''")):
+        return True
+
+    elif (isinstance(exc, IOError) and
+          exc.errno in (errno.EPIPE, errno.ECONNRESET, errno.ETIMEDOUT,
+                        errno.EINTR)):
+        return True
+
+    # Formally this is a permanent error. However, it may also indicate
+    # that there is currently no network connection to the DNS server
+    elif (isinstance(exc, socket.gaierror) 
+          and exc.errno in (socket.EAI_AGAIN, socket.EAI_NONAME)):
+        return True 
+              
+    return False
+    
+    
 def http_connection(hostname, port, ssl=False):
     '''Return http connection to *hostname*:*port*
     
@@ -851,7 +878,10 @@ class DecryptFilter(AbstractInputFilter):
                 self.hmac.update(inbuf)
                 outbuf += inbuf
                 break
-
+            elif len(inbuf) <= self.remaining + self.off_size:
+                inbuf += self.fh.read(self.off_size)
+                continue
+                
             outbuf += inbuf[:self.remaining]
             self.hmac.update(inbuf[:self.remaining + self.off_size])
             paket_size = struct.unpack(b'<I', inbuf[self.remaining
@@ -1025,11 +1055,11 @@ class NoSuchObject(Exception):
     def __str__(self):
         return 'Backend does not have anything stored under key %r' % self.key
 
-class DanglingStorageURL(Exception):
+class DanglingStorageURLError(Exception):
     '''Raised if the backend can't store data at the given location'''
 
     def __init__(self, loc):
-        super(DanglingStorageURL, self).__init__()
+        super(DanglingStorageURLError, self).__init__()
         self.loc = loc
 
     def __str__(self):
@@ -1173,7 +1203,7 @@ def get_backend_factory(options, plain=False):
         # (e.g. wrong credentials)
         _ = backend['s3ql_passphrase']
         
-    except DanglingStorageURL as exc:
+    except DanglingStorageURLError as exc:
         raise QuietError(str(exc))   
     
     except AuthorizationError:
diff --git a/src/s3ql/backends/local.py b/src/s3ql/backends/local.py
index 6d4e817..426d855 100644
--- a/src/s3ql/backends/local.py
+++ b/src/s3ql/backends/local.py
@@ -8,7 +8,7 @@ This program can be distributed under the terms of the GNU GPLv3.
 
 from __future__ import division, print_function, absolute_import
 
-from .common import AbstractBackend, DanglingStorageURL, NoSuchObject, ChecksumError
+from .common import AbstractBackend, DanglingStorageURLError, NoSuchObject, ChecksumError
 from ..common import BUFSIZE
 import shutil
 import logging
@@ -39,7 +39,7 @@ class Backend(AbstractBackend):
         self.name = name
 
         if not os.path.exists(name):
-            raise DanglingStorageURL(name)
+            raise DanglingStorageURLError(name)
 
     def __str__(self):
         return 'local://%s' % self.name
diff --git a/src/s3ql/backends/s3.py b/src/s3ql/backends/s3.py
index a41f512..ed6b8a0 100644
--- a/src/s3ql/backends/s3.py
+++ b/src/s3ql/backends/s3.py
@@ -38,6 +38,11 @@ class Backend(s3c.Backend):
             raise QuietError('Invalid storage URL')
 
         bucket_name = hit.group(1)
+        
+        # http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/BucketRestrictions.html
+        if not re.match('^[a-z][a-z0-9.-]{1,60}[a-z0-9]$', bucket_name):
+            raise QuietError('Invalid bucket name.')
+        
         hostname = '%s.s3.amazonaws.com' % bucket_name
         prefix = hit.group(2) or ''
         port = 443 if use_ssl else 80
diff --git a/src/s3ql/backends/s3c.py b/src/s3ql/backends/s3c.py
index 5fba441..994922b 100644
--- a/src/s3ql/backends/s3c.py
+++ b/src/s3ql/backends/s3c.py
@@ -10,11 +10,10 @@ from __future__ import division, print_function, absolute_import
 from ..common import BUFSIZE, QuietError
 from .common import AbstractBackend, NoSuchObject, retry, AuthorizationError, http_connection, \
     AuthenticationError
-from .common import DanglingStorageURL as DanglingStorageURL_common
+from .common import DanglingStorageURLError
 from base64 import b64encode
 from email.utils import parsedate_tz, mktime_tz
 from urlparse import urlsplit
-import errno
 import hashlib
 import hmac
 import httplib
@@ -24,12 +23,13 @@ import tempfile
 import time
 import urllib
 import xml.etree.cElementTree as ElementTree
+from s3ql.backends.common import is_temp_network_error
 
 
 C_DAY_NAMES = [ 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun' ]
 C_MONTH_NAMES = [ 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec' ]
 
-XML_CONTENT_RE = re.compile('^application/xml(?:;\s+|$)', re.IGNORECASE)
+XML_CONTENT_RE = re.compile(r'^(?:application|text)/xml(?:;|$)', re.IGNORECASE)
 
 log = logging.getLogger("backends.s3c")
 
@@ -102,19 +102,12 @@ class Backend(AbstractBackend):
         problems and so that the request can be manually restarted if applicable.
         '''
 
-        if isinstance(exc, (InternalError, BadDigest, IncompleteBody, RequestTimeout,
-                            OperationAborted, SlowDown, RequestTimeTooSkewed,
-                            httplib.IncompleteRead)):
+        if isinstance(exc, (InternalError, BadDigestError, IncompleteBodyError, 
+                            RequestTimeoutError, OperationAbortedError, SlowDownError, 
+                            RequestTimeTooSkewedError)):
             return True
 
-        # Server closed connection
-        elif (isinstance(exc, httplib.BadStatusLine)
-              and (not exc.line or exc.line == "''")):
-            return True
-
-        elif (isinstance(exc, IOError) and
-              exc.errno in (errno.EPIPE, errno.ECONNRESET, errno.ETIMEDOUT,
-                            errno.EINTR)):
+        elif is_temp_network_error(exc):
             return True
         
         elif isinstance(exc, HTTPError) and exc.status >= 500 and exc.status <= 599:
@@ -130,7 +123,7 @@ class Backend(AbstractBackend):
         try:
             resp = self._do_request('DELETE', '/%s%s' % (self.prefix, key))
             assert resp.length == 0
-        except NoSuchKey:
+        except NoSuchKeyError:
             if force:
                 pass
             else:
@@ -279,7 +272,7 @@ class Backend(AbstractBackend):
 
         try:
             resp = self._do_request('GET', '/%s%s' % (self.prefix, key))
-        except NoSuchKey:
+        except NoSuchKeyError:
             raise NoSuchObject(key)
 
         return ObjectR(key, resp, self, extractmeta(resp))
@@ -325,7 +318,7 @@ class Backend(AbstractBackend):
                                                                                 self.prefix, src)})
             # Discard response body
             resp.read()
-        except NoSuchKey:
+        except NoSuchKeyError:
             raise NoSuchObject(src)
 
     def _do_request(self, method, path, subres=None, query_string=None,
@@ -551,7 +544,7 @@ class ObjectR(object):
             if etag != self.md5.hexdigest():
                 log.warn('ObjectR(%s).close(): MD5 mismatch: %s vs %s', self.key, etag,
                          self.md5.hexdigest())
-                raise BadDigest('BadDigest', 'ETag header does not agree with calculated MD5')
+                raise BadDigestError('BadDigest', 'ETag header does not agree with calculated MD5')
             
             return buf
 
@@ -624,7 +617,7 @@ class ObjectW(object):
             except:
                 log.exception('Objectw(%s).close(): unable to delete corrupted object!',
                               self.key)
-            raise BadDigest('BadDigest', 'Received ETag does not agree with our calculations.')
+            raise BadDigestError('BadDigest', 'Received ETag does not agree with our calculations.')
 
     def __enter__(self):
         return self
@@ -642,7 +635,12 @@ class ObjectW(object):
 def get_S3Error(code, msg):
     '''Instantiate most specific S3Error subclass'''
 
-    return globals().get(code, S3Error)(code, msg)
+    class_ = globals().get(code + 'Error', S3Error)
+
+    if not issubclass(class_, S3Error):
+        return S3Error(code, msg)
+    
+    return class_(code, msg)
 
 def extractmeta(resp):
     '''Extract metadata from HTTP response object'''
@@ -711,16 +709,17 @@ class S3Error(Exception):
     def __str__(self):
         return '%s: %s' % (self.code, self.msg)
 
-class NoSuchKey(S3Error): pass
-class AccessDenied(S3Error, AuthorizationError): pass
-class BadDigest(S3Error): pass
-class IncompleteBody(S3Error): pass
+class NoSuchKeyError(S3Error): pass
+class AccessDeniedError(S3Error, AuthorizationError): pass
+class BadDigestError(S3Error): pass
+class IncompleteBodyError(S3Error): pass
 class InternalError(S3Error): pass
-class InvalidAccessKeyId(S3Error, AuthenticationError): pass
-class InvalidSecurity(S3Error, AuthenticationError): pass
-class SignatureDoesNotMatch(S3Error, AuthenticationError): pass
-class OperationAborted(S3Error): pass
-class RequestTimeout(S3Error): pass
-class SlowDown(S3Error): pass
-class RequestTimeTooSkewed(S3Error): pass
-class DanglingStorageURL(S3Error, DanglingStorageURL_common): pass
+class InvalidAccessKeyIdError(S3Error, AuthenticationError): pass
+class InvalidSecurityError(S3Error, AuthenticationError): pass
+class SignatureDoesNotMatchError(S3Error, AuthenticationError): pass
+class OperationAbortedError(S3Error): pass
+class RequestTimeoutError(S3Error): pass
+class TimeoutError(RequestTimeoutError): pass
+class SlowDownError(S3Error): pass
+class RequestTimeTooSkewedError(S3Error): pass
+class NoSuchBucketError(S3Error, DanglingStorageURLError): pass
diff --git a/src/s3ql/backends/swift.py b/src/s3ql/backends/swift.py
index f2fe4b7..51dfdad 100644
--- a/src/s3ql/backends/swift.py
+++ b/src/s3ql/backends/swift.py
@@ -8,14 +8,13 @@ This program can be distributed under the terms of the GNU GPLv3.
 
 from __future__ import division, print_function, absolute_import
 from ..common import QuietError, BUFSIZE
-from .common import (AbstractBackend, NoSuchObject, retry, AuthorizationError, http_connection,
-                     DanglingStorageURL)
-from .s3c import HTTPError, BadDigest
+from .common import (AbstractBackend, NoSuchObject, retry, AuthorizationError, http_connection, 
+    DanglingStorageURLError)
+from .s3c import HTTPError, BadDigestError
+from s3ql.backends.common import is_temp_network_error
 from urlparse import urlsplit
-import json
-import errno
 import hashlib
-import httplib
+import json
 import logging
 import re
 import tempfile
@@ -59,7 +58,7 @@ class Backend(AbstractBackend):
             resp = self._do_request('GET', '/', query_string={'limit': 1 })
         except HTTPError as exc:
             if exc.status == 404:
-                raise DanglingStorageURL(self.container_name)
+                raise DanglingStorageURLError(self.container_name)
             raise
         resp.read()   
                     
@@ -98,23 +97,13 @@ class Backend(AbstractBackend):
         problems and so that the request can be manually restarted if applicable.
         '''
 
-        if isinstance(exc, (httplib.IncompleteRead,)):
-            return True
-
-        # Server closed connection
-        elif (isinstance(exc, httplib.BadStatusLine)
-              and (not exc.line or exc.line == "''")):
+        if isinstance(exc, (AuthenticationExpired,)):
             return True
 
-        elif (isinstance(exc, IOError) and
-              exc.errno in (errno.EPIPE, errno.ECONNRESET, errno.ETIMEDOUT,
-                            errno.EINTR)):
-            return True
-        
         elif isinstance(exc, HTTPError) and exc.status >= 500 and exc.status <= 599:
             return True
 
-        elif isinstance(exc, AuthenticationExpired):
+        elif is_temp_network_error(exc):
             return True
                 
         return False
@@ -415,7 +404,7 @@ class Backend(AbstractBackend):
                                                                   'limit': batch_size })
             except HTTPError as exc:
                 if exc.status == 404:
-                    raise DanglingStorageURL(self.container_name)
+                    raise DanglingStorageURLError(self.container_name)
                 raise
             
             if resp.status == 204:
@@ -498,7 +487,7 @@ class ObjectW(object):
             except:
                 log.exception('Objectw(%s).close(): unable to delete corrupted object!',
                               self.key)            
-            raise BadDigest('BadDigest', 'Received ETag does not agree with our calculations.')
+            raise BadDigestError('BadDigest', 'Received ETag does not agree with our calculations.')
 
     def __enter__(self):
         return self
@@ -562,7 +551,7 @@ class ObjectR(object):
             if etag != self.md5.hexdigest():
                 log.warn('ObjectR(%s).close(): MD5 mismatch: %s vs %s', self.key, etag,
                          self.md5.hexdigest())
-                raise BadDigest('BadDigest', 'ETag header does not agree with calculated MD5')
+                raise BadDigestError('BadDigest', 'ETag header does not agree with calculated MD5')
             return buf
 
         self.md5.update(buf)
@@ -603,4 +592,4 @@ class AuthenticationExpired(Exception):
     def __str__(self):
         return 'Auth token expired. Server said: %s' % self.msg
     
-        
\ No newline at end of file
+