getmoto / moto Goto Github PK

    def wait_for_state(self, state):
        found = False
        retries = 0
        while not found and retries < 15:
            self.ec2.update()
            if self.ec2.state == state:
                found = True
                break
            else:
                retries = retries + 1 
                logging.info("Waiting 5 seconds for host to change from %s to %s..." % (self.ec2.state, state))
                time.sleep(5)

        if not found:
            raise Exception("Instance did not change state to %s " % state)

bucket.get_key(name).size

@mock_s3
def test_get_key_size():
    conn = boto.connect_s3('the_key', 'the_secret')
    b = conn.create_bucket("test")
    k = Key(b, "hi")
    k.set_contents_from_string("hello")
    k.size.should.equal(5)
    b.get_key("hi")
    b.get_key("hi").size.should.equal(5)

conn = boto.ec2.connect_to_region(
        self.region, aws_access_key_id=self.access_key,
        aws_secret_access_key=self.secret_key)
reservations = conn.get_all_instances()

============================= test session starts ==============================
platform darwin -- Python 2.7.2 -- pytest-2.3.4
collected 9 items

tests/test_s3.py F

=================================== FAILURES ===================================
__________________________________ test_touch __________________________________

args = (), kwargs = {}

    def wrapper(*args, **kwargs):
        with self:
>           result = func(*args, **kwargs)

../../../../.virtualenvs/echo-compactor/src/moto/moto/core/models.py:47: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

    @mock_s3
    def test_touch():
        bucket = s3.get_bucket(create=True)

>       s3.touch(bucket, 'foo')

tests/test_s3.py:59: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

bucket = <Bucket: fireteam-test-lucian>, key_name = 'foo'

    def touch(bucket, key_name):
        key = bucket.new_key(key_name)
>       key.set_contents_from_string('')

compactor/s3.py:27: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <Key: fireteam-test-lucian,foo>, s = '', headers = None, replace = True
cb = None, num_cb = 10, policy = None, md5 = None, reduced_redundancy = False
encrypt_key = False

    def set_contents_from_string(self, s, headers=None, replace=True,
                                 cb=None, num_cb=10, policy=None, md5=None,
                                 reduced_redundancy=False,
                                 encrypt_key=False):
        """
            Store an object in S3 using the name of the Key object as the
            key in S3 and the string 's' as the contents.
            See set_contents_from_file method for details about the
            parameters.

            :type headers: dict
            :param headers: Additional headers to pass along with the
                request to AWS.

            :type replace: bool
            :param replace: If True, replaces the contents of the file if
                it already exists.

            :type cb: function
            :param cb: a callback function that will be called to report
                progress on the upload.  The callback should accept two
                integer parameters, the first representing the number of
                bytes that have been successfully transmitted to S3 and
                the second representing the size of the to be transmitted
                object.

            :type cb: int
            :param num_cb: (optional) If a callback is specified with the
                cb parameter this parameter determines the granularity of
                the callback by defining the maximum number of times the
                callback will be called during the file transfer.

            :type policy: :class:`boto.s3.acl.CannedACLStrings`
            :param policy: A canned ACL policy that will be applied to the
                new key in S3.

            :type md5: A tuple containing the hexdigest version of the MD5
                checksum of the file as the first element and the
                Base64-encoded version of the plain checksum as the second
                element.  This is the same format returned by the
                compute_md5 method.
            :param md5: If you need to compute the MD5 for any reason
                prior to upload, it's silly to have to do it twice so this
                param, if present, will be used as the MD5 values of the
                file.  Otherwise, the checksum will be computed.

            :type reduced_redundancy: bool
            :param reduced_redundancy: If True, this will set the storage
                class of the new Key to be REDUCED_REDUNDANCY. The Reduced
                Redundancy Storage (RRS) feature of S3, provides lower
                redundancy at lower storage cost.

            :type encrypt_key: bool
            :param encrypt_key: If True, the new copy of the object will
                be encrypted on the server-side by S3 and will be stored
                in an encrypted form while at rest in S3.
            """
        if isinstance(s, unicode):
            s = s.encode("utf-8")
        fp = StringIO.StringIO(s)
        r = self.set_contents_from_file(fp, headers, replace, cb, num_cb,
                                        policy, md5, reduced_redundancy,
>                                       encrypt_key=encrypt_key)

../../../../.virtualenvs/echo-compactor/lib/python2.7/site-packages/boto/s3/key.py:1253: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <Key: fireteam-test-lucian,foo>
fp = <StringIO.StringIO instance at 0x10d05ac68>, headers = {}, replace = True
cb = None, num_cb = 10, policy = None
md5 = ('d41d8cd98f00b204e9800998ecf8427e', '1B2M2Y8AsgTpgAmY7PhCfg==')
reduced_redundancy = False, query_args = None, encrypt_key = False, size = 0
rewind = False

    def set_contents_from_file(self, fp, headers=None, replace=True,
                               cb=None, num_cb=10, policy=None, md5=None,
                               reduced_redundancy=False, query_args=None,
                               encrypt_key=False, size=None, rewind=False):
        """
            Store an object in S3 using the name of the Key object as the
            key in S3 and the contents of the file pointed to by 'fp' as the
            contents. The data is read from 'fp' from its current position until
            'size' bytes have been read or EOF.

            :type fp: file
            :param fp: the file whose contents to upload

            :type headers: dict
            :param headers: Additional HTTP headers that will be sent with
                the PUT request.

            :type replace: bool
            :param replace: If this parameter is False, the method will
                first check to see if an object exists in the bucket with
                the same key.  If it does, it won't overwrite it.  The
                default value is True which will overwrite the object.

            :type cb: function
            :param cb: a callback function that will be called to report
                progress on the upload.  The callback should accept two
                integer parameters, the first representing the number of
                bytes that have been successfully transmitted to S3 and
                the second representing the size of the to be transmitted
                object.

            :type cb: int
            :param num_cb: (optional) If a callback is specified with the
                cb parameter this parameter determines the granularity of
                the callback by defining the maximum number of times the
                callback will be called during the file transfer.

            :type policy: :class:`boto.s3.acl.CannedACLStrings`
            :param policy: A canned ACL policy that will be applied to the
                new key in S3.

            :type md5: A tuple containing the hexdigest version of the MD5
                checksum of the file as the first element and the
                Base64-encoded version of the plain checksum as the second
                element.  This is the same format returned by the
                compute_md5 method.
            :param md5: If you need to compute the MD5 for any reason
                prior to upload, it's silly to have to do it twice so this
                param, if present, will be used as the MD5 values of the
                file.  Otherwise, the checksum will be computed.

            :type reduced_redundancy: bool
            :param reduced_redundancy: If True, this will set the storage
                class of the new Key to be REDUCED_REDUNDANCY. The Reduced
                Redundancy Storage (RRS) feature of S3, provides lower
                redundancy at lower storage cost.

            :type encrypt_key: bool
            :param encrypt_key: If True, the new copy of the object will
                be encrypted on the server-side by S3 and will be stored
                in an encrypted form while at rest in S3.

            :type size: int
            :param size: (optional) The Maximum number of bytes to read
                from the file pointer (fp). This is useful when uploading
                a file in multiple parts where you are splitting the file
                up into different ranges to be uploaded. If not specified,
                the default behaviour is to read all bytes from the file
                pointer. Less bytes may be available.

            :type rewind: bool
            :param rewind: (optional) If True, the file pointer (fp) will
                be rewound to the start before any bytes are read from
                it. The default behaviour is False which reads from the
                current position of the file pointer (fp).

            :rtype: int
            :return: The number of bytes written to the key.
            """
        provider = self.bucket.connection.provider
        headers = headers or {}
        if policy:
            headers[provider.acl_header] = policy
        if encrypt_key:
            headers[provider.server_side_encryption_header] = 'AES256'

        if rewind:
            # caller requests reading from beginning of fp.
            fp.seek(0, os.SEEK_SET)
        else:
            # The following seek/tell/seek logic is intended
            # to detect applications using the older interface to
            # set_contents_from_file(), which automatically rewound the
            # file each time the Key was reused. This changed with commit
            # 14ee2d03f4665fe20d19a85286f78d39d924237e, to support uploads
            # split into multiple parts and uploaded in parallel, and at
            # the time of that commit this check was added because otherwise
            # older programs would get a success status and upload an empty
            # object. Unfortuantely, it's very inefficient for fp's implemented
            # by KeyFile (used, for example, by gsutil when copying between
            # providers). So, we skip the check for the KeyFile case.
            # TODO: At some point consider removing this seek/tell/seek
            # logic, after enough time has passed that it's unlikely any
            # programs remain that assume the older auto-rewind interface.
            if not isinstance(fp, KeyFile):
                spos = fp.tell()
                fp.seek(0, os.SEEK_END)
                if fp.tell() == spos:
                    fp.seek(0, os.SEEK_SET)
                    if fp.tell() != spos:
                        # Raise an exception as this is likely a programming
                        # error whereby there is data before the fp but nothing
                        # after it.
                        fp.seek(spos)
                        raise AttributeError('fp is at EOF. Use rewind option '
                                             'or seek() to data start.')
                # seek back to the correct position.
                fp.seek(spos)

        if reduced_redundancy:
            self.storage_class = 'REDUCED_REDUNDANCY'
            if provider.storage_class_header:
                headers[provider.storage_class_header] = self.storage_class
                # TODO - What if provider doesn't support reduced reduncancy?
                # What if different providers provide different classes?
        if hasattr(fp, 'name'):
            self.path = fp.name
        if self.bucket != None:
            if not md5 and provider.supports_chunked_transfer():
                # defer md5 calculation to on the fly and
                # we don't know anything about size yet.
                chunked_transfer = True
                self.size = None
            else:
                chunked_transfer = False
                if isinstance(fp, KeyFile):
                    # Avoid EOF seek for KeyFile case as it's very inefficient.
                    key = fp.getkey()
                    size = key.size - fp.tell()
                    self.size = size
                    # At present both GCS and S3 use MD5 for the etag for
                    # non-multipart-uploaded objects. If the etag is 32 hex
                    # chars use it as an MD5, to avoid having to read the file
                    # twice while transferring.
                    if (re.match('^"[a-fA-F0-9]{32}"$', key.etag)):
                        etag = key.etag.strip('"')
                        md5 = (etag, base64.b64encode(binascii.unhexlify(etag)))
                if not md5:
                    # compute_md5() and also set self.size to actual
                    # size of the bytes read computing the md5.
                    md5 = self.compute_md5(fp, size)
                    # adjust size if required
                    size = self.size
                elif size:
                    self.size = size
                else:
                    # If md5 is provided, still need to size so
                    # calculate based on bytes to end of content
                    spos = fp.tell()
                    fp.seek(0, os.SEEK_END)
                    self.size = fp.tell() - spos
                    fp.seek(spos)
                    size = self.size
                self.md5 = md5[0]
                self.base64md5 = md5[1]

            if self.name == None:
                self.name = self.md5
            if not replace:
                if self.bucket.lookup(self.name):
                    return

            self.send_file(fp, headers=headers, cb=cb, num_cb=num_cb,
                           query_args=query_args,
>                          chunked_transfer=chunked_transfer, size=size)

../../../../.virtualenvs/echo-compactor/lib/python2.7/site-packages/boto/s3/key.py:1121: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <Key: fireteam-test-lucian,foo>
fp = <StringIO.StringIO instance at 0x10d05ac68>
headers = {'Content-Length': '0', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Type': 'application/octet-stream', 'Expect': '100-Continue', ...}
cb = None, num_cb = 10, query_args = None, chunked_transfer = False, size = 0

    def send_file(self, fp, headers=None, cb=None, num_cb=10,
                  query_args=None, chunked_transfer=False, size=None):
        """
            Upload a file to a key into a bucket on S3.

            :type fp: file
            :param fp: The file pointer to upload. The file pointer must
                point point at the offset from which you wish to upload.
                ie. if uploading the full file, it should point at the
                start of the file. Normally when a file is opened for
                reading, the fp will point at the first byte. See the
                bytes parameter below for more info.

            :type headers: dict
            :param headers: The headers to pass along with the PUT request

            :type cb: function
            :param cb: a callback function that will be called to report
                progress on the upload.  The callback should accept two
                integer parameters, the first representing the number of
                bytes that have been successfully transmitted to S3 and
                the second representing the size of the to be transmitted
                object.

            :type num_cb: int
            :param num_cb: (optional) If a callback is specified with the
                cb parameter this parameter determines the granularity of
                the callback by defining the maximum number of times the
                callback will be called during the file
                transfer. Providing a negative integer will cause your
                callback to be called with each buffer read.

            :type size: int
            :param size: (optional) The Maximum number of bytes to read
                from the file pointer (fp). This is useful when uploading
                a file in multiple parts where you are splitting the file
                up into different ranges to be uploaded. If not specified,
                the default behaviour is to read all bytes from the file
                pointer. Less bytes may be available.
            """
        provider = self.bucket.connection.provider
        try:
            spos = fp.tell()
        except IOError:
            spos = None
            self.read_from_stream = False

        def sender(http_conn, method, path, data, headers):
            # This function is called repeatedly for temporary retries
            # so we must be sure the file pointer is pointing at the
            # start of the data.
            if spos is not None and spos != fp.tell():
                fp.seek(spos)
            elif spos is None and self.read_from_stream:
                # if seek is not supported, and we've read from this
                # stream already, then we need to abort retries to
                # avoid setting bad data.
                raise provider.storage_data_error(
                    'Cannot retry failed request. fp does not support seeking.')

            http_conn.putrequest(method, path)
            for key in headers:
                http_conn.putheader(key, headers[key])
            http_conn.endheaders()

            # Calculate all MD5 checksums on the fly, if not already computed
            if not self.base64md5:
                m = md5()
            else:
                m = None

            save_debug = self.bucket.connection.debug
            self.bucket.connection.debug = 0
            # If the debuglevel < 3 we don't want to show connection
            # payload, so turn off HTTP connection-level debug output (to
            # be restored below).
            # Use the getattr approach to allow this to work in AppEngine.
            if getattr(http_conn, 'debuglevel', 0) < 3:
                http_conn.set_debuglevel(0)

            data_len = 0
            if cb:
                if size:
                    cb_size = size
                elif self.size:
                    cb_size = self.size
                else:
                    cb_size = 0
                if chunked_transfer and cb_size == 0:
                    # For chunked Transfer, we call the cb for every 1MB
                    # of data transferred, except when we know size.
                    cb_count = (1024 * 1024) / self.BufferSize
                elif num_cb > 1:
                    cb_count = int(math.ceil(cb_size / self.BufferSize / (num_cb - 1.0)))
                elif num_cb < 0:
                    cb_count = -1
                else:
                    cb_count = 0
                i = 0
                cb(data_len, cb_size)

            bytes_togo = size
            if bytes_togo and bytes_togo < self.BufferSize:
                chunk = fp.read(bytes_togo)
            else:
                chunk = fp.read(self.BufferSize)
            if spos is None:
                # read at least something from a non-seekable fp.
                self.read_from_stream = True
            while chunk:
                chunk_len = len(chunk)
                data_len += chunk_len
                if chunked_transfer:
                    http_conn.send('%x;\r\n' % chunk_len)
                    http_conn.send(chunk)
                    http_conn.send('\r\n')
                else:
                    http_conn.send(chunk)
                if m:
                    m.update(chunk)
                if bytes_togo:
                    bytes_togo -= chunk_len
                    if bytes_togo <= 0:
                        break
                if cb:
                    i += 1
                    if i == cb_count or cb_count == -1:
                        cb(data_len, cb_size)
                        i = 0
                if bytes_togo and bytes_togo < self.BufferSize:
                    chunk = fp.read(bytes_togo)
                else:
                    chunk = fp.read(self.BufferSize)

            self.size = data_len

            if m:
                # Use the chunked trailer for the digest
                hd = m.hexdigest()
                self.md5, self.base64md5 = self.get_md5_from_hexdigest(hd)

            if chunked_transfer:
                http_conn.send('0\r\n')
                    # http_conn.send("Content-MD5: %s\r\n" % self.base64md5)
                http_conn.send('\r\n')

            if cb and (cb_count <= 1 or i > 0) and data_len > 0:
                cb(data_len, cb_size)

            response = http_conn.getresponse()
            body = response.read()
            http_conn.set_debuglevel(save_debug)
            self.bucket.connection.debug = save_debug
            if ((response.status == 500 or response.status == 503 or
                    response.getheader('location')) and not chunked_transfer):
                # we'll try again.
                return response
            elif response.status >= 200 and response.status <= 299:
                self.etag = response.getheader('etag')
                if self.etag != '"%s"' % self.md5:
                    raise provider.storage_data_error(
                        'ETag from S3 did not match computed MD5')
                return response
            else:
                raise provider.storage_response_error(
                    response.status, response.reason, body)

        if not headers:
            headers = {}
        else:
            headers = headers.copy()
        headers['User-Agent'] = UserAgent
        if self.storage_class != 'STANDARD':
            headers[provider.storage_class_header] = self.storage_class
        if 'Content-Encoding' in headers:
            self.content_encoding = headers['Content-Encoding']
        if 'Content-Language' in headers:
            self.content_encoding = headers['Content-Language']
        if 'Content-Type' in headers:
            # Some use cases need to suppress sending of the Content-Type
            # header and depend on the receiving server to set the content
            # type. This can be achieved by setting headers['Content-Type']
            # to None when calling this method.
            if headers['Content-Type'] is None:
                # Delete null Content-Type value to skip sending that header.
                del headers['Content-Type']
            else:
                self.content_type = headers['Content-Type']
        elif self.path:
            self.content_type = mimetypes.guess_type(self.path)[0]
            if self.content_type == None:
                self.content_type = self.DefaultContentType
            headers['Content-Type'] = self.content_type
        else:
            headers['Content-Type'] = self.content_type
        if self.base64md5:
            headers['Content-MD5'] = self.base64md5
        if chunked_transfer:
            headers['Transfer-Encoding'] = 'chunked'
            #if not self.base64md5:
            #    headers['Trailer'] = "Content-MD5"
        else:
            headers['Content-Length'] = str(self.size)
        headers['Expect'] = '100-Continue'
        headers = boto.utils.merge_meta(headers, self.metadata, provider)
        resp = self.bucket.connection.make_request('PUT', self.bucket.name,
                                                   self.name, headers,
                                                   sender=sender,
>                                                  query_args=query_args)

../../../../.virtualenvs/echo-compactor/lib/python2.7/site-packages/boto/s3/key.py:827: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = S3Connection:s3.amazonaws.com, method = 'PUT'
bucket = 'fireteam-test-lucian', key = 'foo'
headers = {'Content-Length': '0', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Type': 'application/octet-stream', 'Expect': '100-Continue', ...}
data = '', query_args = None, sender = <function sender at 0x10d0916e0>
override_num_retries = None

    def make_request(self, method, bucket='', key='', headers=None, data='',
            query_args=None, sender=None, override_num_retries=None):
        if isinstance(bucket, self.bucket_class):
            bucket = bucket.name
        if isinstance(key, Key):
            key = key.name
        path = self.calling_format.build_path_base(bucket, key)
        boto.log.debug('path=%s' % path)
        auth_path = self.calling_format.build_auth_path(bucket, key)
        boto.log.debug('auth_path=%s' % auth_path)
        host = self.calling_format.build_host(self.server_name(), bucket)
        if query_args:
            path += '?' + query_args
            boto.log.debug('path=%s' % path)
            auth_path += '?' + query_args
            boto.log.debug('auth_path=%s' % auth_path)
        return AWSAuthConnection.make_request(self, method, path, headers,
                data, host, auth_path, sender,
>               override_num_retries=override_num_retries)

../../../../.virtualenvs/echo-compactor/lib/python2.7/site-packages/boto/s3/connection.py:490: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = S3Connection:s3.amazonaws.com, method = 'PUT', path = '/foo'
headers = {'Content-Length': '0', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Type': 'application/octet-stream', 'Expect': '100-Continue', ...}
data = '', host = 'fireteam-test-lucian.s3.amazonaws.com'
auth_path = '/fireteam-test-lucian/foo'
sender = <function sender at 0x10d0916e0>, override_num_retries = None
params = {}

    def make_request(self, method, path, headers=None, data='', host=None,
                     auth_path=None, sender=None, override_num_retries=None,
                     params=None):
        """Makes a request to the server, with stock multiple-retry logic."""
        if params is None:
            params = {}
        http_request = self.build_base_http_request(method, path, auth_path,
                                                    params, headers, data, host)
>       return self._mexe(http_request, sender, override_num_retries)

../../../../.virtualenvs/echo-compactor/lib/python2.7/site-packages/boto/connection.py:932: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = S3Connection:s3.amazonaws.com
request = <boto.connection.HTTPRequest object at 0x10d092850>
sender = <function sender at 0x10d0916e0>, override_num_retries = None
retry_handler = None

    def _mexe(self, request, sender=None, override_num_retries=None,
              retry_handler=None):
        """
            mexe - Multi-execute inside a loop, retrying multiple times to handle
                   transient Internet errors by simply trying again.
                   Also handles redirects.

            This code was inspired by the S3Utils classes posted to the boto-users
            Google group by Larry Bates.  Thanks!

            """
        boto.log.debug('Method: %s' % request.method)
        boto.log.debug('Path: %s' % request.path)
        boto.log.debug('Data: %s' % request.body)
        boto.log.debug('Headers: %s' % request.headers)
        boto.log.debug('Host: %s' % request.host)
        response = None
        body = None
        e = None
        if override_num_retries is None:
            num_retries = config.getint('Boto', 'num_retries', self.num_retries)
        else:
            num_retries = override_num_retries
        i = 0
        connection = self.get_http_connection(request.host, self.is_secure)
        while i <= num_retries:
            # Use binary exponential backoff to desynchronize client requests.
            next_sleep = random.random() * (2 ** i)
            try:
                # we now re-sign each request before it is retried
                boto.log.debug('Token: %s' % self.provider.security_token)
                request.authorize(connection=self)
                if callable(sender):
                    response = sender(connection, request.method, request.path,
>                                     request.body, request.headers)

../../../../.virtualenvs/echo-compactor/lib/python2.7/site-packages/boto/connection.py:832: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

http_conn = <httplib.HTTPSConnection instance at 0x10d07e518>, method = 'PUT'
path = '/foo', data = ''
headers = {'Authorization': 'AWS AKIAIE37U5TPZHCR54HA:lVFOGGJoI8vGypGdsjzqUwxzYhM=', 'Content-Length': '0', 'Content-MD5': '1B2M2Y8AsgTpgAmY7PhCfg==', 'Content-Type': 'application/octet-stream', ...}

    def sender(http_conn, method, path, data, headers):
        # This function is called repeatedly for temporary retries
        # so we must be sure the file pointer is pointing at the
        # start of the data.
        if spos is not None and spos != fp.tell():
            fp.seek(spos)
        elif spos is None and self.read_from_stream:
            # if seek is not supported, and we've read from this
            # stream already, then we need to abort retries to
            # avoid setting bad data.
            raise provider.storage_data_error(
                'Cannot retry failed request. fp does not support seeking.')

        http_conn.putrequest(method, path)
        for key in headers:
            http_conn.putheader(key, headers[key])
        http_conn.endheaders()

        # Calculate all MD5 checksums on the fly, if not already computed
        if not self.base64md5:
            m = md5()
        else:
            m = None

        save_debug = self.bucket.connection.debug
        self.bucket.connection.debug = 0
        # If the debuglevel < 3 we don't want to show connection
        # payload, so turn off HTTP connection-level debug output (to
        # be restored below).
        # Use the getattr approach to allow this to work in AppEngine.
        if getattr(http_conn, 'debuglevel', 0) < 3:
            http_conn.set_debuglevel(0)

        data_len = 0
        if cb:
            if size:
                cb_size = size
            elif self.size:
                cb_size = self.size
            else:
                cb_size = 0
            if chunked_transfer and cb_size == 0:
                # For chunked Transfer, we call the cb for every 1MB
                # of data transferred, except when we know size.
                cb_count = (1024 * 1024) / self.BufferSize
            elif num_cb > 1:
                cb_count = int(math.ceil(cb_size / self.BufferSize / (num_cb - 1.0)))
            elif num_cb < 0:
                cb_count = -1
            else:
                cb_count = 0
            i = 0
            cb(data_len, cb_size)

        bytes_togo = size
        if bytes_togo and bytes_togo < self.BufferSize:
            chunk = fp.read(bytes_togo)
        else:
            chunk = fp.read(self.BufferSize)
        if spos is None:
            # read at least something from a non-seekable fp.
            self.read_from_stream = True
        while chunk:
            chunk_len = len(chunk)
            data_len += chunk_len
            if chunked_transfer:
                http_conn.send('%x;\r\n' % chunk_len)
                http_conn.send(chunk)
                http_conn.send('\r\n')
            else:
                http_conn.send(chunk)
            if m:
                m.update(chunk)
            if bytes_togo:
                bytes_togo -= chunk_len
                if bytes_togo <= 0:
                    break
            if cb:
                i += 1
                if i == cb_count or cb_count == -1:
                    cb(data_len, cb_size)
                    i = 0
            if bytes_togo and bytes_togo < self.BufferSize:
                chunk = fp.read(bytes_togo)
            else:
                chunk = fp.read(self.BufferSize)

        self.size = data_len

        if m:
            # Use the chunked trailer for the digest
            hd = m.hexdigest()
            self.md5, self.base64md5 = self.get_md5_from_hexdigest(hd)

        if chunked_transfer:
            http_conn.send('0\r\n')
                # http_conn.send("Content-MD5: %s\r\n" % self.base64md5)
            http_conn.send('\r\n')

        if cb and (cb_count <= 1 or i > 0) and data_len > 0:
            cb(data_len, cb_size)

>       response = http_conn.getresponse()

../../../../.virtualenvs/echo-compactor/lib/python2.7/site-packages/boto/s3/key.py:768: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <httplib.HTTPSConnection instance at 0x10d07e518>, buffering = False

    def getresponse(self, buffering=False):
        "Get the response from the server."

        # if a prior response has been completed, then forget about it.
        if self.__response and self.__response.isclosed():
            self.__response = None

        #
        # if a prior response exists, then it must be completed (otherwise, we
        # cannot read this response's header to determine the connection-close
        # behavior)
        #
        # note: if a prior response existed, but was connection-close, then the
        # socket and response were made independent of this HTTPConnection
        # object since a new request requires that we open a whole new
        # connection
        #
        # this means the prior response had one of two states:
        #   1) will_close: this connection was reset and the prior socket and
        #                  response operate independently
        #   2) persistent: the response was retained and we await its
        #                  isclosed() status to become true.
        #
        if self.__state != _CS_REQ_SENT or self.__response:
            raise ResponseNotReady()

        args = (self.sock,)
        kwds = {"strict":self.strict, "method":self._method}
        if self.debuglevel > 0:
            args += (self.debuglevel,)
        if buffering:
            #only add this keyword if non-default, for compatibility with
            #other response_classes.
            kwds["buffering"] = True;
>       response = self.response_class(*args, **kwds)

/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py:1025: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <boto.connection.HTTPResponse instance at 0x10d0b2b48>
args = (<moto.packages.httpretty.socket object at 0x10d092490>,)
kwargs = {'method': 'PUT', 'strict': 0}

    def __init__(self, *args, **kwargs):
>       httplib.HTTPResponse.__init__(self, *args, **kwargs)

../../../../.virtualenvs/echo-compactor/lib/python2.7/site-packages/boto/connection.py:389: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <boto.connection.HTTPResponse instance at 0x10d0b2b48>
sock = <moto.packages.httpretty.socket object at 0x10d092490>, debuglevel = 0
strict = 0, method = 'PUT', buffering = False

    def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
        if buffering:
            # The caller won't be using any sock.recv() calls, so buffering
            # is fine and recommended for performance.
            self.fp = sock.makefile('rb')
        else:
            # The buffer size is specified as zero, because the headers of
            # the response are read with readline().  If the reads were
            # buffered the readline() calls could consume some of the
            # response, which make be read via a recv() on the underlying
            # socket.
>           self.fp = sock.makefile('rb', 0)

/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py:346: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <moto.packages.httpretty.socket object at 0x10d092490>, mode = 'rb'
bufsize = 0

    def makefile(self, mode='r', bufsize=-1):
        self._mode = mode
        self._bufsize = bufsize

        if self._entry:
>           self._entry.fill_filekind(self.fd, self._request)

../../../../.virtualenvs/echo-compactor/src/moto/moto/packages/httpretty.py:262: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <[TypeError("__repr__ returned non-string (type NoneType)") raised in repr()] SafeRepr object at 0x10d3d0488>
fk = <moto.packages.httpretty.FakeSockFile instance at 0x10d0930e0>
request = <[TypeError("__repr__ returned non-string (type NoneType)") raised in repr()] SafeRepr object at 0x10d3d0710>

    def fill_filekind(self, fk, request):
        now = datetime.utcnow()

        headers = {
            'status': self.status,
            'date': now.strftime('%a, %d %b %Y %H:%M:%S GMT'),
            'server': 'Python/HTTPretty',
            'connection': 'close',
        }

        if self.forcing_headers:
            headers = self.forcing_headers

        if self.dynamic_response:
            req_info, req_body, req_headers = request
            response = self.body(req_info, self.method, req_body, req_headers)
            if isinstance(response, basestring):
                body = response
            else:
>               body, new_headers = response
E               TypeError: 'NoneType' object is not iterable

../../../../.virtualenvs/echo-compactor/src/moto/moto/packages/httpretty.py:562: TypeError
===================== 8 tests deselected by '-ktest_touch' =====================
==================== 1 failed, 8 deselected in 0.83 seconds ====================

2) ERROR: test_postSimpleMessage (test_CSMessage.TestPostMessage)

   Traceback (most recent call last):
    /usr/local/Cellar/python/2.7.4/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/moto-0.1.4-py2.7.egg/moto/core/models.py line 47 in wrapper
      result = func(*args, **kwargs)
    test_CSMessage.py line 46 in test_postSimpleMessage
  mitem = table.get_item(hash_key=i['PK_MID'])
    /usr/local/lib/python2.7/site-packages/boto/dynamodb/table.py line 287 in get_item
  item_class)
    /usr/local/lib/python2.7/site-packages/boto/dynamodb/layer2.py line 487 in get_item
  object_hook=self.dynamizer.decode)
    /usr/local/lib/python2.7/site-packages/boto/dynamodb/layer1.py line 307 in get_item
  object_hook=object_hook)
    /usr/local/lib/python2.7/site-packages/boto/dynamodb/layer1.py line 118 in make_request
  retry_handler=self._retry_handler)
    /usr/local/lib/python2.7/site-packages/boto/connection.py line 845 in _mexe
  status = retry_handler(response, i, next_sleep)
    /usr/local/lib/python2.7/site-packages/boto/dynamodb/layer1.py line 158 in _retry_handler
  data)
   DynamoDBResponseError: DynamoDBResponseError: 400 Bad Request
   {u'__type': u'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException'}

def wrapper(*args, **kwargs):
    with self:
        result = func(*args, **kwargs)

moto_server s3 --port 23222

from boto.s3.connection import S3Connection, OrdinaryCallingFormat

conn = S3Connection('123', 'abc', is_secure=False, port=23222,
    host='localhost', calling_format=OrdinaryCallingFormat())
conn.create_bucket('test')

boto.exception.S3ResponseError: S3ResponseError: 404 NOT FOUND
None

127.0.0.1 - - [25/Oct/2013 15:59:22] "PUT /test/ HTTP/1.1" 404 -

elif method == 'PUT':
    ...
    return template.render(bucket=new_bucket)

elif method == 'DELETE':
    ...
    elif removed_bucket:
        ...
        return 204, headers, template.render(bucket=removed_bucket)

elif method == 'PUT':
    ...
    return 204, headers, template.render(bucket=removed_bucket)

127.0.0.1 - - [20/Dec/2013 16:30:33] "POST /123456789012/blah HTTP/1.1" 500 -
Error on request:
Traceback (most recent call last):
  File "/Users/zach/.virtualenvs/yipit_ph/lib/python2.7/site-packages/werkzeug/serving.py", line 177, in run_wsgi
    execute(self.server.app)
  File "/Users/zach/.virtualenvs/yipit_ph/lib/python2.7/site-packages/werkzeug/serving.py", line 165, in execute
    application_iter = app(environ, start_response)
  File "/Users/zach/.virtualenvs/yipit_ph/lib/python2.7/site-packages/flask/app.py", line 1836, in __call__
    return self.wsgi_app(environ, start_response)
  File "/Users/zach/.virtualenvs/yipit_ph/lib/python2.7/site-packages/flask/app.py", line 1820, in wsgi_app
    response = self.make_response(self.handle_exception(e))
  File "/Users/zach/.virtualenvs/yipit_ph/lib/python2.7/site-packages/flask/app.py", line 1403, in handle_exception
    reraise(exc_type, exc_value, tb)
  File "/Users/zach/.virtualenvs/yipit_ph/lib/python2.7/site-packages/flask/app.py", line 1817, in wsgi_app
    response = self.full_dispatch_request()
  File "/Users/zach/.virtualenvs/yipit_ph/lib/python2.7/site-packages/flask/app.py", line 1477, in full_dispatch_request
    rv = self.handle_user_exception(e)
  File "/Users/zach/.virtualenvs/yipit_ph/lib/python2.7/site-packages/flask/app.py", line 1381, in handle_user_exception
    reraise(exc_type, exc_value, tb)
  File "/Users/zach/.virtualenvs/yipit_ph/lib/python2.7/site-packages/flask/app.py", line 1475, in full_dispatch_request
    rv = self.dispatch_request()
  File "/Users/zach/.virtualenvs/yipit_ph/lib/python2.7/site-packages/flask/app.py", line 1461, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "/Users/zach/.virtualenvs/yipit_ph/lib/python2.7/site-packages/moto/core/utils.py", line 68, in __call__
    result = self.callback(request, request.url, headers)
  File "/Users/zach/.virtualenvs/yipit_ph/lib/python2.7/site-packages/moto/core/responses.py", line 32, in dispatch
    return self.call_action()
  File "/Users/zach/.virtualenvs/yipit_ph/lib/python2.7/site-packages/moto/core/responses.py", line 49, in call_action
    raise NotImplementedError("The {0} action has not been implemented".format(action))
NotImplementedError: The  action has not been implemented

@mock_s3
def test_key_size(self):
    conn = boto.connect_s3('the_key', 'the_secret')
    b = conn.create_bucket('test')
    k = Key(b, 'hi')
    k.set_contents_from_string('hello')
    k2 = Key(b, 'hi')
    self.assertEqual(k2.size, 5)

@mock_s3
def test_key_size_2(self):
    conn = boto.connect_s3('the_key', 'the_secret')
    b = conn.create_bucket('test')
    k = Key(b, 'hi')
    k.set_contents_from_string('hello')
    k2 = Key(b, 'hi')
    k2.get_contents_as_string()
    self.assertEqual(k2.size, 5)

/home/ubuntu/cloud-status/venv/local/lib/python2.7/site-packages/moto/s3/models.py:2: DeprecationWarning: the md5 module is deprecated; use hashlib instead
  import md5

>>> import boto
>>> sqs = boto.connect_sqs()
>>> q = sqs.create_queue('testqueue')
>>> q.write(q.new_message('foo bar baz'))
<boto.sqs.message.Message instance at 0x1023f2368>
>>> message = q.read(1)
>>> print(message.get_body())
foo bar baz

>>> import moto
>>> patch = moto.mock_sqs()
>>> patch.start()
>>> import boto
>>> sqs = boto.connect_sqs()
>>> q = sqs.create_queue('testqueue')
>>> q.write(q.new_message('foo bar baz'))
<boto.sqs.message.Message instance at 0x10f79c680>
>>> message = q.read(1)
>>> print(message.get_body())
Zm9vIGJhciBiYXo=

@mock_dynamodb
def test_query():
    conn = boto.connect_dynamodb()
    table = create_table(conn)

    item_data = {
        'Body': 'http://url_to_lolcat.gif',
        'SentBy': 'User A',
        'ReceivedTime': '12/9/2011 11:36:03 PM',
    }
    # the extra item
    item = table.new_item(
        hash_key='the-key1',
        range_key='4561',
        attrs=item_data,
    )
    item.put()

    item = table.new_item(
        hash_key='the-key',
        range_key='456',
        attrs=item_data,
    )
    item.put()

    item = table.new_item(
        hash_key='the-key',
        range_key='123',
        attrs=item_data,
    )
    item.put()

    item = table.new_item(
        hash_key='the-key',
        range_key='789',
        attrs=item_data,
    )
    item.put()

    results = table.query(hash_key='the-key', range_key_condition=condition.GT('1'))
    results.response['Items'].should.have.length_of(3)

[u'toplevel/x/key', u'toplevel/x/y/key', u'toplevel/x/y/z/key']
[u'toplevel/x/']

[u'toplevel/x/key', u'toplevel/x/y/key', u'toplevel/x/y/z/key']
[u'toplevel/x/key', u'toplevel/xy/']

ipdb> self.ec2.create_security_group('test', 'test', 'vpc_123')
SecurityGroup:test
ipdb> self.ec2.get_all_security_groups()
[SecurityGroup:test, SecurityGroup:test_vpc]
ipdb> self.ec2.get_all_security_groups()[0]
SecurityGroup:test
ipdb> self.ec2.get_all_security_groups()[0].vpc_id
''
ipdb> self.ec2.create_security_group('test1', 'test', 'vpc_123').vpc_id
'vpc_123'

tests/test_s3writer.py:10: in <module>
>   bucket = s3.get_bucket()
compactor/s3.py:19: in get_bucket
>       return connect_s3().get_bucket(config.AWS_BUCKET)
../../../../.virtualenvs/echo-compactor/lib/python2.7/site-packages/boto/s3/connection.py:409: in get_bucket
>           bucket.get_all_keys(headers, maxkeys=0)
../../../../.virtualenvs/echo-compactor/lib/python2.7/site-packages/boto/s3/bucket.py:371: in get_all_keys
>                            '', headers, **params)
../../../../.virtualenvs/echo-compactor/lib/python2.7/site-packages/boto/s3/bucket.py:334: in _get_all
>           xml.sax.parseString(body, h)
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/xml/sax/__init__.py:49: in parseString
>       parser.parse(inpsrc)
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/xml/sax/expatreader.py:107: in parse
>       xmlreader.IncrementalParser.parse(self, source)
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/xml/sax/xmlreader.py:123: in parse
>           self.feed(buffer)
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/xml/sax/expatreader.py:211: in feed
>           self._err_handler.fatalError(exc)
/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/xml/sax/handler.py:38: in fatalError
>       raise exception
E       SAXParseException: <unknown>:1:0: not well-formed (invalid token)