# Copyright (c) 2010-2012 OpenStack, LLC.

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or

# implied.

# See the License for the specific language governing permissions and

# limitations under the License.

"""

OpenStack Swift client library used internally

"""

import socket

import re

import logging

from urllib3.exceptions import HTTPError as urllib_http_error

import warnings

from requests.exceptions import RequestException, SSLError

import http.client as http_client

from requests.structures import CaseInsensitiveDict

from urllib.parse import quote, unquote

from urllib.parse import urljoin, urlparse, urlunparse

from time import sleep, time

from swiftclient import version as swiftclient_version

from swiftclient.exceptions import ClientException

from swiftclient.requests_compat import SwiftClientRequestsSession

from swiftclient.utils import (

iter_wrapper, LengthWrapper, ReadableToIterable, parse_api_response,

get_body)

# Default is 100, increase to 256

http_client._MAXHEADERS = 256

VERSIONFUL_AUTH_PATH = re.compile(r'v[2-3](?:\.0)?$')

AUTH_VERSIONS_V1 = ('1.0', '1', 1)

AUTH_VERSIONS_V2 = ('2.0', '2', 2)

AUTH_VERSIONS_V3 = ('3.0', '3', 3)

USER_METADATA_TYPE = tuple('x-%s-meta-' % type_ for type_ in

('container', 'account', 'object'))

URI_PATTERN_INFO = re.compile(r'/info')

URI_PATTERN_VERSION = re.compile(r'\/v\d+\.?\d*(\/.*)?')

ksexceptions = ksclient_v2 = ksclient_v3 = ksa_v3 = None

try:

from keystoneclient import exceptions as ksexceptions

# prevent keystoneclient warning us that it has no log handlers

logging.getLogger('keystoneclient').addHandler(logging.NullHandler())

from keystoneclient.v2_0 import client as ksclient_v2

except ImportError:

pass

try:

from keystoneclient.v3 import client as ksclient_v3

from keystoneauth1.identity import v3 as ksa_v3

from keystoneauth1 import session as ksa_session

from keystoneauth1 import exceptions as ksauthexceptions

except ImportError:

pass

logger = logging.getLogger("swiftclient")

logger.addHandler(logging.NullHandler())

#: Default behaviour is to redact header values known to contain secrets,

#: such as ``X-Auth-Key`` and ``X-Auth-Token``. Up to the first 16 chars

#: may be revealed.

#:

#: To disable, set the value of ``redact_sensitive_headers`` to ``False``.

#:

#: When header redaction is enabled, ``reveal_sensitive_prefix`` configures the

#: maximum length of any sensitive header data sent to the logs. If the header

#: is less than twice this length, only ``int(len(value)/2)`` chars will be

#: logged; if it is less than 15 chars long, even less will be logged.

logger_settings = {

'redact_sensitive_headers': True,

'reveal_sensitive_prefix': 16

}

#: A list of sensitive headers to redact in logs. Note that when extending this

#: list, the header names must be added in all lower case.

LOGGER_SENSITIVE_HEADERS = [

'x-auth-token', 'x-auth-key', 'x-service-token', 'x-storage-token',

'x-account-meta-temp-url-key', 'x-account-meta-temp-url-key-2',

'x-container-meta-temp-url-key', 'x-container-meta-temp-url-key-2',

'set-cookie'

]

def safe_value(name, value):

"""

Only show up to logger_settings['reveal_sensitive_prefix'] characters

from a sensitive header.

:param name: Header name

:param value: Header value

:return: Safe header value

"""

if name.lower() in LOGGER_SENSITIVE_HEADERS:

prefix_length = logger_settings.get('reveal_sensitive_prefix', 16)

prefix_length = int(

min(prefix_length, (len(value) ** 2) / 32, len(value) / 2)

)

redacted_value = value[0:prefix_length]

return redacted_value + '...'

return value

def scrub_headers(headers):

"""

Redact header values that can contain sensitive information that

should not be logged.

:param headers: Either a dict or an iterable of two-element tuples

:return: Safe dictionary of headers with sensitive information removed

"""

if isinstance(headers, dict):

headers = headers.items()

headers = [

(parse_header_string(key), parse_header_string(val))

for (key, val) in headers

]

if not logger_settings.get('redact_sensitive_headers', True):

return dict(headers)

if logger_settings.get('reveal_sensitive_prefix', 16) < 0:

logger_settings['reveal_sensitive_prefix'] = 16

return {key: safe_value(key, val) for (key, val) in headers}

def http_log(args, kwargs, resp, body):

if not logger.isEnabledFor(logging.INFO):

return

# create and log equivalent curl command

string_parts = ['curl -i']

for element in args:

if element == 'HEAD':

string_parts.append(' -I')

elif element in ('GET', 'POST', 'PUT'):

string_parts.append(' -X %s' % element)

else:

string_parts.append(' %s' % parse_header_string(element))

if 'headers' in kwargs:

headers = scrub_headers(kwargs['headers'])

for element in headers:

header = ' -H "%s: %s"' % (element, headers[element])

string_parts.append(header)

# log response as debug if good, or info if error

if resp.status < 300:

log_method = logger.debug

else:

log_method = logger.info

log_method("REQ: %s", "".join(string_parts))

log_method("RESP STATUS: %s %s", resp.status, resp.reason)

log_method("RESP HEADERS: %s", scrub_headers(resp.getheaders()))

if body:

resp_headers = resp_header_dict(resp)

nbody = get_body(resp_headers, body)

log_method("RESP BODY: %s", nbody)

def parse_header_string(data):

if not isinstance(data, (str, bytes)):

data = str(data)

if isinstance(data, bytes):

# Under Python3 requests only returns text_type and tosses (!) the

# rest of the headers. If that ever changes, this should be a sane

# approach.

try:

data = data.decode('ascii')

except UnicodeDecodeError:

data = quote(data)

try:

unquoted = unquote(data, errors='strict')

except UnicodeDecodeError:

return data

return unquoted

def encode_utf8(value):

if type(value) in (int, float, bool):

# As of requests 2.11.0, headers must be byte- or unicode-strings.

# Convert some known-good types as a convenience for developers.

# Note that we *don't* convert subclasses, as they may have overriddden

# __str__ or __repr__.

# See https://github.com/kennethreitz/requests/pull/3366 for more info

value = str(value)

if isinstance(value, str):

value = value.encode('utf8')

return value

def encode_meta_headers(headers):

"""Only encode metadata headers keys"""

ret = {}

for header, value in headers.items():

value = encode_utf8(value)

header = header.lower()

if (isinstance(header, str) and

header.startswith(USER_METADATA_TYPE)):

header = encode_utf8(header)

ret[header] = value

return ret

class LowerKeyCaseInsensitiveDict(CaseInsensitiveDict):

"""

CaseInsensitiveDict returning lower case keys for items()

"""

def __iter__(self):

return iter(self._store.keys())

class _ObjectBody:

"""

Readable and iterable object body response wrapper.

"""

def __init__(self, resp, chunk_size, conn_to_close):

"""

Wrap the underlying response

:param resp: the response to wrap

:param chunk_size: number of bytes to return each iteration/next call

"""

self.resp = resp

self.chunk_size = chunk_size

self.conn_to_close = conn_to_close

def read(self, length=None):

buf = self.resp.read(length)

if length != 0 and not buf:

self.close()

return buf

def __iter__(self):

return self

def next(self):

buf = self.read(self.chunk_size)

if not buf:

raise StopIteration()

return buf

def __next__(self):

return self.next()

def close(self):

self.resp.close()

if self.conn_to_close:

self.conn_to_close.close()

class _RetryBody(_ObjectBody):

"""

Wrapper for object body response which triggers a retry

(from offset) if the connection is dropped after partially

downloading the object.

"""

def __init__(self, resp, connection, container, obj,

resp_chunk_size=None, query_string=None, response_dict=None,

headers=None):

"""

Wrap the underlying response

:param resp: the response to wrap

:param connection: Connection class instance

:param container: the name of the container the object is in

:param obj: the name of object we are downloading

:param resp_chunk_size: if defined, chunk size of data to read

:param query_string: if set will be appended with '?' to generated path

:param response_dict: an optional dictionary into which to place

the response - status, reason and headers

:param headers: an optional dictionary with additional headers to

include in the request

"""

super(_RetryBody, self).__init__(resp, resp_chunk_size, None)

self.expected_length = int(self.resp.getheader('Content-Length'))

self.conn = connection

self.container = container

self.obj = obj

self.query_string = query_string

self.response_dict = response_dict

self.headers = dict(headers) if headers is not None else {}

self.bytes_read = 0

def read(self, length=None):

buf = None

try:

buf = self.resp.read(length)

self.bytes_read += len(buf)

except (socket.error, urllib_http_error, RequestException):

if self.conn.attempts > self.conn.retries:

raise

if (not buf and self.bytes_read < self.expected_length and

self.conn.attempts <= self.conn.retries):

self.headers['Range'] = 'bytes=%d-' % self.bytes_read

self.headers['If-Match'] = self.resp.getheader('ETag')

hdrs, body = self.conn._retry(None, get_object,

self.container, self.obj,

resp_chunk_size=self.chunk_size,

query_string=self.query_string,

response_dict=self.response_dict,

headers=self.headers,

attempts=self.conn.attempts)

expected_range = 'bytes %d-%d/%d' % (

self.bytes_read,

self.expected_length - 1,

self.expected_length)

if 'content-range' not in hdrs:

# Server didn't respond with partial content; manually seek

logger.warning('Received 200 while retrying %s/%s; seeking...',

self.container, self.obj)

to_read = self.bytes_read

while to_read > 0:

buf = body.resp.read(min(to_read, self.chunk_size))

to_read -= len(buf)

elif hdrs['content-range'] != expected_range:

msg = ('Expected range "%s" while retrying %s/%s '

'but got "%s"' % (expected_range, self.container,

self.obj, hdrs['content-range']))

raise ClientException(msg)

self.resp = body.resp

buf = self.read(length)

return buf

class HTTPConnection:

def __init__(self, url, proxy=None, cacert=None, insecure=False,

cert=None, cert_key=None, ssl_compression=False,

default_user_agent=None, timeout=None):

"""

Make an HTTPConnection or HTTPSConnection

:param url: url to connect to

:param proxy: proxy to connect through, if any; None by default; str

of the format 'http://127.0.0.1:8888' to set one

:param cacert: A CA bundle file to use in verifying a TLS server

certificate.

:param insecure: Allow to access servers without checking SSL certs.

The server's certificate will not be verified.

:param cert: Client certificate file to connect on SSL server

requiring SSL client certificate.

:param cert_key: Client certificate private key file.

:param ssl_compression: SSL compression should be disabled by default

and this setting is not usable as of now. The

parameter is kept for backward compatibility.

:param default_user_agent: Set the User-Agent header on every request.

If set to None (default), the user agent

will be "python-swiftclient-<version>". This

may be overridden on a per-request basis by

explicitly setting the user-agent header on

a call to request().

:param timeout: socket read timeout value, passed directly to

the requests library.

:raises ClientException: Unable to handle protocol scheme

"""

self.url = url

self.parsed_url = urlparse(url)

self.host = self.parsed_url.netloc

self.port = self.parsed_url.port

self.requests_args = {}

self.request_session = SwiftClientRequestsSession()

# Don't use requests's default headers

self.request_session.headers = None

self.resp = None

if self.parsed_url.scheme not in ('http', 'https'):

raise ClientException('Unsupported scheme "%s" in url "%s"'

% (self.parsed_url.scheme, url))

self.requests_args['verify'] = not insecure

if cacert and not insecure:

# verify requests parameter is used to pass the CA_BUNDLE file

# see: http://docs.python-requests.org/en/latest/user/advanced/

self.requests_args['verify'] = cacert

if cert:

# NOTE(cbrandily): cert requests parameter is used to pass client

# cert path or a tuple with client certificate/key paths.

if cert_key:

self.requests_args['cert'] = cert, cert_key

else:

self.requests_args['cert'] = cert

if proxy:

proxy_parsed = urlparse(proxy)

if not proxy_parsed.scheme:

raise ClientException("Proxy's missing scheme")

self.requests_args['proxies'] = {

proxy_parsed.scheme: '%s://%s' % (

proxy_parsed.scheme, proxy_parsed.netloc

)

}

self.requests_args['stream'] = True

if default_user_agent is None:

default_user_agent = \

'python-swiftclient-%s' % swiftclient_version.version_string

self.default_user_agent = default_user_agent

if timeout:

self.requests_args['timeout'] = timeout

def _request(self, *arg, **kwarg):

"""Final wrapper before requests call, to be patched in tests"""

return self.request_session.request(*arg, **kwarg)

def request(self, method, full_path, data=None, headers=None, files=None):

"""Encode url and header, then call requests.request"""

if headers is None:

headers = {}

else:

headers = encode_meta_headers(headers)

# set a default User-Agent header if it wasn't passed in

if 'user-agent' not in headers:

headers['user-agent'] = self.default_user_agent

url = "%s://%s%s" % (

self.parsed_url.scheme,

self.parsed_url.netloc,

full_path)

self.resp = self._request(method, url, headers=headers, data=data,

files=files, **self.requests_args)

return self.resp

def putrequest(self, full_path, data=None, headers=None, files=None):

"""

Use python-requests files upload

:param data: Use data generator for chunked-transfer

:param files: Use files for default transfer

"""

return self.request('PUT', full_path, data, headers, files)

def getresponse(self):

"""Adapt requests response to httplib interface"""

self.resp.status = self.resp.status_code

old_getheader = self.resp.headers.get

def _decode_header(string):

if string is None:

return string

return string.encode('iso-8859-1').decode('utf-8')

def _encode_header(string):

if string is None:

return string

return string.encode('utf-8').decode('iso-8859-1')

def getheaders():

return [(_decode_header(k), _decode_header(v))

for k, v in self.resp.headers.items()]

def getheader(k, v=None):

return _decode_header(old_getheader(

_encode_header(k.lower()), _encode_header(v)))

def releasing_read(*args, **kwargs):

chunk = self.resp.raw.read(*args, **kwargs)

if not chunk:

# NOTE(sigmavirus24): Release the connection back to the

# urllib3's connection pool. This will reduce the number of

# log messages seen in bug #1341777. This does not actually

# close a socket. It will also prevent people from being

# misled as to the cause of a bug as in bug #1424732.

self.resp.close()

return chunk

self.resp.getheaders = getheaders

self.resp.getheader = getheader

self.resp.read = releasing_read

return self.resp

def close(self):

if self.resp:

self.resp.close()

self.request_session.close()

def http_connection(*arg, **kwarg):

""":returns: tuple of (parsed url, connection object)"""

conn = HTTPConnection(*arg, **kwarg)

return conn.parsed_url, conn

def get_auth_1_0(url, user, key, **kwargs):

cacert = kwargs.get('cacert', None)

insecure = kwargs.get('insecure', False)

cert = kwargs.get('cert')

cert_key = kwargs.get('cert_key')

timeout = kwargs.get('timeout', None)

parsed, conn = http_connection(url, cacert=cacert, insecure=insecure,

cert=cert, cert_key=cert_key,

timeout=timeout)

method = 'GET'

headers = {'X-Auth-User': user, 'X-Auth-Key': key}

conn.request(method, parsed.path, '', headers)

resp = conn.getresponse()

body = resp.read()

resp.close()

conn.close()

http_log((url, method,), headers, resp, body)

url = resp.getheader('x-storage-url')

# There is a side-effect on current Rackspace 1.0 server where a

# bad URL would get you that document page and a 200. We error out

# if we don't have a x-storage-url header and if we get a body.

if resp.status < 200 or resp.status >= 300 or (body and not url):

raise ClientException.from_response(resp, 'Auth GET failed', body)

token = resp.getheader('x-storage-token', resp.getheader('x-auth-token'))

return url, token

def get_keystoneclient_2_0(auth_url, user, key, os_options, **kwargs):

# this function is only here to preserve the historic 'public'

# interface of this module

kwargs.update({'auth_version': '2.0'})

return get_auth_keystone(auth_url, user, key, os_options, **kwargs)

def get_auth_keystone(auth_url, user, key, os_options, **kwargs):

"""

Authenticate against a keystone server.

We are using the keystoneclient library for authentication.

"""

insecure = kwargs.get('insecure', False)

timeout = kwargs.get('timeout', None)

auth_version = kwargs.get('auth_version', None)

debug = logger.isEnabledFor(logging.DEBUG)

# Add the version suffix in case of versionless Keystone endpoints. If

# auth_version is also unset it is likely that it is v3

if not VERSIONFUL_AUTH_PATH.match(

urlparse(auth_url).path.rstrip('/').rsplit('/', 1)[-1]):

# Normalize auth_url to end in a slash because urljoin

auth_url = auth_url.rstrip('/') + '/'

if auth_version and auth_version in AUTH_VERSIONS_V2:

auth_url = urljoin(auth_url, "v2.0")

else:

auth_url = urljoin(auth_url, "v3")

auth_version = '3'

logger.debug("Versionless auth_url - using %s as endpoint" % auth_url)

# Legacy default if not set

if auth_version is None:

auth_version = '2'

ksclient = None

if auth_version in AUTH_VERSIONS_V3:

if ksclient_v3 is not None:

ksclient = ksclient_v3

else:

if ksclient_v2 is not None:

ksclient = ksclient_v2

if ksclient is None:

raise ClientException('''

Auth versions 2.0 and 3 require python-keystoneclient, install it or use Auth

version 1.0 which requires ST_AUTH, ST_USER, and ST_KEY environment

variables to be set or overridden with -A, -U, or -K.''')

filter_kwargs = {}

service_type = os_options.get('service_type') or 'object-store'

endpoint_type = os_options.get('endpoint_type') or 'public'

if os_options.get('region_name'):

filter_kwargs['attr'] = 'region'

filter_kwargs['filter_value'] = os_options['region_name']

if os_options.get('auth_type') and os_options['auth_type'] not in (

'password', 'v2password', 'v3password',

'v3applicationcredential'):

raise ClientException(

'Swiftclient currently only supports v3applicationcredential '

'for auth_type')

elif os_options.get('auth_type') == 'v3applicationcredential':

if ksa_v3 is None:

raise ClientException('Auth v3applicationcredential requires '

'keystoneauth1 package; consider upgrading '

'to python-keystoneclient>=2.0.0')

try:

auth = ksa_v3.ApplicationCredential(

auth_url=auth_url,

application_credential_secret=os_options.get(

'application_credential_secret'),

application_credential_id=os_options.get(

'application_credential_id'))

sess = ksa_session.Session(auth=auth, timeout=timeout)

token = sess.get_token()

except ksauthexceptions.Unauthorized:

msg = 'Unauthorized. Check application credential id and secret.'

raise ClientException(msg)

except ksauthexceptions.AuthorizationFailure as err:

raise ClientException('Authorization Failure. %s' % err)

try:

endpoint = sess.get_endpoint_data(service_type=service_type,

endpoint_type=endpoint_type,

**filter_kwargs)

return endpoint.catalog_url, token

except ksauthexceptions.EndpointNotFound:

raise ClientException(

'Endpoint for %s not found - '

'have you specified a region?' % service_type)

try:

_ksclient = ksclient.Client(

username=user,

password=key,

token=os_options.get('auth_token'),

tenant_name=os_options.get('tenant_name'),

tenant_id=os_options.get('tenant_id'),

user_id=os_options.get('user_id'),

user_domain_name=os_options.get('user_domain_name'),

user_domain_id=os_options.get('user_domain_id'),

project_name=os_options.get('project_name'),

project_id=os_options.get('project_id'),

project_domain_name=os_options.get('project_domain_name'),

project_domain_id=os_options.get('project_domain_id'),

debug=debug,

cacert=kwargs.get('cacert'),

cert=kwargs.get('cert'),

key=kwargs.get('cert_key'),

auth_url=auth_url, insecure=insecure, timeout=timeout)

except ksexceptions.Unauthorized:

msg = 'Unauthorized. Check username, password and tenant name/id.'

if auth_version in AUTH_VERSIONS_V3:

msg = ('Unauthorized. Check username/id, password, '

'tenant name/id and user/tenant domain name/id.')

raise ClientException(msg)

except ksexceptions.AuthorizationFailure as err:

raise ClientException('Authorization Failure. %s' % err)

try:

endpoint = _ksclient.service_catalog.url_for(

service_type=service_type,

endpoint_type=endpoint_type,

**filter_kwargs)

except ksexceptions.EndpointNotFound:

raise ClientException('Endpoint for %s not found - '

'have you specified a region?' % service_type)

return endpoint, _ksclient.auth_token

def get_auth(auth_url, user, key, **kwargs):

"""

Get authentication/authorization credentials.

:kwarg auth_version: the api version of the supplied auth params

:kwarg os_options: a dict, the openstack identity service options

:returns: a tuple, (storage_url, token)

N.B. if the optional os_options parameter includes a non-empty

'object_storage_url' key it will override the default storage url returned

by the auth service.

"""

session = kwargs.get('session', None)

auth_version = kwargs.get('auth_version', '1')

os_options = kwargs.get('os_options', {})

cacert = kwargs.get('cacert', None)

insecure = kwargs.get('insecure', False)

cert = kwargs.get('cert')

cert_key = kwargs.get('cert_key')

timeout = kwargs.get('timeout', None)

if session:

service_type = os_options.get('service_type', 'object-store')

interface = os_options.get('endpoint_type', 'public')

region_name = os_options.get('region_name')

storage_url = session.get_endpoint(service_type=service_type,

interface=interface,

region_name=region_name)

token = session.get_token()

elif auth_version in AUTH_VERSIONS_V1:

storage_url, token = get_auth_1_0(auth_url,

user,

key,

cacert=cacert,

insecure=insecure,

cert=cert,

cert_key=cert_key,

timeout=timeout)

elif auth_version in AUTH_VERSIONS_V2 + AUTH_VERSIONS_V3:

# We are handling a special use case here where the user argument

# specifies both the user name and tenant name in the form tenant:user

if user and not kwargs.get('tenant_name') and ':' in user:

os_options['tenant_name'], user = user.split(':')

# We are allowing to have a tenant_name argument in get_auth

# directly without having os_options

if kwargs.get('tenant_name'):

os_options['tenant_name'] = kwargs['tenant_name']

if os_options.get('auth_type') == 'v3applicationcredential':

pass

elif not (os_options.get('tenant_name') or

os_options.get('tenant_id') or

os_options.get('project_name') or

os_options.get('project_id')):

if auth_version in AUTH_VERSIONS_V2:

raise ClientException('No tenant specified')

raise ClientException('No project name or project id specified.')

storage_url, token = get_auth_keystone(auth_url, user,

key, os_options,

cacert=cacert,

insecure=insecure,

cert=cert,

cert_key=cert_key,

timeout=timeout,

auth_version=auth_version)

else:

raise ClientException('Unknown auth_version %s specified and no '

'session found.' % auth_version)

# Override storage url, if necessary

if os_options.get('object_storage_url'):

return os_options['object_storage_url'], token

else:

return storage_url, token

def resp_header_dict(resp):

resp_headers = LowerKeyCaseInsensitiveDict()

for header, value in resp.getheaders():

header = parse_header_string(header)

resp_headers[header] = parse_header_string(value)

return resp_headers

def store_response(resp, response_dict):

"""

store information about an operation into a dict

:param resp: an http response object containing the response

headers

:param response_dict: a dict into which are placed the

status, reason and a dict of lower-cased headers

"""

if response_dict is not None:

response_dict['status'] = resp.status

response_dict['reason'] = resp.reason

response_dict['headers'] = resp_header_dict(resp)

def get_account(url, token, marker=None, limit=None, prefix=None,

end_marker=None, http_conn=None, full_listing=False,

service_token=None, headers=None, delimiter=None):

"""

Get a listing of containers for the account.

:param url: storage URL

:param token: auth token

:param marker: marker query

:param limit: limit query

:param prefix: prefix query

:param end_marker: end_marker query

:param http_conn: a tuple of (parsed url, HTTPConnection object),

(If None, it will create the conn object)

:param full_listing: if True, return a full listing, else returns a max

of 10000 listings

:param service_token: service auth token

:param headers: additional headers to include in the request

:param delimiter: delimiter query

:returns: a tuple of (response headers, a list of containers) The response

headers will be a dict and all header names will be lowercase.

:raises ClientException: HTTP GET request failed

"""

req_headers = {'X-Auth-Token': token, 'Accept-Encoding': 'gzip'}

if service_token:

req_headers['X-Service-Token'] = service_token

if headers:

req_headers.update(headers)

close_conn = False

if not http_conn:

http_conn = http_connection(url)

close_conn = True

if full_listing:

rv = get_account(url, token, marker, limit, prefix, end_marker,

http_conn, headers=req_headers, delimiter=delimiter)

listing = rv[1]

while listing:

marker = listing[-1]['name']

listing = get_account(url, token, marker, limit, prefix,

end_marker, http_conn, headers=req_headers,

delimiter=delimiter)[1]

if listing:

rv[1].extend(listing)

return rv

parsed, conn = http_conn

qs = 'format=json'

if marker:

qs += '&marker=%s' % quote(marker)

if limit:

qs += '&limit=%d' % limit

if prefix:

qs += '&prefix=%s' % quote(prefix)

if delimiter:

qs += '&delimiter=%s' % quote(delimiter)

if end_marker:

qs += '&end_marker=%s' % quote(end_marker)

full_path = '%s?%s' % (parsed.path, qs)

method = 'GET'

conn.request(method, full_path, '', req_headers)

resp = conn.getresponse()

body = resp.read()

if close_conn:

conn.close()

http_log(("%s?%s" % (url, qs), method,), {'headers': req_headers},

resp, body)

resp_headers = resp_header_dict(resp)

if resp.status < 200 or resp.status >= 300:

raise ClientException.from_response(resp, 'Account GET failed', body)

if resp.status == 204:

return resp_headers, []

return resp_headers, parse_api_response(resp_headers, body)

def head_account(url, token, http_conn=None, headers=None,

service_token=None):

"""

Get account stats.

:param url: storage URL

:param token: auth token

:param http_conn: a tuple of (parsed url, HTTPConnection object),

(If None, it will create the conn object)

:param headers: additional headers to include in the request

:param service_token: service auth token

:returns: a dict containing the response's headers (all header names will

be lowercase)

:raises ClientException: HTTP HEAD request failed

"""

close_conn = False

if http_conn:

parsed, conn = http_conn

else:

parsed, conn = http_connection(url)

close_conn = True

method = "HEAD"

req_headers = {'X-Auth-Token': token}

if service_token:

req_headers['X-Service-Token'] = service_token

if headers:

req_headers.update(headers)

conn.request(method, parsed.path, '', req_headers)

resp = conn.getresponse()

body = resp.read()

if close_conn:

conn.close()

http_log((url, method,), {'headers': req_headers}, resp, body)

if resp.status < 200 or resp.status >= 300:

raise ClientException.from_response(resp, 'Account HEAD failed', body)

resp_headers = resp_header_dict(resp)

return resp_headers

def post_account(url, token, headers, http_conn=None, response_dict=None,

service_token=None, query_string=None, data=None):

"""

Update an account's metadata.

:param url: storage URL

:param token: auth token

:param headers: additional headers to include in the request

:param http_conn: a tuple of (parsed url, HTTPConnection object),

(If None, it will create the conn object)

:param response_dict: an optional dictionary into which to place

the response - status, reason and headers

:param service_token: service auth token

:param query_string: if set will be appended with '?' to generated path

:param data: an optional message body for the request

:raises ClientException: HTTP POST request failed

:returns: resp_headers, body

"""

close_conn = False

if http_conn:

parsed, conn = http_conn

else:

parsed, conn = http_connection(url)

close_conn = True

method = 'POST'

path = parsed.path

if query_string:

path += '?' + query_string

req_headers = {'X-Auth-Token': token}

if service_token:

req_headers['X-Service-Token'] = service_token

if headers:

req_headers.update(headers)

conn.request(method, path, data, req_headers)

resp = conn.getresponse()

body = resp.read()

if close_conn:

conn.close()

http_log((url, method,), {'headers': req_headers}, resp, body)

store_response(resp, response_dict)

if resp.status < 200 or resp.status >= 300:

raise ClientException.from_response(resp, 'Account POST failed', body)

resp_headers = {}

for header, value in resp.getheaders():

resp_headers[header.lower()] = value

return resp_headers, body

def get_container(url, token, container, marker=None, limit=None,

prefix=None, delimiter=None, end_marker=None,

version_marker=None, path=None, http_conn=None,

full_listing=False, service_token=None, headers=None,

query_string=None):

"""

Get a listing of objects for the container.

:param url: storage URL

:param token: auth token

:param container: container name to get a listing for

:param marker: marker query

:param limit: limit query

:param prefix: prefix query

:param delimiter: string to delimit the queries on

:param end_marker: marker query

:param version_marker: version marker query

:param path: path query (equivalent: "delimiter=/" and "prefix=path/")

:param http_conn: a tuple of (parsed url, HTTPConnection object),

(If None, it will create the conn object)

:param full_listing: if True, return a full listing, else returns a max

of 10000 listings

:param service_token: service auth token

:param headers: additional headers to include in the request

:param query_string: if set will be appended with '?' to generated path

:returns: a tuple of (response headers, a list of objects) The response

headers will be a dict and all header names will be lowercase.

:raises ClientException: HTTP GET request failed

"""

close_conn = False

if not http_conn:

http_conn = http_connection(url)

close_conn = True

if full_listing:

rv = get_container(url, token, container, marker, limit, prefix,

delimiter, end_marker, version_marker, path=path,

http_conn=http_conn, service_token=service_token,

headers=headers)

listing = rv[1]

while listing:

if not delimiter:

marker = listing[-1]['name']

else:

marker = listing[-1].get('name', listing[-1].get('subdir'))

version_marker = listing[-1].get('version_id')

listing = get_container(url, token, container, marker, limit,

prefix, delimiter, end_marker,

version_marker, path, http_conn,

service_token=service_token,

headers=headers)[1]

if listing:

rv[1].extend(listing)

return rv

parsed, conn = http_conn

cont_path = '%s/%s' % (parsed.path, quote(container))

qs = 'format=json'

if marker:

qs += '&marker=%s' % quote(marker)

if limit:

qs += '&limit=%d' % limit

if prefix:

qs += '&prefix=%s' % quote(prefix)

if delimiter:

qs += '&delimiter=%s' % quote(delimiter)

if end_marker:

qs += '&end_marker=%s' % quote(end_marker)

if version_marker:

qs += '&version_marker=%s' % quote(version_marker)

if path:

qs += '&path=%s' % quote(path)

if query_string:

qs += '&%s' % query_string.lstrip('?')

req_headers = {'X-Auth-Token': token, 'Accept-Encoding': 'gzip'}

if service_token:

req_headers['X-Service-Token'] = service_token

if headers:

req_headers.update(headers)