Sitelet https://github.com/openstack/python-swiftclient/commit/ed6fd60915fed7fd2a319fbbf1daec5f04169ebf
Skip to content

Commit ed6fd60

Browse files
claygyanxiaonv
authored and
tipabu
committed
Fix swiftclient output regression
Fix swiftclient output regression introduced by the related change: - output for SLO object download: fix incorrect error from SwiftReader about SLO object ETag header not matching MD5 checksum - output for object stat: fix duplicated ETag - output for account/container stat: fix duplicated byte/object counts Co-Authored-By: Yan Xiao <yanxiao@nvidia.com> Related-Change: Ice9cc9fe68684563f18ee527996e5a4292230a96 Change-Id: I5b2d79f89d1b6016de69d6b58879e5c2ef31e107
1 parent ce4fb27 commit ed6fd60

5 files changed

Lines changed: 79 additions & 10 deletions

File tree

‎swiftclient/client.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,15 @@ def encode_meta_headers(headers):
212212
return ret
213213

214214

215+
class LowerKeyCaseInsensitiveDict(CaseInsensitiveDict):
216+
"""
217+
CaseInsensitiveDict returning lower case keys for items()
218+
"""
219+
220+
def __iter__(self):
221+
return iter(self._store.keys())
222+
223+
215224
class _ObjectBody:
216225
"""
217226
Readable and iterable object body response wrapper.
@@ -738,7 +747,7 @@ def get_auth(auth_url, user, key, **kwargs):
738747

739748

740749
def resp_header_dict(resp):
741-
resp_headers = CaseInsensitiveDict()
750+
resp_headers = LowerKeyCaseInsensitiveDict()
742751
for header, value in resp.getheaders():
743752
header = parse_header_string(header)
744753
resp_headers[header] = parse_header_string(value)

‎test/unit/test_command_helpers.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from unittest import mock
1919

2020
from swiftclient import command_helpers as h
21+
from swiftclient.client import LowerKeyCaseInsensitiveDict
2122
from swiftclient.multithreading import OutputManager
2223

2324

@@ -245,5 +246,32 @@ def test_stat_object_verbose(self):
245246
ETag: 68b329da9893e34099c7d8ad5cb9c940
246247
Meta Color: blue
247248
Content-Encoding: gzip
249+
"""
250+
self.assertOut(expected)
251+
252+
def test_stat_object_case_insensitive_headers(self):
253+
self.options['verbose'] += 1
254+
# stub head object request
255+
stub_headers = LowerKeyCaseInsensitiveDict({
256+
'content-length': 2 ** 20,
257+
'x-object-meta-color': 'blue',
258+
'ETag': '68b329da9893e34099c7d8ad5cb9c940',
259+
'content-encoding': 'gzip',
260+
})
261+
self.conn.head_object.return_value = stub_headers
262+
args = ('c', 'o')
263+
with self.output_manager as output_manager:
264+
items, headers = h.stat_object(self.conn, self.options, *args)
265+
h.print_object_stats(items, headers, output_manager)
266+
expected = """
267+
URL: http://storage/v1/a/c/o
268+
Auth Token: tk12345
269+
Account: a
270+
Container: c
271+
Object: o
272+
Content Length: 1048576
273+
ETag: 68b329da9893e34099c7d8ad5cb9c940
274+
Meta Color: blue
275+
Content-Encoding: gzip
248276
"""
249277
self.assertOut(expected)

‎test/unit/test_service.py‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,13 @@
2727
from concurrent.futures import Future
2828
from hashlib import md5
2929
from queue import Queue, Empty as QueueEmptyError
30-
from requests.structures import CaseInsensitiveDict
3130
from time import sleep
3231

3332
import swiftclient
3433
import swiftclient.utils as utils
35-
from swiftclient.client import Connection, ClientException
34+
from swiftclient.client import (
35+
Connection, ClientException, LowerKeyCaseInsensitiveDict
36+
)
3637
from swiftclient.service import (
3738
SwiftService, SwiftError, SwiftUploadObject, SwiftDeleteObject
3839
)
@@ -242,6 +243,26 @@ def _consume(sr):
242243
self.assertEqual(sr._actual_md5.hexdigest(),
243244
md5('abc'.encode() * 3).hexdigest())
244245

246+
def test_swift_reader_knows_slo_etag_is_not_md5(self):
247+
segment_bodies = [b'abc', b'def', b'ghi']
248+
# slo etag is md5 of the sum of md5 of segments
249+
slo_etag = md5(b''.join(
250+
md5(b).hexdigest().encode()
251+
for b in segment_bodies
252+
)).hexdigest()
253+
headers = LowerKeyCaseInsensitiveDict({
254+
'Content-Length': len(b''.join(segment_bodies)),
255+
'X-Static-Large-Object': 'true',
256+
'ETag': '"%s"' % slo_etag
257+
})
258+
sr = self.sr('path', segment_bodies, headers)
259+
# x-static-large-object; so no exception is raised!
260+
actual_md5 = md5(b''.join(sr)).hexdigest()
261+
self.assertEqual(sr._actual_read, 9)
262+
self.assertIsNone(sr._actual_md5)
263+
self.assertEqual(actual_md5,
264+
md5(b''.join(segment_bodies)).hexdigest())
265+
245266

246267
class _TestServiceBase(unittest.TestCase):
247268
def _get_mock_connection(self, attempts=2):
@@ -674,7 +695,7 @@ def test_swifterror_creation(self):
674695
def test_swifterror_clientexception_creation(self):
675696
test_exc = ClientException(
676697
Exception('test exc'),
677-
http_response_headers=CaseInsensitiveDict({
698+
http_response_headers=LowerKeyCaseInsensitiveDict({
678699
'x-trans-id': 'someTransId'})
679700
)
680701
se = SwiftError(5, 'con', 'obj', 'seg', test_exc)

‎test/unit/test_shell.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import json
2323
import logging
2424
import os
25-
from requests.structures import CaseInsensitiveDict
2625
import tempfile
2726
import unittest
2827
from unittest import mock
@@ -32,6 +31,7 @@
3231
from urllib3.exceptions import HTTPError
3332

3433
import swiftclient
34+
from swiftclient.client import LowerKeyCaseInsensitiveDict
3535
from swiftclient.service import SwiftError
3636
import swiftclient.shell
3737
import swiftclient.utils
@@ -245,7 +245,7 @@ def test_stat_container_not_found(self, connection):
245245
swiftclient.ClientException(
246246
'test',
247247
http_status=404,
248-
http_response_headers=CaseInsensitiveDict({
248+
http_response_headers=LowerKeyCaseInsensitiveDict({
249249
'x-trans-id': 'someTransId'})
250250
)
251251
argv = ["", "stat", "container"]
@@ -344,7 +344,7 @@ def test_stat_object_not_found(self, connection):
344344
connection.return_value.head_object.side_effect = \
345345
swiftclient.ClientException(
346346
'test', http_status=404,
347-
http_response_headers=CaseInsensitiveDict({
347+
http_response_headers=LowerKeyCaseInsensitiveDict({
348348
'x-trans-id': 'someTransId'})
349349
)
350350
argv = ["", "stat", "container", "object"]
@@ -791,7 +791,7 @@ def _fake_retry(conn, *args, **kwargs):
791791

792792
body = mock.MagicMock()
793793
body.resp.read.side_effect = RequestException('test_exc')
794-
return (CaseInsensitiveDict({
794+
return (LowerKeyCaseInsensitiveDict({
795795
'content-type': 'text/plain',
796796
'etag': '2cbbfe139a744d6abbe695e17f3c1991',
797797
'x-trans-id': 'someTransId'}),
@@ -841,7 +841,7 @@ def _fake_retry(conn, *args, **kwargs):
841841

842842
body = mock.MagicMock()
843843
body.__iter__.side_effect = RequestException('test_exc')
844-
return (CaseInsensitiveDict({
844+
return (LowerKeyCaseInsensitiveDict({
845845
'content-type': 'text/plain',
846846
'etag': '2cbbfe139a744d6abbe695e17f3c1991',
847847
'x-trans-id': 'someTransId'}),
@@ -871,7 +871,7 @@ def _fake_retry(conn, *args, **kwargs):
871871
def test_download_bad_content_length(self, connection):
872872
objcontent = io.BytesIO(b'objcontent')
873873
connection.return_value.get_object.side_effect = [
874-
(CaseInsensitiveDict({
874+
(LowerKeyCaseInsensitiveDict({
875875
'content-type': 'text/plain',
876876
'content-length': 'BAD',
877877
'etag': '2cbbfe139a744d6abbe695e17f3c1991',

‎test/unit/test_swiftclient.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1117,6 +1117,17 @@ def test_response_headers(self):
11171117
self.assertEqual('t\xe9st', headers.get('x-utf-8-header', ''))
11181118
self.assertEqual('%ff', headers.get('x-non-utf-8-header', ''))
11191119
self.assertEqual('%FF', headers.get('x-binary-header', ''))
1120+
for k, v in headers.items():
1121+
# N.B. k is always lower case!
1122+
self.assertTrue(k.islower())
1123+
for k in headers.keys():
1124+
# N.B. k is always lower case!
1125+
self.assertTrue(k.islower())
1126+
self.assertTrue(set([
1127+
'x-utf-8-header',
1128+
'x-non-utf-8-header',
1129+
'x-binary-header',
1130+
]).intersection(headers))
11201131

11211132
self.assertEqual('t\xe9st', headers.get('X-Utf-8-Header', ''))
11221133
self.assertEqual('%ff', headers.get('X-Non-Utf-8-Header', ''))

0 commit comments

Comments
 (0)